JoinMemory 解决数据装配问题一

业务场景

• 我的订单：查询用户的全部订单，包括订单信息、用户信息、地址信息等；
• 订单详情：查询某一个订单的详细信息，包括订单信息、用户信息、支付信息等；

常规处理方案

数据库 Join

• 最简单的方案
• 可能会产生慢 SQL 问题、以及分库分表情况下无法进行数据库 Join

Foreach + 单条抓取

• 查询出主表数据后，遍历主表数据在内存中进行数据装配
• Foreach 会导致查询数据库次数过多

/**
 * @author coderxdh
 * @date 2023/8/6 15:22
 */
public interface JoinMemoryService {
    List<? extends OrderListVO> getByUserId(Long userId);

    OrderDetailVO getDetailByOrderId(Long orderId);
}

/**
 * @author coderxdh
 * @date 2023/8/6 15:36
 */

@Slf4j
@Service
public class JoinMemoryServiceV1Impl implements JoinMemoryService {

    @Resource
    private OrderMapper orderMapper;

    @Resource
    private UserMapper userMapper;

    @Resource
    private AddressMapper addressMapper;

    @Resource
    private PayInfoMapper payInfoMapper;

    @Override
    public List<? extends OrderListVO> getByUserId(Long userId) {
        List<OrderPO> orderList = orderMapper.getByUserId(userId);
        return orderList.stream()
                .map(this::converterToOrderListVO)
                .collect(Collectors.toList());
    }

    @Override
    public OrderDetailVO getDetailByOrderId(Long orderId) {
        OrderPO order = this.orderMapper.getOrderById(orderId);
        return converterToOrderDetailVO(order);
    }

    private OrderDetailVO converterToOrderDetailVO(OrderPO order) {
        OrderVO orderVO = OrderVO.apply(order);
        UserPO user = this.userMapper.getUserById(order.getUserId());
        UserVO userVO = UserVO.apply(user);

        AddressPO address = this.addressMapper.getAddressById(order.getAddressId());
        AddressVO addressVO = AddressVO.apply(address);

        List<PayInfoPO> payInfoList = this.payInfoMapper.getPayInfoByOrderId(order.getId());
        List<PayInfoVO> payInfoVOList = PayInfoVO.apply(payInfoList);


        OrderDetailVOV1 orderDetailVO = new OrderDetailVOV1();
        orderDetailVO.setOrder(orderVO);
        orderDetailVO.setUser(userVO);
        orderDetailVO.setAddress(addressVO);
        orderDetailVO.setPayInfo(payInfoVOList);

        return orderDetailVO;
    }

    private OrderListVOV1 converterToOrderListVO(OrderPO order) {
        OrderVO orderVO = OrderVO.apply(order);
        UserPO user = this.userMapper.getUserById(order.getUserId());
        UserVO userVO = UserVO.apply(user);

        AddressPO address = this.addressMapper.getAddressById(order.getAddressId());
        AddressVO addressVO = AddressVO.apply(address);

        OrderListVOV1 orderListVO = new OrderListVOV1();
        orderListVO.setOrder(orderVO);
        orderListVO.setUser(userVO);
        orderListVO.setAddress(addressVO);

        return orderListVO;
    }
}

批量查询+内存Join

• 查询出主表数据后，取出主表数据中相关的外键，通过这些外键批量的去其他表进行查询，获取数据后在内存中进行数据装配
• 减少了数据库交互次数

/**
 * @author coderxdh
 * @date 2023/8/6 15:36
 */
@Slf4j
@Service
public class JoinMemoryServiceV2Impl implements JoinMemoryService {

    @Resource
    private OrderMapper orderMapper;

    @Resource
    private UserMapper userMapper;

    @Resource
    private AddressMapper addressMapper;

    @Resource
    private PayInfoMapper payInfoMapper;

    @Override
    public List<? extends OrderListVO> getByUserId(Long userId) {
        List<OrderPO> orders = this.orderMapper.getByUserId(userId);

        List<OrderListVOV2> orderListVOS = orders.stream()
                .map(order -> {
                    OrderListVOV2 orderDetailVO= new OrderListVOV2();
                    orderDetailVO.setOrder(OrderVO.apply(order));
                    return orderDetailVO;
                })
                .collect(toList());

        List<Long> userIds = orders.stream()
                .map(OrderPO::getUserId)
                .collect(toList());
        List<UserPO> users = this.userMapper.getByIds(userIds);
        Map<Long, UserPO> userMap = users.stream()
                .collect(toMap(UserPO::getId, Function.identity(), (a, b) -> a));
        for (OrderListVOV2 item : orderListVOS){
            UserPO user = userMap.get(item.getOrder().getUserId());
            UserVO userVO = UserVO.apply(user);
            item.setUser(userVO);
        }

        List<Long> addressIds = orders.stream()
                .map(OrderPO::getAddressId)
                .collect(toList());
        List<AddressPO> addresses = this.addressMapper.getByIds(addressIds);
        Map<Long, AddressPO> addressMap = addresses.stream()
                .collect(toMap(AddressPO::getId, Function.identity(), (a, b) -> a));
        for (OrderListVOV2 item : orderListVOS){
            AddressPO address = addressMap.get(item.getOrder().getAddressId());
            AddressVO addressVO = AddressVO.apply(address);
            item.setAddress(addressVO);
        }

        // 上面两部分是类似的代码，不变的是主流程，变的是每一个具体的对象。那么可以对不变的部分封装好

        return orderListVOS;
    }

    @Override
    public OrderDetailVO getDetailByOrderId(Long orderId) {
        return null;
    }
}

并发批量查询+内存Join

• 查询出主表数据后，取出主表数据中相关的外键，通过这些外键批量的去其他表进行查询，获取数据后在内存中进行数据装配，对于无关联的数据，可以采用并行的方式去查询，提高数据装配效率

/**
 * @author coderxdh
 * @date 2023/8/6 15:36
 */
@Slf4j
@Service
public class JoinMemoryServiceV3Impl implements JoinMemoryService {

    @Resource
    private OrderMapper orderMapper;

    @Resource
    private UserMapper userMapper;

    @Resource
    private AddressMapper addressMapper;

    private ExecutorService executorService;

    @PostConstruct
    public void init() {
        this.executorService = Executors.newFixedThreadPool(20);
    }

    @Override
    public List<? extends OrderListVO> getByUserId(Long userId) {
        List<OrderPO> orders = this.orderMapper.getByUserId(userId);

        List<OrderListVOV3> orderListVOS = orders.stream()
                .map(order -> {
                    OrderListVOV3 orderDetailVO = new OrderListVOV3();
                    orderDetailVO.setOrder(OrderVO.apply(order));
                    return orderDetailVO;
                })
                .collect(toList());

        List<Callable<Void>> callables = Lists.newArrayListWithCapacity(2);

        callables.add(() -> {
            buildUser(orders, orderListVOS);
            return null;
        });

        callables.add(() -> {
            buildAddress(orders, orderListVOS);
            return null;
        });

        try {
            this.executorService.invokeAll(callables);
        } catch (InterruptedException e) {
            throw new RuntimeException(e);
        }

        return orderListVOS;
    }

    @Override
    public OrderDetailVO getDetailByOrderId(Long orderId) {
        return null;
    }

    private void buildUser(List<OrderPO> orderPOList, List<OrderListVOV3> orderListVOList) {
        List<Long> userIds = orderPOList.stream()
                .map(OrderPO::getUserId)
                .collect(toList());

        List<UserPO> users = this.userMapper.getByIds(userIds);
        Map<Long, UserPO> userMap = users.stream()
                .collect(toMap(UserPO::getId, Function.identity(), (a, b) -> a));
        for (OrderListVOV3 item : orderListVOList) {
            UserPO user = userMap.get(item.getOrder().getUserId());
            UserVO userVO = UserVO.apply(user);
            item.setUser(userVO);
        }
    }

    public void buildAddress(List<OrderPO> orderPOList, List<OrderListVOV3> orderListVOList) {
        List<Long> addressIds = orderPOList.stream()
                .map(OrderPO::getAddressId)
                .collect(toList());
        List<AddressPO> addresses = this.addressMapper.getByIds(addressIds);
        Map<Long, AddressPO> addressMap = addresses.stream()
                .collect(toMap(AddressPO::getId, Function.identity(), (a, b) -> a));
        for (OrderListVOV3 item : orderListVOList) {
            AddressPO address = addressMap.get(item.getOrder().getAddressId());
            AddressVO addressVO = AddressVO.apply(address);
            item.setAddress(addressVO);
        }
    }
}

• 代码结构

升级处理方案

• 统一数据处理逻辑
• 使用注解简化代码
• 统一注解配置
• 增加并发处理能力

统一数据处理逻辑一

• 核心处理流程都是：1、通过主表的主键ID查询出数据，2、拿到这些数据中作为其他表的主键的字段，3、根据步骤2取出的字段去其他表进行查询，4、将查询结果转化为需要的类型后设置到返回结果中
• 那么可以对上述流程进行抽象，封装出一个 fetch 方法

/**
 * @author coderxdh
 * @date 2023/8/19 22:46
 */
@Component
public class JoinMemoryServiceFetcherV1 implements JoinMemoryService {

    @Resource
    private OrderMapper orderMapper;

    @Resource
    private UserVOFetcherExecutorV1 userVOFetcherExecutorV1;

    @Resource
    private AddressVOFetcherExecutorV1 addressVOFetcherExecutorV1;

    @Override
    public List<? extends OrderListVO> getByUserId(Long userId) {
        List<OrderPO> orders = this.orderMapper.getByUserId(userId);

        List<OrderListVOFetcherV1> orderListVOFetcherV1s = orders.stream()
                .map(orderPO -> {
                    OrderListVOFetcherV1 orderListVOFetcherV1 = new OrderListVOFetcherV1();
                    OrderVO apply = OrderVO.apply(orderPO);
                    orderListVOFetcherV1.setOrder(apply);
                    return orderListVOFetcherV1;
                }).collect(Collectors.toList());

        this.userVOFetcherExecutorV1.fetch(orderListVOFetcherV1s);
        this.addressVOFetcherExecutorV1.fetch(orderListVOFetcherV1s);

        return orderListVOFetcherV1s;
    }

    @Override
    public OrderDetailVO getDetailByOrderId(Long orderId) {
        return null;
    }
}

/**
 * @author coderxdh
 * @date 2023/8/19 22:50
 */
@Component
public class UserVOFetcherExecutorV1 {

    @Resource
    private UserMapper userMapper;

    public void fetch(List<? extends UserVOFetcherV1> fetchers) {
        // 获取主键 ids
        List<Long> ids = fetchers.stream()
                .map(UserVOFetcherV1::getUserId)
                .collect(Collectors.toList());

        // 根据主键 ids，批量查询
        List<UserPO> users = this.userMapper.getByIds(ids);

        // 拼成 map 结构
        Map<Long, UserPO> userMap = users.stream()
                .collect(Collectors.toMap(UserPO::getId, Function.identity(), (a, b) -> a));

        // 根据主键值，将数据装配到返回结果中
        fetchers.forEach(fetcher -> {
            Long userId = fetcher.getUserId();
            UserPO user = userMap.get(userId);
            if (Objects.nonNull(user)) {
                UserVO userVO = UserVO.apply(user);
                fetcher.setUser(userVO);
            }
        });
    }
}

/**
 * @author coderxdh
 * @date 2023/8/19 22:50
 */
@Component
public class AddressVOFetcherExecutorV1 {

    @Resource
    private AddressMapper addressMapper;

    public void fetch(List<? extends AddressVOFetcherV1> fetchers) {
        // 获取主键 ids
        List<Long> ids = fetchers.stream()
                .map(AddressVOFetcherV1::getAddressId)
                .collect(Collectors.toList());

        // 根据主键 ids，批量查询
        List<AddressPO> addresses = this.addressMapper.getByIds(ids);

        // 拼成 map 结构
        Map<Long, AddressPO> addressMap = addresses.stream()
                .collect(toMap(AddressPO::getId, Function.identity(), (a, b) -> a));

        // 根据主键值，将数据装配到返回结果中
        fetchers.forEach(fetcher -> {
            AddressPO addressPO = addressMap.get(fetcher.getAddressId());
            if (Objects.nonNull(addressPO)) {
                AddressVO apply = AddressVO.apply(addressPO);
                fetcher.setAddress(apply);
            }
        });
    }
}

抽象数据处理逻辑(重要!!!)

可以发现，在上述的 AddressVOFetcherExecutorV1 和 UserVOFetcherExecutorV1 的 fetch 方法中，其处理流程是一样的，那么就可以对上述的几个步骤再次进行抽象

• 定义一个空接口，其作用是为了聚合某一些类

/**
 * @author coderxdh
 * @date 2023/8/19 22:15
 */
public interface ItemFetcher {
    // 定义一个空接口，其作用是为了"聚合"某一些类
}

• 执行器接口

/**
 * @author coderxdh
 * @date 2023/8/19 22:15
 */
public interface ItemFetcherExecutor<F extends ItemFetcher> {

    boolean support(Class<F> cls);

    void fetch(List<F>fetchers);
}

• 执行器接口的抽象实现类，是为了编排一系列的抽象方法，完成 fetch 的功能。定义的抽象方法则在具体的实现类中进行自定义实现

/**
 * @author coderxdh
 * @date 2023/8/19 22:16
 */
public abstract class BaseItemFetcherExecutor<FETCHER extends ItemFetcher, DATA, RESULT> implements ItemFetcherExecutor<FETCHER> {
    @Override
    public boolean support(Class<FETCHER> cls) {
        return true;
    }

    @Override
    public void fetch(List<FETCHER> fetchers) {
        List<Long> ids = fetchers.stream()
                .map(this::getFetchId)
                .distinct()
                .collect(Collectors.toList());

        List<DATA> datas = loadData(ids);

        Map<Long, List<DATA>> dataMap = datas.stream()
                .collect(groupingBy(this::getDataId));

        fetchers.forEach(fetcher -> {
            Long id = getFetchId(fetcher);
            List<DATA> ds = dataMap.get(id);
            if (!CollectionUtils.isEmpty(ds)) {
                List<RESULT> result = ds.stream()
                        .map(data -> convertToVo(data))
                        .collect(Collectors.toList());
                setResult(fetcher, result);
            }
        });
    }
    
    // 获取主键 ids
    protected abstract Long getFetchId(FETCHER fetcher);

    // 根据主键 ids 加载数据
    protected abstract List<DATA> loadData(List<Long> ids);

    // 获取主表中作为其他表的主键的字段
    protected abstract Long getDataId(DATA data);

    // 将查询结果转换为需要装配的类型
    protected abstract RESULT convertToVo(DATA data);

    // 将转换结果设置到需要返回的结果中
    protected abstract void setResult(FETCHER fetcher, List<RESULT> results);
}

• FetcherService，提供统一服务供外部调用。注入所有 ItemFetcherExecutor，遍历执行 fetch 方法，完成数据的装配

/**
 * @author coderxdh
 * @date 2023/8/20 0:02
 */
@Service
public class FetcherService {

    @Resource
    private List<ItemFetcherExecutor> itemFetcherExecutors;

    public <F extends  ItemFetcher> void fetch(Class<F> cls, List<F> fetchers) {
        if (!CollectionUtils.isEmpty(fetchers)) {
            this.itemFetcherExecutors.stream()
                    .filter(itemFetcherExecutor -> itemFetcherExecutor.support(cls))
                    .forEach(itemFetcherExecutor ->  itemFetcherExecutor.fetch(fetchers));
        }
    }
}

• 具体实现类-获取用户信息

/**
 * @author coderxdh
 * @date 2023/8/19 22:36
 */
public interface UserVOFetcherV2 extends ItemFetcher {

    Long getUserId();

    void setUser(UserVO user);
}

/**
 * @author coderxdh
 * @date 2023/8/19 22:37
 */
@Component
public class UserVOFetcherExecutorV2 extends BaseItemFetcherExecutor<UserVOFetcherV2, UserPO, UserVO> {

    @Resource
    private UserMapper userMapper;

    @Override
    protected Long getFetchId(UserVOFetcherV2 fetcher) {
        return fetcher.getUserId();
    }

    @Override
    protected List<UserPO> loadData(List<Long> ids) {
        return userMapper.getByIds(ids);
    }

    @Override
    protected Long getDataId(UserPO userPO) {
        return userPO.getId();
    }

    @Override
    protected UserVO convertToVo(UserPO userPO) {
        return UserVO.apply(userPO);
    }

    @Override
    protected void setResult(UserVOFetcherV2 fetcher, List<UserVO> userVOS) {
        if (!CollectionUtils.isEmpty(userVOS)) {
            fetcher.setUser(userVOS.get(0));
        }
    }
}

• 具体实现类-获取地址信息

/**
 * @author coderxdh
 * @date 2023/8/20 0:27
 */
public interface AddressVOFetcherV2 extends ItemFetcher{

    Long getAddressId();

    void setAddress(AddressVO address);
}

/**
 * @author coderxdh
 * @date 2023/8/19 22:37
 */
@Component
public class AddressVOFetcherExecutorV2 extends BaseItemFetcherExecutor<AddressVOFetcherV2, AddressPO, AddressVO> {

    @Resource
    private AddressMapper addressMapper;

    @Override
    protected Long getFetchId(AddressVOFetcherV2 fetcher) {
        return fetcher.getAddressId();
    }

    @Override
    protected List<AddressPO> loadData(List<Long> ids) {
        return addressMapper.getByIds(ids);
    }

    @Override
    protected Long getDataId(AddressPO addressPO) {
        return addressPO.getId();
    }

    @Override
    protected AddressVO convertToVo(AddressPO addressPO) {
        return AddressVO.apply(addressPO);
    }

    @Override
    protected void setResult(AddressVOFetcherV2 fetcher, List<AddressVO> addressVOS) {
        if (!CollectionUtils.isEmpty(addressVOS)) {
            fetcher.setAddress(addressVOS.get(0));
        }
    }
}

• 调用服务

/**
 * @author coderxdh
 * @date 2023/8/19 22:46
 */
@Component
public class JoinMemoryServiceFetcherV2 implements JoinMemoryService {

    @Resource
    private OrderMapper orderMapper;

    @Resource
    private UserVOFetcherExecutorV1 userVOFetcherExecutorV1;

    @Resource
    private AddressVOFetcherExecutorV1 addressVOFetcherExecutorV1;

    @Resource
    private FetcherService fetcherService;

    @Override
    public List<? extends OrderListVO> getByUserId(Long userId) {
        List<OrderPO> orders = this.orderMapper.getByUserId(userId);

        List<OrderListVOFetcherV2> orderListVOFetcherV2s = orders.stream()
                .map(orderPO -> {
                    OrderListVOFetcherV2 orderListVOFetcherV2 = new OrderListVOFetcherV2();
                    OrderVO apply = OrderVO.apply(orderPO);
                    orderListVOFetcherV2.setOrder(apply);
                    return orderListVOFetcherV2;
                }).collect(Collectors.toList());

        this.fetcherService.fetch(OrderListVOFetcherV2.class, orderListVOFetcherV2s);

        return orderListVOFetcherV2s;
    }

    @Override
    public OrderDetailVO getDetailByOrderId(Long orderId) {
        return null;
    }
}

并行处理

完成上述抽象封装后，可以再次考虑并行处理的场景，因此可以定义一个 ConcurrentFetcherService，对外提供并行处理的 fetch 方法

/**
 * @author coderxdh
 * @date 2023/8/20 0:02
 */
@Service
public class ConcurrentFetcherService {

    @Resource
    private List<ItemFetcherExecutor> itemFetcherExecutors;

    private ExecutorService executorService;

    @PostConstruct
    public void init() {
        this.executorService = Executors.newFixedThreadPool(20);
    }

    public <F extends ItemFetcher> void fetch(Class<F> cls, List<F> fetchers) {
        if (CollectionUtils.isEmpty(fetchers)) {
            return;
        }

        List<Callable<Void>> callables = new ArrayList<>(2);
        for (ItemFetcherExecutor<F> itemFetcherExecutor : itemFetcherExecutors) {
            if (itemFetcherExecutor.support(cls)) {
                callables.add(() -> {
                    itemFetcherExecutor.fetch(fetchers);
                    return null;
                });
            }
        }

        try {
            executorService.invokeAll(callables);
        } catch (InterruptedException e) {
            throw new RuntimeException(e);
        }
    }
}

代码结构

使用注解简化代码

• JoinMemory 解决数据装配问题二