一、为什么需要池化技术
池化技术是一种重要的资源管理策略，其应用广泛且效果显著，主要体现在以下几个方面：
1、节约资源
池化技术通过复用预先创建和初始化的资源，显著减少了资源的创建与销毁开销。例如，线程的创建和销毁、数据库连接的建立与关闭等操作都可能涉及耗时的系统调用和资源消耗。使用池化技术，如线程池和数据库连接池，可以预先创建一组资源实例，并在需要时从池中获取，使用完毕后归还到池中，以便后续的请求重复利用。这种方式避免了频繁的资源创建和销毁，从而节约了系统资源。
2、提高响应速度
池化技术通过预先初始化好的资源和重用现有资源，避免了每次请求时的初始化延迟。例如，在数据库连接池中，当应用程序需要访问数据库时，可以直接从连接池中获取一个已经建立好的连接，而无需等待新连接的建立。这大大提高了系统的响应速度，特别是在高并发场景下，能够显著减少用户的等待时间。
3、简化资源管理
池化技术将资源的创建、分配、重用和回收等过程集中管理，简化了资源管理的复杂性。通过监控池中资源的使用情况，我们可以及时调整池的大小和配置，以满足系统需求。此外，池化技术还可以提供资源的健康状态检查和回收策略，确保资源的有效利用和避免资源泄漏。
综上所述，程序中需要池化技术主要是出于资源节约、提高响应速度和简化资源管理和等方面的考虑。池化技术的应用场景广泛，包括但不限于线程池、内存池(kafka)、数据库连接池、网络请求连接池等领域，此处主要关注数据库druid连接池实现原理。
二、druid连接池初始化
1、DruidDataSource#createDruidDataSource
public DataSource createDruidDataSource(DataSourceProperty dataSourceProperty) {
//将配置参数转成DruidDataSource
DruidDataSource dataSource = new DruidDataSource();
dataSource.setUsername(dataSourceProperty.getUsername());
dataSource.setPassword(dataSourceProperty.getPassword());
dataSource.setUrl(dataSourceProperty.getUrl());
dataSource.setDriverClassName(dataSourceProperty.getDriverClassName());
dataSource.setName(dataSourceProperty.getPollName());
DruidConfig config = dataSourceProperty.getDruid();
Properties properties = config.toProperties(druidGlobalConfig);
String filters = properties.getProperty("druid.filters");
List<Filter> proxyFilters = new ArrayList<>(2);
//拦截器
if (!StringUtils.isEmpty(filters) && filters.contains("stat")) {
StatFilter statFilter = new StatFilter();
statFilter.configFromProperties(properties);
proxyFilters.add(statFilter);
}
if (!StringUtils.isEmpty(filters) && filters.contains("wall")) {
WallConfig wallConfig = DruidWallConfigUtil.toWallConfig(dataSourceProperty.getDruid().getWall(), druidGlobalConfig.getWall());
WallFilter wallFilter = new WallFilter();
wallFilter.setConfig(wallConfig);
proxyFilters.add(wallFilter);
}
//省略非关键代码
try {
//初始化数据源
dataSource.init();
} catch (SQLException e) {
log.error("druid数据源启动失败", e);
}
return dataSource;
}
2、DruidDataSource#init
public void init() throws SQLException {
//一个是数据源只初始化一次
//protected volatile boolean inited = false;
if (inited) {
return;
}
//默认为非公平锁
final ReentrantLock lock = this.lock;
try {
lock.lockInterruptibly();
} catch (InterruptedException e) {
throw new SQLException("interrupt", e);
}
boolean init = false;
try {
if (inited) {
return;
}
//找db类型 Mysql Oracle等
if (this.dbTypeName == null || this.dbTypeName.length() == 0) {
this.dbTypeName = JdbcUtils.getDbType(jdbcUrl, null);
}
//根据SPI加载Filter
initFromSPIServiceLoader();
//解析驱动
resolveDriver();
//初始化check
initCheck();
initExceptionSorter();
//初始化连接校验，不同的数据库校验方式不同
initValidConnectionChecker();
validationQueryCheck();
//省略非关键代码
dataSourceStat = new JdbcDataSourceStat(this.name, this.jdbcUrl, this.dbTypeName, this.connectProperties);
dataSourceStat.setResetStatEnable(this.resetStatEnable);
//生成连接管理数组
connections = new DruidConnectionHolder[maxActive];
evictConnections = new DruidConnectionHolder[maxActive];
keepAliveConnections = new DruidConnectionHolder[maxActive];
//异步初始化
if (createScheduler != null && asyncInit) {
for (int i = 0; i < initialSize; ++i) {
submitCreateTask(true);
}
} else if (!asyncInit) {
// init connections
while (poolingCount < initialSize) {
try {
//创建连接并放在数组中
PhysicalConnectionInfo pyConnectInfo = createPhysicalConnection();
DruidConnectionHolder holder = new DruidConnectionHolder(this, pyConnectInfo);
connections[poolingCount++] = holder;
} catch (SQLException ex) {
//异常后是否继续初始化
if (initExceptionThrow) {
connectError = ex;
break;
} else {
Thread.sleep(3000);
}
}
}
if (poolingCount > 0) {
poolingPeak = poolingCount;
poolingPeakTime = System.currentTimeMillis();
}
}
//创建三个线程
createAndLogThread();
//连接创建线程
createAndStartCreatorThread();
//连接销毁小程
createAndStartDestroyThread();
//初始化数量为2，创建和销毁线程里面分别消耗1
initedLatch.await();
init = true;
initedTime = new Date();
registerMbean();
if (connectError != null && poolingCount == 0) {
throw connectError;
}
if (keepAlive) {
// async fill to minIdle
if (createScheduler != null) {
for (int i = 0; i < minIdle; ++i) {
submitCreateTask(true);
}
} else {
this.emptySignal();
}
}
} catch (SQLException e) {
throw e;
} catch (InterruptedException e) {
throw new SQLException(e.getMessage(), e);
} catch (RuntimeException e){
throw e;
} catch (Error e){
throw e;
} finally {
inited = true;
//释放锁
lock.unlock();
}
}
3、createPhysicalConnection
先遍历Filter，然后利用Driver生成连接：
public Connection createPhysicalConnection(String url, Properties info) throws SQLException {
Connection conn;
if (getProxyFilters().size() == 0) {
conn = getDriver().connect(url, info);
} else {
//两个Filter
conn = new FilterChainImpl(this).connection_connect(info);
}
createCountUpdater.incrementAndGet(this);
return conn;
}
public ConnectionProxy connection_connect(Properties info) throws SQLException {
//先执行过滤器（StatFiler、WallFilter）,各个Filter需要连接信息
if (this.pos < filterSize) {
return nextFilter()
.connection_connect(this, info);
}
//执行完过滤器后根据驱动获取连接
Driver driver = dataSource.getRawDriver();
String url = dataSource.getRawJdbcUrl();
Connection nativeConnection = driver.connect(url, info);
if (nativeConnection == null) {
return null;
}
//组装连接信息
return new ConnectionProxyImpl(dataSource, nativeConnection, info, dataSource.createConnectionId());
}
4、CreateConnectionThread
CreateConnectionThread：创建连接池连接线程，用于连接池连接不足时创建连接，下面重点分析：
public class CreateConnectionThread extends Thread {
public CreateConnectionThread(String name){
super(name);
this.setDaemon(true);
}
public void run() {
//用于主线程等待此线程已经启动
initedLatch.countDown();
long lastDiscardCount = 0;
int errorCount = 0;
for (;;) {
// addLast
try {
lock.lockInterruptibly();
} catch (InterruptedException e2) {
break;
}
long discardCount = DruidDataSource.this.discardCount;
boolean discardChanged = discardCount - l
astDiscardCount > 0;
lastDiscardCount = discardCount;
try {
boolean emptyWait = true;
if (createError != null && poolingCount == 0&& !discardChanged) {
emptyWait = false;
}
if (emptyWait&& asyncInit && createCount < initialSize) {
emptyWait = false;
}
if (emptyWait) {
// 必须存在线程等待，才创建连接
if (poolingCount >= notEmptyWaitThreadCount && (!(keepAlive && activeCount + poolingCount < minIdle))&& !isFailContinuous()) {
empty.await();
}
// 防止创建超过maxActive数量的连接
if (activeCount + poolingCount >= maxActive) {
empty.await();
continue;
}
}
} catch (InterruptedException e) {
lastCreateError = e;
lastErrorTimeMillis = System.currentTimeMillis();
if (!closing) {
LOG.error("create connection Thread Interrupted, url: " + jdbcUrl, e);
}
break;
} finally {
lock.unlock();
}
PhysicalConnectionInfo connection = null;
try {
connection = createPhysicalConnection();
} catch (SQLException e) {
errorCount++;
if (errorCount > connectionErrorRetryAttempts && timeBetweenConnectErrorMillis > 0) {
// fail over retry attempts
setFailContinuous(true);
if (failFast) {
lock.lock();
try {
notEmpty.signalAll();
} finally {
lock.unlock();
}
}
if (breakAfterAcquireFailure) {
break;
}
try {
Thread.sleep(timeBetweenConnectErrorMillis);
} catch (InterruptedException interruptEx) {
break;
}
}
} catch (RuntimeException e) {
} catch (Error e) {
}
if (connection == null) {
continue;
}
//存入连接数组中，并唤醒notEmpty
boolean result = put(connection);
//省略非关键代码
}
}
}
private boolean put(DruidConnectionHolder holder, long createTaskId) {
lock.lock();
try {
if (poolingCount >= maxActive) {
if (createScheduler != null) {
clearCreateTask(createTaskId);
}
return false;
}
connections[poolingCount] = holder;
incrementPoolingCount();
if (poolingCount > poolingPeak) {
poolingPeak = poolingCount;
poolingPeakTime = System.currentTimeMillis();
}
//唤醒阻塞在非空等待
notEmpty.signal();
notEmptySignalCount++;
//省略异步创建
} finally {
lock.unlock();
}
return true;
}
5、createAndStartDestroyThread
检测连接，判断是否需要回收处理：
public class DestroyTask implements Runnable {
public DestroyTask() {
}
@Override
public void run() {
//缩减连接
shrink(true, keepAlive);
//连接泄漏检测：removeAbandoned
// 连接回收的超时时间，默认5分钟；
if (isRemoveAbandoned()) {
removeAbandoned();
}
}
}
public void shrink(boolean checkTime, boolean keepAlive) {
try {
lock.lockInterruptibly();
} catch (InterruptedException e) {
return;
}
boolean needFill = false;
int evictCount = 0;
int keepAliveCount = 0;
int fatalErrorIncrement = fatalErrorCount - fatalErrorCountLastShrink;
fatalErrorCountLastShrink = fatalErrorCount;
try {
if (!inited) {
return;
}
//检测数量为总连接数数减去最小空闲数
final int checkCount = poolingCount - minIdle;
final long currentTimeMillis = System.currentTimeMillis();
for (int i = 0; i < poolingCount; ++i) {
DruidConnectionHolder connection = connections[i];
//发生错误且上一次重大错误时间大于连接时间，则需要再次检测保活
if ((onFatalError || fatalErrorIncrement > 0) && (lastFatalErrorTimeMillis > connection.connectTimeMillis)) {
keepAliveConnections[keepAliveCount++] = connection;
continue;
}
if (checkTime) {
//省略非关键代码
} else {
//小于checkCount则直接放入驱逐连接中
if (i < checkCount) {
evictConnections[evictCount++] = connection;
} else {
break;
}
}
}
int removeCount = evictCount + keepAliveCount;
if (removeCount > 0) {
//将poolingCount - removeCount数前移到0开始的位置，然后将poolingCount - removeCount往 //后重置为null
System.arraycopy(connections, removeCount, connections, 0, poolingCount - removeCount);
Arrays.fill(connections, poolingCount - removeCount, poolingCount, null);
poolingCount -= removeCount;
}
keepAliveCheckCount += keepAliveCount;
//如果需要保活且当前连接数小于最小空闲数，则需要再次创建连接
if (keepAlive && poolingCount + activeCount < minIdle) {
needFill = true;
}
} finally {
lock.unlock();
}
//驱逐关闭处理
if (evictCount > 0) {
for (int i = 0; i < evictCount; ++i) {
DruidConnectionHolder item = evictConnections[i];
Connection connection = item.getConnection();
JdbcUtils.close(connection);
destroyCountUpdater.incrementAndGet(this);
}
Arrays.fill(evictConnections, null);
}
//先检测连接如果正常则继续使用，否则进行关闭
if (keepAliveCount > 0) {
// keep order
for (int i = keepAliveCount - 1; i >= 0; --i) {
DruidConnectionHolder holer = keepAliveConnections[i];
Connection connection = holer.getConnection();
holer.incrementKeepAliveCheckCount();
boolean validate = false;
try {
this.validateConnection(connection);
validate = true;
} catch (Throwable error) {
}
boolean discard = !validate;
if (validate) {
holer.lastKeepTimeMillis = System.currentTimeMillis();
boolean putOk = put(holer, 0L);
if (!putOk) {
discard = true;
}
}
if (discard) {
try {
connection.close();
} catch (Exception e) {
// skip
}
lock.lock();
try {
discardCount++;
if (activeCount + poolingCount <= minIdle) {
emptySignal();
}
} finally {
lock.unlock();
}
}
}
this.getDataSourceStat().addKeepAliveCheckCount(keepAliveCount);
Arrays.fill(keepAliveConnections, null);
}
//需要补充连接则唤醒连接生产线程
if (needFill) {
lock.lock();
try {
int fillCount = minIdle - (activeCount + poolingCount + createTaskCount);
for (int i = 0; i < fillCount; ++i) {
emptySignal();
}
} finally {
lock.unlock();
}
} else if (onFatalError || fatalErrorIncrement > 0) {
lock.lock();
try {
emptySignal();
} finally {
lock.unlock();
}
}
}
以上整个初始化过程流程图示意图如下：
三、获取连接
1、DruidDataSource#getConnection
因启动时已经完成初始化，inited为true则直接返回，存在两个Filter，则执行两个Filter里面的增强逻辑，执行完后执行dataSource.getConnectionDirect方法。
public DruidPooledConnection getConnection(long maxWaitMillis) throws SQLException {
init();
if (filters.size() > 0) {
FilterChainImpl filterChain = new FilterChainImpl(this);
return filterChain.dataSource_connect(this, maxWaitMillis);
} else {
return getConnectionDirect(maxWaitMillis);
}
}
public DruidPooledConnection dataSource_connect(DruidDataSource dataSource, long maxWaitMillis) throws SQLException {
//可以先不关注每个Filter里面的逻辑，执行
if (this.pos < filterSize) {
DruidPooledConnection conn = nextFilter().dataSource_getConnection(this, dataSource, maxWaitMillis);
return conn;
}
return dataSource.getConnectionDirect(maxWaitMillis);
}
2、DruidDataSource#getConnectionDirect
获取连接后根据条件对连接进行校验，如果连接不可用则丢弃并进行相应的后置处理。
public DruidPooledConnection getConnectionDirect(long maxWaitMillis) throws SQLException {
int notFullTimeoutRetryCnt = 0;
for (;;) {
// handle notFullTimeoutRetry
DruidPooledConnection poolableConnection;
try {
//获取连接，后面重点分析
poolableConnection = getConnectionInternal(maxWaitMillis);
} catch (GetConnectionTimeoutException ex) {
//超时重试次数是否超过阈值且线程池未满，如果是则重试否则抛出异常
if (notFullTimeoutRetryCnt <= this.notFullTimeoutRetryCount && !isFull()) {
notFullTimeoutRetryCnt++;
continue;
}
throw ex;
}
//获取连接时是否校验
if (testOnBorrow) {
boolean validate = testConnectionInternal(poolableConnection.holder, poolableConnection.conn);
if (!validate) {
//释放连接
discardConnection(poolableConnection.holder);
continue;
}
} else {
if (poolableConnection.conn.isClosed()) {
//传入null，避免重复关闭
discardConnection(poolableConnection.holder);
continue;
}
// 如果配置testOnBorrow = fasle但testWhileIdle = true则判断连接空闲时间是否大于等于timeBetweenEvictionRunsMillis 如果是，则校验连接的有效性
if (testWhileIdle) {
final DruidConnectionHolder holder = poolableConnection.holder;
long currentTimeMillis = System.currentTimeMillis();
//上一次连接活跃时间
long lastActiveTimeMillis = holder.lastActiveTimeMillis;
//上一次执行时间
long lastExecTimeMillis= holder.lastExecTimeMillis;
//上一次保活时间
long lastKeepTimeMillis = holder.lastKeepTimeMillis;
//优先使用执行时间
if (checkExecuteTime&& lastExecTimeMillis != lastActiveTimeMillis) {
lastActiveTimeMillis = lastExecTimeMillis;
}
//优先使用保活时间
if (lastKeepTimeMillis > lastActiveTimeMillis) {
lastActiveTimeMillis = lastKeepTimeMillis;
}
long idleMillis = currentTimeMillis - lastActiveTimeMillis;
long timeBetweenEvictionRunsMillis = this.timeBetweenEvictionRunsMillis;
if (timeBetweenEvictionRunsMillis <= 0) {
timeBetweenEvictionRunsMillis = DEFAULT_TIME_BETWEEN_EVICTION_RUNS_MILLIS;
}
//空闲时间超过阈值则检测连接
if (idleMillis >= timeBetweenEvictionRunsMillis|| idleMillis < 0 ) {
boolean validate = testConnectionInternal(poolableConnection.holder, poolableConnection.conn);
if (!validate) {
discardConnection(poolableConnection.holder);
continue;
}
}
}
}
//省略非关键代码
return poolableConnection;
}
}
3、DruidDataSource#pollLast
当获取连接有等待时间时则执行pollLast，否则执行takeLast，此处以pollLast为例分析：
//初始化时生成锁和两个对象监视器
public DruidAbstractDataSource(boolean lockFair){
lock = new ReentrantLock(lockFair);
notEmpty = lock.newCondition();
empty = lock.newCondition();
}
private DruidConnectionHolder pollLast(long nanos) throws InterruptedException, SQLException {
long estimate = nanos;
for (;;) {
//如果连接池空了则唤醒创建线程
if (poolingCount == 0) {
emptySignal(); // send signal to CreateThread create connection
if (failFast && isFailContinuous()) {
throw new DataSourceNotAvailableException(createError);
}
if (estimate <= 0) {
waitNanosLocal.set(nanos - estimate);
return null;
}
notEmptyWaitThreadCount++;
if (notEmptyWaitThreadCount > notEmptyWaitThreadPeak) {
notEmptyWaitThreadPeak = notEmptyWaitThreadCount;
}
try {
long startEstimate = estimate;
//最多等待estimate，返回还剩多少时间
estimate = notEmpty.awaitNanos(estimate); // signal by recycle or creator
notEmptyWaitCount++;
notEmptyWaitNanos += (startEstimate - estimate);
//省略非关键代码
} catch (InterruptedException ie) {
notEmpty.signal(); // propagate to non-interrupted thread
notEmptySignalCount++;
throw ie;
} finally {
notEmptyWaitThreadCount--;
}
if (poolingCount == 0) {
//如果等待时间大于0则继续等待，否则返回null
if (estimate > 0) {
continue;
}
waitNanosLocal.set(nanos - estimate);
return null;
}
}
decrementPoolingCount();
//取出连接
DruidConnectionHolder last = connections[poolingCount];
connections[poolingCount] = null;
long waitNanos = nanos - estimate;
last.setLastNotEmptyWaitNanos(waitNanos);
return last;
}
}
四、连接池生产消费者模型
通过以上分析得知：由业务线程、CreateConnectionThread、DestroyConnectionThread组成生产消费者模型，通过ReentrentLock的两个Condition：empty和notEmpty，来实现生产者和消费者的阻塞和通知：
五、总结
通过分析druid连接池原理，进一步熟悉连接池各个核心参数的作用，用于指导核心参数的配置，同时可以带着同样的思路拓展到内存池等池化技术。
