如何使用 Yii2 Model 操作 ElasticSearch

谷人希
1,307 阅读2分钟

ElasticSearch 就不多介绍了、ES 的查询语法是 Query DSL 那么我们想要在 Yii2 中像操作 MySQL 一样操作 ES，使用 Yii2 强大的 AR MODEL 就需要重写 Query 方法、将 SQL 转为 DSL 。按照我下面的操作步骤你就能使用 Yii2 的 Model 愉快的查询 ES 数据而无需关心 DSL。

安装 ES SQL 解析插件

具体安装步骤：github.com/NLPchina/el…

创建 ES 组件

<?php
namespace app\components;

use app\esmodels\BaseQuery;
use app\utils\CommonConsts;
use app\utils\Helper;
use Elasticsearch\Client;
use Elasticsearch\ClientBuilder;
use GuzzleHttp\Exception\GuzzleException;
use yii\base\Component;
use yii\db\Exception;
use yii\helpers\ArrayHelper;

class ElasticSearch extends Component
{
    public $hosts;

    public $maxRetry;

    public $esResultLimit = 10000;

    /**
     * 通过sql 搜索.
     *
     * @param BaseQuery $query
     *
     * @return bool|mixed
     *
     * @throws Exception
     * @throws GuzzleException
     */
    public function searchBySql($query)
    {
        $sql = $query->getSql();
        $url = $this->hosts[array_rand($this->hosts)].'/_sql';

        $res = Helper::curl($url, CommonConsts::HTTP_METHOD_POST, $sql);
        if (false === $res) {
            $message = 'Elastic search error query sql is: '.$sql;
            throw new Exception($message);
        }

        $rows = [];
        $total = ArrayHelper::getValue($res, 'hits.total');
        if ($total && isset($res['_scroll_id'])) {
            $offset = $query->offset + $query->limit;
            if ($total <= $query->offset) {
                $rows = [];
            } else {
                $num = $limit = 0;
                $depth = ceil(($offset) / $this->esResultLimit);
                for ($i = 0; $i < $depth; $i++) {
                    if ($limit) {
                        $rows = ArrayHelper::merge($rows, array_slice($this->getEsRows($res), 0, $limit));
                        break;
                    }

                    $num += $this->esResultLimit;
                    if ($num > $query->offset) {
                        $size = $num - $query->offset;
                        $index = $this->esResultLimit - $size;

                        $rows = ArrayHelper::merge(
                            $rows,
                            array_slice(
                                $this->getEsRows($res),
                                $index > 0 ? $index : 0,
                                $size < $query->limit ? $size : $query->limit
                            )
                        );

                        if ($size >= $query->limit) {
                            break ;
                        }

                        $limit = $query->limit - $size;
                    }
                    $res = $this->scrollSearch($res['_scroll_id']);
                }
            }
        } else {
            $rows = $this->getEsRows($res);
        }

        return ['total' => $total, 'rows' => $rows];
    }

    /**
     * 游标查询.
     *
     * @param string $scrollId
     *
     * @return bool|mixed
     *
     * @throws Exception
     * @throws GuzzleException
     */
    public function scrollSearch(string $scrollId)
    {
        $url = $this->hosts[array_rand($this->hosts)].'/_search/scroll';
        $res = Helper::curl(
            $url,
            CommonConsts::HTTP_METHOD_GET,
            ['scroll' => '1m', 'scroll_id' => $scrollId]
        );

        if (false === $res) {
            $message = 'Elastic scroll search error scroll id is: '.$scrollId;
            throw new Exception($message);
        }

        return $res;
    }

    /**
     * 将sql 解析成 dsl.
     *
     * @param string $sql
     *
     * @return bool|mixed
     *
     * @throws GuzzleException
     */
    public function explainSql(string $sql)
    {
        $url = $this->hosts[array_rand($this->hosts)].'/_sql/_explain';
        $res = Helper::curl($url, CommonConsts::HTTP_METHOD_POST, $sql);
        if (false === $res) {
            return false;
        }

        return $res;
    }

    /**
     * 获取es 查询返回结果集.
     *
     * @param array $res
     *
     * @return array
     */
    private function getEsRows(array $res): array
    {
        return array_map(static function ($item) {
            return $item['_source'];
        }, ArrayHelper::getValue($res, 'hits.hits', []));
    }
}
  1. ES form-size 查询仅支持查询1w条数据以下数据、from + size 超过1w 需使用游标查询方法
  2. ES 的索引默认刷新策略 1s，查询结果的近实时针对一些列表对数据实效性要求高的需要使用一些补偿策略

重写 Yii2 Model

  • 重写 Query
    <?php
namespace app\esmodels;

use app\exceptions\ElasticSearchException;
use GuzzleHttp\Exception\GuzzleException;
use Yii;
use yii\db\ActiveQuery;
use yii\db\ActiveRecord;
use yii\db\Exception;
use yii\helpers\ArrayHelper;

class BaseQuery extends ActiveQuery
{
    private $totalNum;

    /**
     * @param null $db
     *
     * @return array|ActiveRecord[]
     *
     * @throws GuzzleException
     * @throws Exception
     */
    public function all($db = null)
    {
        if ($this->emulateExecution) {
            return [];
        }

        $res = Yii::$app->es->searchBySql($this);
        $this->totalNum = ArrayHelper::getValue($res, 'total', 0);
        $rows = ArrayHelper::getValue($res, 'rows');
        if (!$rows) {
            return [];
        }

        return array_column($rows, null, $this->indexBy);
    }

    /**
     * @param null $db
     *
     * @return array|bool|mixed|ActiveRecord|null
     *
     * @throws Exception
     * @throws GuzzleException
     */
    public function one($db = null)
    {
        if ($this->emulateExecution) {
            return false;
        }

        $this->limit = 1;

        return ArrayHelper::getValue(Yii::$app->es->searchBySql($this), 'rows.0');
    }

    /**
     * @param string $q
     * @param null   $db
     *
     * @return int|mixed|string
     *
     * @throws Exception
     * @throws GuzzleException
     */
    public function count($q = '*', $db = null)
    {
        if ($this->emulateExecution) {
            return 0;
        }

        $this->limit = 1;

        return ArrayHelper::getValue(Yii::$app->es->searchBySql($this), 'total', 0);
    }

    /**
     * @param null $db
     *
     * @return bool
     *
     * @throws Exception
     * @throws GuzzleException
     */
    public function exists($db = null)
    {
        if ($this->emulateExecution) {
            return false;
        }

        $this->limit = 1;

        return (bool) ArrayHelper::getValue(Yii::$app->es->searchBySql($this), 'rows');
    }

    /**
     * @param null $db
     *
     * @return array
     *
     * @throws Exception
     * @throws GuzzleException
     */
    public function column($db = null)
    {
        if ($this->emulateExecution) {
            return [];
        }

        $rows = ArrayHelper::getValue(Yii::$app->es->searchBySql($this), 'rows');

        $key = '';
        if ($rows) {
            if (isset($rows[0])) {
                $key = array_keys($rows[0])[0];
            } else {
                $key = array_keys($rows)[0];
            }
        }

        return ArrayHelper::getColumn($rows, $key);
    }

    /**
     * @param null $db
     *
     * @return false|string|void|null
     *
     * @throws ElasticSearchException
     */
    public function scalar($db = null)
    {
        throw new ElasticSearchException(ElasticSearchException::UNSUPPORT_FUNCTON);
    }

    /**
     * @return bool|false|mixed|string|string[]|null
     */
    public function getSql()
    {
        $sql = $this->createCommand()->getRawSql();
        $sql = str_replace(['`', '!='], ['', '<>'], $sql);

        $esLimit = Yii::$app->es->esResultLimit;
        if ($this->limit + $this->offset >= $esLimit) {
            $sql = str_replace(
                ['SELECT', "LIMIT {$this->limit} OFFSET {$this->offset}"],
                ["SELECT /*! USE_SCROLL({$esLimit},120000) */", ''],
                $sql
            );
        }

        return $sql;
    }

    /**
     * 获取总数量.
     *
     * @return mixed
     */
    public function getTotalNum()
    {
        return $this->totalNum;
    }
}
    1. ES 匹配文档总数量会放在每次查询响应的 Total 字段
    2. Yii2 生成的 SQL 需要进行一些特殊处理
  • 重写 Model Find 方法
    <?php
namespace app\esmodels;

use Yii;
use yii\base\InvalidConfigException;
use yii\db\ActiveQuery;
use yii\db\ActiveRecord;

class Base extends ActiveRecord
{

    /**
     * @return object|BaseQuery
     *
     * @throws InvalidConfigException
     */
    public static function find()
    {
        return Yii::createObject(BaseQuery::className(), [get_called_class()]);
    }

    /**
     * @param mixed $condition
     *
     * @return BaseQuery|\yii\db\ActiveQueryInterface
     * @throws InvalidConfigException
     */
    public static function findByCondition($condition)
    {
        $query = self::find();

        return $query->andWhere($condition);
    }

    /**
     * @param string $sql
     * @param array  $params
     *
     * @return BaseQuery|ActiveQuery
     * @throws InvalidConfigException
     */
    public static function findBySql($sql, $params = [])
    {
        $query = self::find();
        $query->sql = $sql;

        return $query->params($params);
    }
}
  • 创建 ES Model
    <?php
namespace app\esmodels;

use app\utils\CommonConsts;
use yii\base\InvalidConfigException;

class Member extends Base
{
    public static function tableName()
    {
        return 'member_list';
    }
}
    1. tableName 返回 ES 文档名称
avatar
文章
阅读
粉丝