概览
egg的分析整体因为涉及到的npm包太多, 但对单个包逐一分析会破坏整体性,所以需要从整体入手然后对涉及到的npm包各个击破。
主要包括以下包
- 启动:
egg-binegg-script - 多进程:
egg-cluster - 框架:
eggegg-coreegg-router - 一些主要的依赖库:
common-binkoaget-readyready-callback
启动
我们先从启动开始,包含了 egg-bin 跟 egg-script 。
egg-bin
class DevCommand extends Command {
constructor(rawArgv) {
super(rawArgv);
this.defaultPort = 7001;
// 这里serverBin是egg-bin下的start-cluster文件
this.serverBin = path.join(__dirname, '../start-cluster');
}
* run(context) {
const devArgs = yield this.formatArgs(context);
const env = {
NODE_ENV: 'development',
EGG_MASTER_CLOSE_TIMEOUT: 1000,
};
const options = {
execArgv: context.execArgv,
env: Object.assign(env, context.env),
};
debug('%s %j %j, %j', this.serverBin, devArgs, options.execArgv, options.env.NODE_ENV);
// 其中forkNode,是对child_process.fork方法进行了promise的包装以及graceful的包装
const task = this.helper.forkNode(this.serverBin, devArgs, options);
this.proc = task.proc;
yield task;
}
}
其中,egg-bin start-cluster代码如下
#!/usr/bin/env node
'use strict';
const debug = require('debug')('egg-bin:start-cluster');
const options = JSON.parse(process.argv[2]);
debug('start cluster options: %j', options);
// 其中这里又执行了egg.js下的startCluster
require(options.framework).startCluster(options);
然后,egg.js的startCluster又指向了 egg-cluster 暴露的startCluster方法。
exports.startCluster = require('egg-cluster').startCluster;
问题: 这里的设计没有搞明白,为啥最开始没有指向 egg-cluster 呢?
egg-scripts
start命令
* run(context) {
const { argv, env, cwd, execArgv } = context;
const HOME = homedir();
const logDir = path.join(HOME, 'logs');
// egg-script start
// egg-script start ./server
// egg-script start /opt/app
let baseDir = argv._[0] || cwd;
if (!path.isAbsolute(baseDir)) baseDir = path.join(cwd, baseDir);
argv.baseDir = baseDir;
const isDaemon = argv.daemon;
argv.framework = yield this.getFrameworkPath({
framework: argv.framework,
baseDir,
});
this.frameworkName = yield this.getFrameworkName(argv.framework);
const pkgInfo = require(path.join(baseDir, 'package.json'));
argv.title = argv.title || `egg-server-${pkgInfo.name}`;
argv.stdout = argv.stdout || path.join(logDir, 'master-stdout.log');
argv.stderr = argv.stderr || path.join(logDir, 'master-stderr.log');
// normalize env
env.HOME = HOME;
env.NODE_ENV = 'production';
// it makes env big but more robust
env.PATH = env.Path = [
// for nodeinstall
path.join(baseDir, 'node_modules/.bin'),
// support `.node/bin`, due to npm5 will remove `node_modules/.bin`
path.join(baseDir, '.node/bin'),
// adjust env for win
env.PATH || env.Path,
].filter(x => !!x).join(path.delimiter);
// for alinode
env.ENABLE_NODE_LOG = 'YES';
env.NODE_LOG_DIR = env.NODE_LOG_DIR || path.join(logDir, 'alinode');
yield mkdirp(env.NODE_LOG_DIR);
// cli argv -> process.env.EGG_SERVER_ENV -> `undefined` then egg will use `prod`
if (argv.env) {
// if undefined, should not pass key due to `spwan`, https://github.com/nodejs/node/blob/master/lib/child_process.js#L470
env.EGG_SERVER_ENV = argv.env;
}
const command = argv.node || 'node';
const options = {
execArgv,
env,
stdio: 'inherit',
detached: false,
};
this.logger.info('Starting %s application at %s', this.frameworkName, baseDir);
// remove unused properties from stringify, alias had been remove by `removeAlias`
const ignoreKeys = [ '_', '$0', 'env', 'daemon', 'stdout', 'stderr', 'timeout', 'ignore-stderr', 'node' ];
const clusterOptions = stringify(argv, ignoreKeys);
// Note: `spawn` is not like `fork`, had to pass `execArgv` youself
const eggArgs = [ ...(execArgv || []), this.serverBin, clusterOptions, `--title=${argv.title}` ];
this.logger.info('Run node %s', eggArgs.join(' '));
// whether run in the background.
if (isDaemon) {
this.logger.info(`Save log file to ${logDir}`);
const [ stdout, stderr ] = yield [ getRotatelog(argv.stdout), getRotatelog(argv.stderr) ];
options.stdio = [ 'ignore', stdout, stderr, 'ipc' ];
// 开启 detached,子进程常驻。
options.detached = true;
debug('Run spawn `%s %s`', command, eggArgs.join(' '));
const child = this.child = spawn(command, eggArgs, options);
this.isReady = false;
child.on('message', msg => {
// 当egg启动完毕,从master发送消息给parent进程,通知parent进程退出。
if (msg && msg.action === 'egg-ready') {
this.isReady = true;
this.logger.info('%s started on %s', this.frameworkName, msg.data.address);
// 让父进程不需要等待子进程结束,直接退出
child.unref();
child.disconnect();
this.exit(0);
}
});
// check start status
yield this.checkStatus(argv);
} else {
options.stdio = [ 'inherit', 'inherit', 'inherit', 'ipc' ];
debug('Run spawn `%s %s`', command, eggArgs.join(' '));
const child = this.child = spawn(command, eggArgs, options);
child.once('exit', code => {
// command should exit after child process exit
this.exit(code);
});
// attach master signal to child
let signal;
[ 'SIGINT', 'SIGQUIT', 'SIGTERM' ].forEach(event => {
process.once(event, () => {
debug('Kill child %s with %s', child.pid, signal);
child.kill(event);
});
});
}
}
stop命令
* run(context) {
const { argv } = context;
this.logger.info(`stopping egg application ${argv.title ? `with --title=${argv.title}` : ''}`);
// 通过类ps -ef来寻找进程中包括start-cluster的进程,也就是主进程
// node /Users/tz/Workspaces/eggjs/egg-scripts/lib/start-cluster {"title":"egg-server","workers":4,"port":7001,"baseDir":"/Users/tz/Workspaces/eggjs/test/showcase","framework":"/Users/tz/Workspaces/eggjs/test/showcase/node_modules/egg"}
let processList = yield this.helper.findNodeProcess(item => {
const cmd = item.cmd;
return argv.title ?
cmd.includes('start-cluster') && cmd.includes(util.format(osRelated.titleTemplate, argv.title)) :
cmd.includes('start-cluster');
});
let pids = processList.map(x => x.pid);
if (pids.length) {
this.logger.info('got master pid %j', pids);
this.helper.kill(pids);
// 当主进程被kill掉之后,主进程会通知agent worker进程退出,这里会等待5s来保证全部退出
// wait for 5s to confirm whether any worker process did not kill by master
yield sleep(argv.timeout || '5s');
} else {
this.logger.warn('can\'t detect any running egg process');
}
// 寻找没有被主进程杀死的agent、worker进程
// node --debug-port=5856 /Users/tz/Workspaces/eggjs/test/showcase/node_modules/_egg-cluster@1.8.0@egg-cluster/lib/agent_worker.js {"framework":"/Users/tz/Workspaces/eggjs/test/showcase/node_modules/egg","baseDir":"/Users/tz/Workspaces/eggjs/test/showcase","port":7001,"workers":2,"plugins":null,"https":false,"key":"","cert":"","title":"egg-server","clusterPort":52406}
// node /Users/tz/Workspaces/eggjs/test/showcase/node_modules/_egg-cluster@1.8.0@egg-cluster/lib/app_worker.js {"framework":"/Users/tz/Workspaces/eggjs/test/showcase/node_modules/egg","baseDir":"/Users/tz/Workspaces/eggjs/test/showcase","port":7001,"workers":2,"plugins":null,"https":false,"key":"","cert":"","title":"egg-server","clusterPort":52406}
processList = yield this.helper.findNodeProcess(item => {
const cmd = item.cmd;
return argv.title ?
(cmd.includes(osRelated.appWorkerPath) || cmd.includes(osRelated.agentWorkerPath)) && cmd.includes(util.format(osRelated.titleTemplate, argv.title)) :
(cmd.includes(osRelated.appWorkerPath) || cmd.includes(osRelated.agentWorkerPath));
});
pids = processList.map(x => x.pid);
if (pids.length) {
this.logger.info('got worker/agent pids %j that is not killed by master', pids);
this.helper.kill(pids, 'SIGKILL');
}
this.logger.info('stopped');
}
多进程
多进程主要包括 egg-cluster 。
时序图
在介绍egg-cluster之前,需要先看 get-ready 以及 ready-callback 这两个包,其中基本穿插着整个egg代码。
通信机制
get-ready
'use strict';
const is = require('is-type-of');
const IS_READY = Symbol('isReady');
const READY_CALLBACKS = Symbol('readyCallbacks');
const READY_ARG = Symbol('readyArg');
class Ready {
constructor() {
this[IS_READY] = false;
// callback队列
this[READY_CALLBACKS] = [];
}
ready(flagOrFunction) {
// register a callback
if (flagOrFunction === undefined || is.function(flagOrFunction)) {
return this.register(flagOrFunction);
}
// emit callbacks
this.emit(flagOrFunction);
}
/**
* Call the callbacks that has been registerd, and clean the callback stack.
* If the flag is not false, it will be marked as ready. Then the callbacks will be called immediatly when register.
* @param {Boolean|Error} flag - Set a flag whether it had been ready. If the flag is an error, it's also ready, but the callback will be called with argument `error`
*/
emit(flag) {
// this.ready(true);
// this.ready(false);
// this.ready(err);
this[IS_READY] = flag !== false;
this[READY_ARG] = flag instanceof Error ? flag : undefined;
// this.ready(true)
if (this[IS_READY]) {
this[READY_CALLBACKS]
.splice(0, Infinity)
// 把callback放到process.nextTick队列里,等待执行
.forEach(callback => process.nextTick(() => callback(this[READY_ARG])));
}
}
/**
* @param {Object} obj - an object that be mixed
*/
static mixin(obj) {
if (!obj) return;
const ready = new Ready();
// delegate method
obj.ready = flagOrFunction => ready.ready(flagOrFunction);
}
}
function mixin(object) {
Ready.mixin(object);
}
module.exports = mixin;
module.exports.mixin = mixin;
module.exports.Ready = Ready;
ready-callback
对get-ready的包装
'use strict';
const EventEmitter = require('events');
const ready = require('get-ready');
const debug = require('debug')('ready-callback');
const defaults = {
timeout: 10000,
isWeakDep: false,
};
/**
* @class Ready
*/
class Ready extends EventEmitter {
/**
* @constructor
* @param {Object} opt
* - {Number} [timeout=10000] - emit `ready_timeout` when it doesn't finish but reach the timeout
* - {Boolean} [isWeakDep=false] - whether it's a weak dependency
* - {Boolean} [lazyStart=false] - will not check cache size automatically, if lazyStart is true
*/
constructor(opt) {
super();
ready.mixin(this);
this.opt = opt || {};
this.isError = false;
this.cache = new Map();
if (!this.opt.lazyStart) {
this.start();
}
}
start() {
// 在 check 阶段执行
setImmediate(() => {
// fire callback directly when no registered ready callback
if (this.cache.size === 0) {
debug('Fire callback directly');
this.ready(true);
}
});
}
}
如上源码展示,get-ready实现了一个语义化的事件模型,ready-callback对get-ready进行了包装,让其在 check 阶段进行执行。
接下来进入最核心的地方, egg-core 以及 egg 。
框架
在多进程里头的时序图里头,有 fork agent 以及 fork worker 的消息,其实这里就会进行实例化操作。
其中,agent以及worker的继承关系如下图:
上图有些关键的信息
- 不论是agent还是worker进程,它的ready函数都是通过eggCore里的lifcecycle来提供的。
- loader机制也是在eggcore里初始化的,注意这里说的是在eggCore里初始化的,虽然
egg-core会有个默认的loader父类,但最终初始化的是egg包里提供的agentLoader跟workLoader子类。
egg-core
egg-core 里主要有两个东西 lifecycle 跟 loader ,这两个一个比一个令人头大,先搞最头大的 loader 吧。
loader
class AgentWorkerLoader extends EggLoader {
/**
* loadPlugin first, then loadConfig
*/
loadConfig() {
this.loadPlugin();
super.loadConfig();
}
load() {
this.loadAgentExtend();
this.loadContextExtend();
this.loadCustomAgent();
}
}
module.exports = AgentWorkerLoader;
'use strict';
const EggLoader = require('egg-core').EggLoader;
/**
* App worker process Loader, will load plugins
* @see https://github.com/eggjs/egg-loader
*/
class AppWorkerLoader extends EggLoader {
/**
* loadPlugin first, then loadConfig
* @since 1.0.0
*/
loadConfig() {
this.loadPlugin();
super.loadConfig();
}
/**
* Load all directories in convention
* @since 1.0.0
*/
load() {
// app > plugin > core
this.loadApplicationExtend();
this.loadRequestExtend();
this.loadResponseExtend();
this.loadContextExtend();
this.loadHelperExtend();
this.loadCustomLoader();
// app > plugin
this.loadCustomApp();
// app > plugin
this.loadService();
// app > plugin > core
this.loadMiddleware();
// app
this.loadController();
// app
this.loadRouter(); // Dependent on controllers
}
}
module.exports = AppWorkerLoader;
这两个都是调用loadConfig,然后再调用load方法。其中loadConfig里的调用就不介绍了,相对简单。
load****Extend
这种方法在agent以及worker进程里都会调用。
/**
* mixin Agent.prototype
* @function EggLoader#loadAgentExtend
* @since 1.0.0
*/
loadAgentExtend() {
this.loadExtend('agent', this.app);
},
/**
* mixin Application.prototype
* @function EggLoader#loadApplicationExtend
* @since 1.0.0
*/
loadApplicationExtend() {
this.loadExtend('application', this.app);
},
/**
* mixin Request.prototype
* @function EggLoader#loadRequestExtend
* @since 1.0.0
*/
loadRequestExtend() {
this.loadExtend('request', this.app.request);
},
/**
* mixin Response.prototype
* @function EggLoader#loadResponseExtend
* @since 1.0.0
*/
loadResponseExtend() {
this.loadExtend('response', this.app.response);
},
/**
* mixin Context.prototype
* @function EggLoader#loadContextExtend
* @since 1.0.0
*/
loadContextExtend() {
this.loadExtend('context', this.app.context);
},
/**
* mixin app.Helper.prototype
* @function EggLoader#loadHelperExtend
* @since 1.0.0
*/
loadHelperExtend() {
if (this.app && this.app.Helper) {
this.loadExtend('helper', this.app.Helper.prototype);
}
},
从代码可以看出,主要是调用 this.loadExtend 方法。
loadExtend(name, proto) {
this.timing.start(`Load extend/${name}.js`);
// All extend files
const filepaths = this.getExtendFilePaths(name);
// if use mm.env and serverEnv is not unittest
const isAddUnittest = 'EGG_MOCK_SERVER_ENV' in process.env && this.serverEnv !== 'unittest';
for (let i = 0, l = filepaths.length; i < l; i++) {
const filepath = filepaths[i];
filepaths.push(filepath + `.${this.serverEnv}`);
if (isAddUnittest) filepaths.push(filepath + '.unittest');
}
const mergeRecord = new Map();
for (let filepath of filepaths) {
filepath = this.resolveModule(filepath);
if (!filepath) {
continue;
} else if (filepath.endsWith('/index.js')) {
// TODO: remove support at next version
deprecate(`app/extend/${name}/index.js is deprecated, use app/extend/${name}.js instead`);
}
const ext = this.requireFile(filepath);
// 获取一个对象的属性list,包含了symbol。
// 注: Object.keys以及Object.getOwnPropertyNames都只能获得对象的非symbol属性list
const properties = Object.getOwnPropertyNames(ext)
.concat(Object.getOwnPropertySymbols(ext));
for (const property of properties) {
if (mergeRecord.has(property)) {
debug('Property: "%s" already exists in "%s",it will be redefined by "%s"',
property, mergeRecord.get(property), filepath);
}
// 获取一个agent扩展里一个方法的描述符
let descriptor = Object.getOwnPropertyDescriptor(ext, property);
// 从要注入的对象里获得当前方法的描述符
let originalDescriptor = Object.getOwnPropertyDescriptor(proto, property);
if (!originalDescriptor) {
// try to get descriptor from originalPrototypes
const originalProto = originalPrototypes[name];
if (originalProto) {
// 从koa层获取当前方法的描述符
originalDescriptor = Object.getOwnPropertyDescriptor(originalProto, property);
}
}
// 如果从 注入的对象或koa原生上获取到描述符,则对初始描述符做merge操作,以初始描述符优先级最高
if (originalDescriptor) {
// don't override descriptor
descriptor = Object.assign({}, descriptor);
if (!descriptor.set && originalDescriptor.set) {
descriptor.set = originalDescriptor.set;
}
if (!descriptor.get && originalDescriptor.get) {
descriptor.get = originalDescriptor.get;
}
}
// 注入描述符到要扩展的对象上
Object.defineProperty(proto, property, descriptor);
// 进入set结构,之后进行重复提示
mergeRecord.set(property, filepath);
}
debug('merge %j to %s from %s', Object.keys(ext), name, filepath);
}
this.timing.end(`Load extend/${name}.js`);
}
// 不用care有没有定义app/extend,直接Join起来,后边进行判断,举例
// /Users/ld/Documents/bytedance/test/showcase/node_modules/egg-view/app/extend/agent
// Users/ld/Documents/bytedance/test/showcase/node_modules/egg/app/extend/agent
// /Users/ld/Documents/bytedance/test/showcase/app/extend/agent
getExtendFilePaths(name) {
// getLoadUnits方法返回了所有上层框架层、egg层、以及这两者所涉及到的enable的插件目录
return this.getLoadUnits().map(unit => path.join(unit.path, 'app/extend', name));
},
loadService
loadService(opt) {
this.timing.start('Load Service');
// 载入到 app.serviceClasses
opt = Object.assign({
call: true,
caseStyle: 'lower',
fieldClass: 'serviceClasses',
directory: this.getLoadUnits().map(unit => path.join(unit.path, 'app/service')),
}, opt);
const servicePaths = opt.directory;
this.loadToContext(servicePaths, 'service', opt);
this.timing.end('Load Service');
},
可以看到,最主要的是 this.loadContext 方法,事实上,只要你想着往 ctx 上挂载东西,都需要调用 this.loadToContext 方法。
loadToContext(directory, property, opt) {
opt = Object.assign({}, {
directory,
property,
inject: this.app,
}, opt);
const timingKey = `Load "${String(property)}" to Context`;
this.timing.start(timingKey);
new ContextLoader(opt).load();
this.timing.end(timingKey);
}
'use strict';
const assert = require('assert');
const is = require('is-type-of');
const FileLoader = require('./file_loader');
const CLASSLOADER = Symbol('classLoader');
const EXPORTS = FileLoader.EXPORTS;
class ClassLoader {
constructor(options) {
assert(options.ctx, 'options.ctx is required');
const properties = options.properties;
this._cache = new Map();
this._ctx = options.ctx;
for (const property in properties) {
this.defineProperty(property, properties[property]);
}
}
defineProperty(property, values) {
Object.defineProperty(this, property, {
get() {
let instance = this._cache.get(property);
if (!instance) {
instance = getInstance(values, this._ctx);
this._cache.set(property, instance);
}
return instance;
},
});
}
}
/**
* Same as {@link FileLoader}, but it will attach file to `inject[fieldClass]`. The exports will be lazy loaded, such as `ctx.group.repository`.
* @extends FileLoader
* @since 1.0.0
*/
class ContextLoader extends FileLoader {
/**
* @class
* @param {Object} options - options same as {@link FileLoader}
* @param {String} options.fieldClass - determine the field name of inject object.
*/
constructor(options) {
assert(options.property, 'options.property is required');
assert(options.inject, 'options.inject is required');
const target = options.target = {};
if (options.fieldClass) {
options.inject[options.fieldClass] = target;
}
super(options);
const app = this.options.inject;
const property = options.property;
// define ctx.service
Object.defineProperty(app.context, property, {
get() {
// distinguish property cache,
// cache's lifecycle is the same with this context instance
// e.x. ctx.service1 and ctx.service2 have different cache
if (!this[CLASSLOADER]) {
this[CLASSLOADER] = new Map();
}
const classLoader = this[CLASSLOADER];
let instance = classLoader.get(property);
if (!instance) {
instance = getInstance(target, this);
classLoader.set(property, instance);
}
return instance;
},
});
}
}
module.exports = ContextLoader;
function getInstance(values, ctx) {
// it's a directory when it has no exports
// then use ClassLoader
const Class = values[EXPORTS] ? values : null;
let instance;
if (Class) {
if (is.class(Class)) {
instance = new Class(ctx);
} else {
// it's just an object
instance = Class;
}
// Can't set property to primitive, so check again
// e.x. module.exports = 1;
} else if (is.primitive(values)) {
instance = values;
} else {
instance = new ClassLoader({ ctx, properties: values });
}
return instance;
}
fileLoder的load方法
load() {
const items = this.parse();
const target = this.options.target;
for (const item of items) {
debug('loading item %j', item);
// item { properties: [ 'a', 'b', 'c'], exports }
// => target.a.b.c = exports
item.properties.reduce((target, property, index) => {
let obj;
const properties = item.properties.slice(0, index + 1).join('.');
if (index === item.properties.length - 1) {
if (property in target) {
if (!this.options.override) throw new Error(`can't overwrite property '${properties}' from ${target[property][FULLPATH]} by ${item.fullpath}`);
}
obj = item.exports;
if (obj && !is.primitive(obj)) {
obj[FULLPATH] = item.fullpath;
obj[EXPORTS] = true;
}
} else {
obj = target[property] || {};
}
target[property] = obj;
debug('loaded %s', properties);
return obj;
}, target);
}
return target;
}
先说个总流程,在 loaderService 的时候,他并不是向 loaderExtend 那样,直接全部挂载到ctx上,怀疑是因为业务的service会无限大,如果每次请求的时候把所有service都挂载到 ctx 上,会导致内存消耗巨大。
所以目前的实现方式是,会把所有的service都放到一个闭包里头,当初次访问ctx.service的时候会返回每个serviceClass包装的classLoader对象所组成的大对象,如果接着往下访问, getInstance 会判断你是否是个class,如果是个class,他就会返回实例。
所以,比如当执行 ctx.service.index.index() 的时候,流程图如下:
不论是在获取ctx.service对象,还是获取ctx.service.index的时候,都有map存在,但都是在一个请求下才会生效。
loadToApp
loadToApp(directory, property, opt) {
const target = this.app[property] = {};
opt = Object.assign({}, {
directory,
target,
inject: this.app,
}, opt);
const timingKey = `Load "${String(property)}" to Application`;
this.timing.start(timingKey);
new FileLoader(opt).load();
this.timing.end(timingKey);
}
loadToApp方法相对粗暴,就是直接挂载到 app 上,这里不做赘述。
loadController loadMiddleware
其中 loadMiddleware 相对简单,把获取到的中间件 use 下即可,对于 controller 则需要对齐进行 methodToMiddleware 的包装。
loadMiddleware 的链路走的是 this.loadToApp ,存储到 app.middleware 里,之后startServer的时候,use一波。
对于 loadController ,也是经过 this.loadToApp 将controller挂载到 app 上,但是这里因为跟路由挂钩的关系,需要将 controller wrap一层,搞成一个中间件。然后接下来访问的时候,还是走的 koa-router 的逻辑。
lifecycle
user-images.githubusercontent.com/40081831/47…
看这个官方图就够了。
结束
终于完结了,总体感受下来,egg的源码阅读起来有点零碎,有好多自己封装的npm包,但整体的设计还是很棒的。