概述
Nginx是纯C编写的反向代理服务器，由于商业化和开源之间的平衡，在他之上还衍生出了支持热更新的OpenResty。Nginx的用户接口非常简洁，仅仅一个conf就能做到几乎所有事情。对于二次开发，Nginx有着优秀的架构设计，几乎领先了时代，但也正因为那个时代HTTP特点的掣肘，拓展Nginx并不如想象得那么容易，特别是随着Rust的完善，C的开发吸引力会越来越低，但Nginx源码无疑是最值得学习的C之一。
develop
架构特点
- 多进程: master进程负责管理，worker进程负责处理请求，即使一个worker进程因为错误崩溃，重新拉起一个即可；
- 热加载: 当需要reload配置时，拉起一个新的master，准备完毕后再退出旧的master，实现热加载；
- 网络模型: epoll 事件驱动，解决c10k的通用做法；
- 内存管理: 利用connection生命短的特点，使用pool分配内存，用户无需管理动态内存，用完释放pool即可；
- 模块化: ngx提供ngx_str等封装好的模块库，实现统一的编程风格，这也是缺乏标准的C语言项目常用做法；
- 插件化: 用户通过编写C实现nginx的plugin接口，即函数指针，并利用标准库实现自定义conf解析，将源码通过configuration生成定制makefile，实现模块化的功能支持；
- context保存上下文，不保证调用链: 为了实现高性能事件驱动，通过context保存请求上下文，不向模块保证严格调用时机；
概念
connection
connection是ngx对tcp接接的封装，当tcp连接成功建立时，会根据从连接池中获取链接结构体进行封装。
typedef struct ngx_connection_s ngx_connection_t;
struct ngx_connection_s {
void *data;
ngx_event_t *read;
ngx_event_t *write;
ngx_socket_t fd;
ngx_recv_pt recv;
ngx_send_pt send;
ngx_recv_chain_pt recv_chain;
ngx_send_chain_pt send_chain;
ngx_listening_t *listening;
off_t sent;
ngx_log_t *log;
ngx_pool_t *pool;
int type;
struct sockaddr *sockaddr;
socklen_t socklen;
ngx_str_t addr_text;
ngx_proxy_protocol_t *proxy_protocol;
ngx_udp_connection_t *udp;
struct sockaddr *local_sockaddr;
socklen_t local_socklen;
ngx_buf_t *buffer;
ngx_queue_t queue;
ngx_atomic_uint_t number;
ngx_msec_t start_time;
ngx_uint_t requests;
unsigned buffered:8;
unsigned log_error:3; /* ngx_connection_log_error_e */
unsigned timedout:1;
unsigned error:1;
unsigned destroyed:1;
unsigned pipeline:1;
unsigned idle:1;
unsigned reusable:1;
unsigned close:1;
unsigned shared:1;
unsigned sendfile:1;
unsigned sndlowat:1;
unsigned tcp_nodelay:2; /* ngx_connection_tcp_nodelay_e */
unsigned tcp_nopush:2; /* ngx_connection_tcp_nopush_e */
unsigned need_last_buf:1;
unsigned need_flush_buf:1;
};
request
- ngx_http_init_request: 入口函数
- ngx_http_process_request_line: 处理请求行
- ngx_http_process_request_headers: 处理请求头
- ngx_http_process_request: 处理body，但不会直接读body
- read: ngx_http_block_reading
- write: 设置 write_event_handler = ngx_http_core_run_phases;
协议实现:
- pipeline: 如果处理完一次请求后，buffer中还有数据，则认为是下一个请求
- lingering_close: 当tcp被close，内核会检查read buffer有没有数据，有则发生RST包，对客户端非常不友好，为了避免发送RST包，ngx支持延迟关闭，即关闭HTTP后等待一段时间再关闭TCP
数据结构
ngx_str_t
- 不以'\0'结尾
typedef struct {
size_t len;
u_char *data;
} ngx_str_t;
ngx_pool_t
typedef struct ngx_pool_s ngx_pool_t;
typedef struct {
u_char *last;
u_char *end;
ngx_pool_t *next;
ngx_uint_t failed;
} ngx_pool_data_t;
struct ngx_pool_s {
ngx_pool_data_t d;
size_t max;
ngx_pool_t *current;
ngx_chain_t *chain;
ngx_pool_large_t *large;
ngx_pool_cleanup_t *cleanup;
ngx_log_t *log;
};
ngx_array_t
- 原理类似于std的vector
typedef struct {
void *elts;
ngx_uint_t nelts;
size_t size;
ngx_uint_t nalloc;
ngx_pool_t *pool;
} ngx_array_t;
ngx_hash_t
- hash冲突使用开链法
- 只能一次初始化，之后无法增加或删除
- wildcard_t 可以支持http常用的前缀或后缀匹配
- combined_t 缝合了3种hash表
typedef struct {
ngx_hash_elt_t **buckets;
ngx_uint_t size;
} ngx_hash_t;
typedef struct {
ngx_hash_t hash;
void *value;
} ngx_hash_wildcard_t;
typedef struct {
ngx_str_t key;
ngx_uint_t key_hash;
void *value;
} ngx_hash_key_t;
typedef struct {
ngx_hash_t *hash;
ngx_hash_key_pt key;
ngx_uint_t max_size;
ngx_uint_t bucket_size;
char *name;
ngx_pool_t *pool;
ngx_pool_t *temp_pool;
} ngx_hash_init_t;
typedef struct {
ngx_hash_t hash;
ngx_hash_wildcard_t *wc_head;
ngx_hash_wildcard_t *wc_tail;
} ngx_hash_combined_t;
ngx_int_t
ngx_hash_init(ngx_hash_init_t *hinit, ngx_hash_key_t *names, ngx_uint_t nelts)
ngx_chain_s, ngx_buf_t
struct ngx_chain_s {
ngx_buf_t *buf;
ngx_chain_t *next;
};
typedef struct ngx_buf_s ngx_buf_t;
#define ngx_buf_in_memory(b) ((b)->temporary || (b)->memory || (b)->mmap)
ngx_buf_t可以在内存里，也可以在文件里。它有4个重要的指针，从前往后分别是
- start: buf开始
- pos: 未处理的开始
- last: 数据的结束
- end: buf结束
ngx_list_t
ngx_list_t的本质是链表+数组，每个链表节点ngx_list_part_t里存放的是一个连续数组
typedef struct {
ngx_list_part_t *last;
ngx_list_part_t part;
size_t size;
ngx_uint_t nalloc;
ngx_pool_t *pool;
} ngx_list_t;
// 创建list
ngx_list_t *
ngx_list_create(ngx_pool_t *pool, ngx_uint_t n, size_t size)
// 初始化list结构体
static ngx_inline ngx_int_t
ngx_list_init(ngx_list_t *list, ngx_pool_t *pool, ngx_uint_t n, size_t size)
// 返回可存放元素的指针
void *
ngx_list_push(ngx_list_t *l)
ngx_queue_t
ngx_queue_t是正宗的双向链表，特别的是，它并没有内部存储void*来存储数据，而是用组合的思想，组合了ngx_queue_t的结构体自然成为了链表。
typedef struct ngx_queue_s ngx_queue_t;
struct ngx_queue_s {
ngx_queue_t *prev;
ngx_queue_t *next;
};
#define ngx_queue_init(q) \
(q)->prev = q; \
(q)->next = q
#define ngx_queue_data(q, type, link) \
(type *) ((u_char *) q - offsetof(type, link))
// q是ngx_queue_t，data_typee是ngx_connection_t，q的成员名是queue
c = ngx_queue_data(q, ngx_connection_t, queue);
struct ngx_connection_s {
// ...
ngx_queue_t queue;
}
配置原理
指令上下文（作用域）
- main
- http
- server
- location
参数set函数
函数签名: char *(*set)(ngx_conf_t *cf, ngx_command_t *cmd, void *conf);
一些默认实现：
- ngx_conf_set_str_slot
- ngx_conf_set_keyval_slot
- ngx_conf_set_num_slot
- ngx_conf_set_size_slot
- ngx_conf_set_msec_slot
- ngx_conf_set_sec_slot
modules obj
在objs/ngx_modules.c文件里，有编译进去的所有modules:
ngx_module_t *ngx_modules[] = {
&ngx_core_module,
&ngx_errlog_module,
&ngx_conf_module,
&ngx_regex_module,
&ngx_events_module,
}
三剑客1: handler模块
http-handler模块 ngx_http_module_t
typedef struct {
ngx_int_t (*preconfiguration)(ngx_conf_t *cf);
ngx_int_t (*postconfiguration)(ngx_conf_t *cf);
void *(*create_main_conf)(ngx_conf_t *cf);
char *(*init_main_conf)(ngx_conf_t *cf, void *conf);
void *(*create_srv_conf)(ngx_conf_t *cf);
char *(*merge_srv_conf)(ngx_conf_t *cf, void *prev, void *conf);
void *(*create_loc_conf)(ngx_conf_t *cf);
char *(*merge_loc_conf)(ngx_conf_t *cf, void *prev, void *conf);
} ngx_http_module_t;
nginx模块 ngx_module_t
typedef struct ngx_module_s ngx_module_t;
struct ngx_module_s {
void *ctx;
ngx_command_t *commands;
ngx_uint_t type;
ngx_int_t (*init_master)(ngx_log_t *log);
ngx_int_t (*init_module)(ngx_cycle_t *cycle);
ngx_int_t (*init_process)(ngx_cycle_t *cycle);
ngx_int_t (*init_thread)(ngx_cycle_t *cycle);
void (*exit_thread)(ngx_cycle_t *cycle);
void (*exit_process)(ngx_cycle_t *cycle);
void (*exit_master)(ngx_cycle_t *cycle);
}
三剑客2: filter模块
流程
- copy_filter：复制包体内容
- postpone_filter：处理子请求
- header_filter：构造响应头部
- write_filter：发送响应
- 所有第三方的模块只能加入到copy_filter
src/http/ngx_http_copy_filter_module.c和headers_filtersrc/http/modules/ngx_http_headers_filter_module.c模块之间执行
ngx_http_headers_filter_module
- static ngx_http_set_header_t ngx_http_set_headers[]
- ngx_http_add_multi_header_lines: out增加Cache-Control和Link的header
- ngx_http_set_last_modified: ngx_http_set_response_header
- ngx_http_set_response_header
filter next
extern ngx_http_output_header_filter_pt ngx_http_top_header_filter;
extern ngx_http_output_body_filter_pt ngx_http_top_body_filter;
extern ngx_http_request_body_filter_pt ngx_http_top_request_body_filter;
- ngx_http_top_body_filter是extern全局变量，用于init时告诉模块哪些模块注册在它后面
- ngx_http_next_body_filter是模块静态全局变量，保存原本的ngx_http_top_body_filter,下一个filter
- 因此，整个ngx_http_top_body_filter随着init形成了一个前插法的链表
// ngx_http.h
extern ngx_http_output_body_filter_pt ngx_http_top_body_filter;
static ngx_http_output_body_filter_pt ngx_http_next_body_filter;
static ngx_int_t
ngx_http_copy_filter_init(ngx_conf_t *cf)
{
ngx_http_next_body_filter = ngx_http_top_body_filter;
ngx_http_top_body_filter = ngx_http_copy_filter;
return NGX_OK;
}
ctx->output_filter = (ngx_output_chain_filter_pt)
ngx_http_next_body_filter;
三剑客3: upstream模块
流程
- init_peer: 初始化一张表，将所有upstream的server地址都记录好
- peer.get: 从server表中选择一个server
- connect
- send_request
- peer.free: 释放server一些资源
init upstream
- ngx_http_proxy_handler
- ngx_http_upstream_create: if (ngx_http_upstream_create(r) != NGX_OK)
- ngx_http_upstream_init: c = ngx_http_read_client_request_body(r, ngx_http_upstream_init);
- ngx_http_upstream_init_request: ngx_http_upstream_init_request(r);
- create_request: if (u->create_request(r) != NGX_OK)
- ngx_http_cleanup_add: cln = ngx_http_cleanup_add(r, 0);
- ngx_http_upstream_cleanup: cln->handler = ngx_http_upstream_cleanup;
- ngx_http_upstream_connect: ngx_http_upstream_connect(r, u);
- ngx_event_connect_peer: rc = ngx_event_connect_peer(&u->peer);
- ngx_socket: s = ngx_socket(pc->sockaddr->sa_family, type, 0);
- ngx_get_connection: c = ngx_get_connection(s, pc->log);
- bind: bind(s, pc->local->sockaddr, pc->local->socklen);
- pc->connection = c;
- connect: rc = connect(s, pc->sockaddr, pc->socklen);
- ngx_http_upstream_handler: c->write->handler = ngx_http_upstream_handler;
- ngx_http_upstream_handler: c->read->handler = ngx_http_upstream_handler;
- u->write_event_handler = ngx_http_upstream_send_request_handler;
- u->read_event_handler = ngx_http_upstream_process_header;
- ngx_http_upstream_send_request: ngx_http_upstream_send_request(r, u, 1);
- ngx_http_upstream_send_request_body: rc = ngx_http_upstream_send_request_body(r, u, do_write);
- if (r->request_body->bufs)
- rc = ngx_output_chain(&u->output, out);
- if (ngx_handle_write_event(c->write, u->conf->send_lowat))
- u->write_event_handler = ngx_http_upstream_dummy_handler;
- ngx_http_upstream_send_request_body: rc = ngx_http_upstream_send_request_body(r, u, do_write);
- ngx_event_connect_peer: rc = ngx_event_connect_peer(&u->peer);
connect upstream
- ngx_http_upstream_connect: ngx_http_upstream_connect(r, u);
- ngx_event_connect_peer: rc = ngx_event_connect_peer(&u->peer);
- ngx_socket: s = ngx_socket(pc->sockaddr->sa_family, type, 0);
- ngx_get_connection: c = ngx_get_connection(s, pc->log);
- bind: bind(s, pc->local->sockaddr, pc->local->socklen);
- pc->connection = c;
- connect: rc = connect(s, pc->sockaddr, pc->socklen);
- ngx_event_connect_peer: rc = ngx_event_connect_peer(&u->peer);
send upstream
- ngx_http_upstream_connect: ngx_http_upstream_connect(r, u);
- ngx_event_connect_peer: rc = ngx_event_connect_peer(&u->peer);
- ngx_http_upstream_handler: c->write->handler = ngx_http_upstream_handler;
- ngx_http_upstream_handler: c->read->handler = ngx_http_upstream_handler;
- u->write_event_handler = ngx_http_upstream_send_request_handler;
- u->read_event_handler = ngx_http_upstream_process_header;
- ngx_http_upstream_send_request: ngx_http_upstream_send_request(r, u, 1);
- ngx_http_upstream_send_request_body: rc = ngx_http_upstream_send_request_body(r, u, do_write);
- if (r->request_body->bufs)
- rc = ngx_output_chain(&u->output, out);
- if (ngx_handle_write_event(c->write, u->conf->send_lowat))
- u->write_event_handler = ngx_http_upstream_dummy_handler;
- ngx_http_upstream_send_request_body: rc = ngx_http_upstream_send_request_body(r, u, do_write);
recv upstream
next
- ngx_http_upstream_next
- if (u->peer.tries == 0) -> ngx_http_upstream_finalize_request(r, u, status);
- ngx_destroy_pool(u->peer.connection->pool);
- ngx_http_upstream_connect(r, u);
close upstream
- ngx_http_upstream_init_request
- cln->handler = ngx_http_upstream_cleanup;
- ngx_http_upstream_cleanup
- ngx_http_upstream_finalize_request
- u->finalize_request(r, rc);
- ngx_destroy_pool(u->peer.connection->pool);
- ngx_close_connection(u->peer.connection);
- ngx_delete_file(u->pipe->temp_file->file.name.data)
- ngx_http_finalize_request(r, rc);
- ngx_http_upstream_finalize_request
src/http/ngx_http_upstream_round_robin.c
- ngx_http_upstream_init_round_robin: 根据server配置初始化ngx_http_upstream_srv_conf_t
- ngx_http_upstream_init_round_robin_peer: us->peer.init的钩子函数，内部还设置了r->upstream->peer.get这类钩子
- ngx_http_upstream_create_round_robin_peer: 参数是ngx_http_upstream_resolved_t *ur？什么东西
- ngx_http_upstream_get_round_robin_peer: r->upstream->peer.get
- ngx_http_upstream_free_round_robin_peer: r->upstream->peer.free
- ngx_http_upstream_get_peer: ngx_http_upstream_get_round_robin_peer用于get_peer，具体原理？
nginx-core
事件驱动 queue & event
流程
- ngx_event_process_init: 初始化进程，遍历所有listening，设置rev的回调
- ngx_single_process_cycle/ngx_worker_process_cycle/ngx_cache_manager_process_cycle
- ngx_process_events_and_timers: ngx_process_events
- ngx_process_events: ngx_event_actions.process_events
核心函数
- ngx_event_accept: 实际调用accept，并将socket转化为connection_t
- ngx_event_recvmsg
typedef struct ngx_event_s ngx_event_t;
struct ngx_event_s {
// ngx_connection_t*
void *data;
// some flags
ngx_event_handler_pt handler;
ngx_rbtree_node_t timer;
}
// ngx_event_process_init
rev->handler = (c->type == SOCK_STREAM) ? ngx_event_accept
: ngx_event_recvmsg;
typedef void (*ngx_event_handler_pt)(ngx_event_t *ev);
extern ngx_queue_t ngx_posted_accept_events;
extern ngx_queue_t ngx_posted_next_events;
extern ngx_queue_t ngx_posted_events;
void
ngx_event_process_posted(ngx_cycle_t *cycle, ngx_queue_t *posted)
nginx 变量读写
- nginx定义了一些保留关键字，称为变量，例如$host，和upstream里的ngx_http_upstream_vars
- nginx会把这些变量存储在全局变量中，这样任何模块都可以获取到
- 具体的获取方式为，首先根据变量名拿到index，然后通过index拿到变量的值
typedef struct ngx_http_variable_s ngx_http_variable_t;
struct ngx_http_variable_s {
ngx_str_t name; /* must be first to build the hash */
ngx_http_set_variable_pt set_handler;
ngx_http_get_variable_pt get_handler;
uintptr_t data;
ngx_uint_t flags;
ngx_uint_t index;
};
// add in upstream
var = ngx_http_add_variable(cf, &v->name, v->flags);
var->get_handler = v->get_handler;
var->data = v->data;
// example in http_log
index = ngx_http_get_variable_index(cf, value);
op->data = index;
value = ngx_http_get_indexed_variable(r, op->data);
内存池 ngx_pool_s
- nginx的内存池是内存分配的核心，由于HTTP的会话特性，可以用一个内存池统一分配内存，不考虑释放，等请求结束的时候直接释放内存池中的内存，ngx_destroy_pool
- ngx_pool_s本身管理一块连续的内存，分为头部和data，头部占据ngx_pool_s size的大小，剩下的就是data(ngx_pool_data_t)
- data本身按照递增指针分配内存，如果内存不够，它的next会指向下一个ngx_pool_s，用下一块连续内存尝试分配
- ngx_pool_large_t则是专门为大内存分配的链表，大内存都是即时通过malloc分配的
- 为什么要有current指向pool自己？看看 ngx_palloc_block 函数，创建时pool->current确实就是pool，但是随着内存池内存不足进行扩容时，应该把新的pool插入哪里呢？答案是第一个failed>4的pool，此时它之前的内存池都被放弃分配，让new_pool称为pool->current。因此current只是一个下一个用于分配内存的pool的快速索引，摧毁内存池的时候还是按照d.next的顺序释放。
typedef struct {
u_char *last;
u_char *end;
ngx_pool_t *next;
ngx_uint_t failed;
} ngx_pool_data_t;
struct ngx_pool_s {
ngx_pool_data_t d;
size_t max;
ngx_pool_t *current;
ngx_chain_t *chain;
ngx_pool_large_t *large;
ngx_pool_cleanup_t *cleanup;
ngx_log_t *log;
};
void *
ngx_palloc(ngx_pool_t *pool, size_t size)
{
if (size <= pool->max) {
return ngx_palloc_small(pool, size, 1);
}
return ngx_palloc_large(pool, size);
}
static ngx_inline void *
ngx_palloc_small(ngx_pool_t *pool, size_t size, ngx_uint_t align)
{
u_char *m;
ngx_pool_t *p;
p = pool->current;
do {
m = p->d.last;
if (align) {
m = ngx_align_ptr(m, NGX_ALIGNMENT);
}
if ((size_t) (p->d.end - m) >= size) {
p->d.last = m + size;
return m;
}
p = p->d.next;
} while (p);
return ngx_palloc_block(pool, size);
}
其他内置struct
addr_t
typedef struct {
struct sockaddr *sockaddr;
socklen_t socklen;
ngx_str_t name;
} ngx_addr_t;
ngx_url_t
typedef struct {
ngx_str_t url;
ngx_str_t host;
ngx_str_t port_text;
ngx_str_t uri;
in_port_t port;
in_port_t default_port;
in_port_t last_port;
int family;
unsigned listen:1;
unsigned uri_part:1;
unsigned no_resolve:1;
unsigned no_port:1;
unsigned wildcard:1;
socklen_t socklen;
ngx_sockaddr_t sockaddr;
ngx_addr_t *addrs;
ngx_uint_t naddrs;
char *err;
} ngx_url_t;
ngx_table_elt_s
typedef struct ngx_table_elt_s ngx_table_elt_t;
struct ngx_table_elt_s {
ngx_uint_t hash;
ngx_str_t key;
ngx_str_t value;
u_char *lowcase_key;
ngx_table_elt_t *next;
};
其他内置函数
ngx_http_upstream_init_round_robin_peer
ngx_handle_write_event
ngx_create_temp_buf
nginx 启动
ngx_master_process_cycle
- ngx_master_process_cycle: 启动master
- ngx_init_cycle: 解析配置
- ngx_conf_parse: 解析配置文件
- ngx_conf_read_token: for循环解析配置
