回顾
上一期上文中我们讲解了
- 如何使用gin构建一个标准的web工程
- 如何使用gin写出标准的api接口
- ...
我们在上一期中编写了一个api接口、配置了数据库链接。学会了上一期内容的话，基本上编写增删查改的业务已经没有问题了。这一期我们主要讲一下在golang的一些编码思路，来帮我们补充一些框架上的功能，以及gin的一些拓展应用。
这一期我们的主要目标是
- 完善web框架 （借助golang的协程和chan来完成一个限流中间件）
- gin框架的进阶 （文件上传、静态文件、重定向...）
同样的我们接着上一期的工程（server1分支）继续
但是工程地址的分支从这一期开始切换为server1分支。希望大家注意
查看一下目录
➜ go-gin-test git:(server1) ✗ tree -L 3
.
├── api
│ └── api.go
├── db
│ └── db.go
├── go.mod
├── go.sum
├── hello
├── mian.go
├── model
│ └── user.go
├── routerex
│ ├── mid
│ │ └── mid.go
│ └── router.go
├── server
│ └── server.go
└── service
完善web框架 -- 实现一个限流中间件
目标： 实现一个限流中间件
说明： 首先我们假设现在有这么一个需求，我们需要限制某一个接口同一时间最多只处理来自于10个不同的IP的请求并且同一个IP在同一时间只允许10个请求。也就是说我们限制为最多有100个请求。但是，如果在我们满负载的情况，我们不能丢弃或者直接返回后续的请求，而是应该让他们排队。
要点：
- 数据结构的设计
- 利用go的协程
- 借助chan的特性
思路分析：
- 通过分析需求，感觉和队列有些相似，那么在数据结构上多半为数组的形态。实际上我们会控制两个数组，一个是IP的，另一个是IP的请求。那在我们的大脑中一个大概的数据结构就已经出现
- go的协程能够很好的帮助我们完成异步的处理，我们只需要控制同时开始的协程数量就好了
- 借助有大小的chan的特性，我们就能控制同一时间最大有几个协程运行，而不需要一个线程池来控制
代码:
在mid目录下新建limit.go
package mid
import (
"container/list"
"fmt"
"reflect"
"sync"
"github.com/gin-gonic/gin"
)
//限流中间价
var LimitMid *MidLimit
//用来存放请求的token
var TokenMap sync.Map
//限流中间价的接口
type IMidLimit interface {
Put(gCtx *gin.Context, Token string) bool
Start()
}
//限流中间价的结构体
type MidLimit struct {
//使用sync Map 防止并发问题
sm sync.Map
//同一时间访问的IP的chan，用来限制IP数量
mc chan *Task
}
//初始化限流中间件
func InitLimitMid() {
defer func() {
if err := recover(); err != nil {
fmt.Println(fmt.Errorf("InitLimitMid err:%+v", err))
fmt.Println(fmt.Errorf("InitLimitMid err:%+v", err))
}
}()
l := &MidLimit{}
l.sm = sync.Map{}
//最大为10个IP
l.mc = make(chan *Task, 10)
LimitMid = l
TokenMap = sync.Map{}
//启动一个协程去监听这个chan
go LimitMid.Start()
}
//IP 的chan的监听
func (c *MidLimit) Start() {
defer func() {
if err := recover(); err != nil {
fmt.Println(fmt.Errorf("MidLimit Start err:%+v", err))
fmt.Println(fmt.Errorf("MidLimit Start err:%+v", err))
}
}()
if c.mc == nil {
fmt.Println("MidLimit err: chan is empty")
return
}
//如果有任务，就直接运行
for task := range c.mc {
go task.Run()
}
}
//放入请求
func (c *MidLimit) Put(gCtx *gin.Context, Token string) bool {
defer func() {
if err := recover(); err != nil {
fmt.Println(fmt.Errorf("MidLimit Put err:%+v", err))
fmt.Println(fmt.Errorf("MidLimit Put err:%+v", err))
}
}()
if c.mc == nil {
fmt.Println("MidLimit chan is empty")
return false
}
//获取IP
ip := gCtx.ClientIP()
load, ok := c.sm.Load(ip)
task := &Task{}
if ok {
if reflect.TypeOf(load).Kind() != reflect.Ptr {
return false
}
task = load.(*Task)
}
//上锁，防止数据出问题
task.lk.Lock()
defer task.lk.Unlock()
if task.ls == nil {
task.ls = list.New()
task.IP = ip
task.tc = make(chan *Item, 500)
}
//放到队列的最末端
task.ls.PushBack(&Item{Token: Token, GCtx: gCtx})
//发送给chan
LimitMid.mc <- task
c.sm.Store(ip, task)
TokenMap.Store(Token, Token)
return true
}
//IP结构体，里面保存的是同一个IP的所有请求
type Task struct {
IP string
//锁，主要是保证安全
lk sync.Mutex
//请求的list
ls *list.List
//处理请求的chan
tc chan *Item
}
func (c *Task) Run() {
defer func() {
if err := recover(); err != nil {
fmt.Println(fmt.Errorf("Task Run err:%+v", err))
fmt.Println(fmt.Errorf("Task Run err:%+v", err))
}
}()
go func() {
for item := range c.tc {
TokenMap.Delete(item.Token)
}
}()
c.lk.Lock()
defer c.lk.Unlock()
//每次最多处理10个请求
max := 10
if c.ls.Len() > 0 && c.ls.Len() < 10 {
max = c.ls.Len()
}
fmt.Printf("Before ------ IP : %v count: %v", c.IP, c.ls.Len())
fmt.Println()
//TODO 这里可以修改逻辑，不一定要按我的思路
for i := 0; i < max; i++ {
front := c.ls.Front()
if front == nil {
continue
}
if reflect.TypeOf(front.Value).Kind() != reflect.Ptr {
fmt.Println("Task Run reflect err not Ptr")
}
item := front.Value.(*Item)
c.tc <- item
//处理一个，就删除一个
c.ls.Remove(front)
}
fmt.Printf("After ------ IP : %v count: %v", c.IP, c.ls.Len())
fmt.Println()
//大于10个的部分，再把它放回IP的chan里面重新排队
if c.ls.Len() > 0 {
LimitMid.mc <- c
}
}
//实际的请求
type Item struct {
Token string
GCtx *gin.Context
}
获取一下uuid
go get github.com/satori/go.uuid
修改一下router.go和server.go
router.go
package routerex
import (
"fmt"
"example.com/m/v2/api"
"example.com/m/v2/routerex/mid"
"github.com/gin-gonic/gin"
uuid "github.com/satori/go.uuid"
)
func MidLimit(c *gin.Context) {
fmt.Println(fmt.Printf("---- %v", c.GetHeader("User-Agent")))
s := uuid.NewV4().String()
mid.LimitMid.Put(c, s)
for {
_, ok := mid.TokenMap.Load(s)
if !ok {
break
}
}
c.Next()
}
//用作注册请求地址
func RegRouter(g *gin.Engine) {
g1 := g.Group("/api")
//中间件
g1.Use(mid.MidCors)
g1.Use(MidLimit)
g1.POST("/register", api.Register)
g1.GET("/limit", func(c *gin.Context) {
c.JSON(200, gin.H{
"msg": "ok",
})
})
}
server.go
package server
import (
"fmt"
"example.com/m/v2/db"
"example.com/m/v2/routerex"
"example.com/m/v2/routerex/mid"
"github.com/gin-gonic/gin"
)
func ServerStart(port string) {
r := gin.Default()
routerex.RegRouter(r)
mid.InitLimitMid()
err := db.InitDB()
if err != nil {
fmt.Println(fmt.Errorf("init db err: %v", err))
}
r.Run(":" + port) // listen and serve on 0.0.0.0:8080 (for windows "localhost:8080")
}
随后就是编译和启动
这里我们使用一个go编写的测试工具进行并发测试
地址：https://github.com/link1st/go-stress-testing
下载二进制
运行
20个并发，一个并发请求10次
./go-stress-testing-mac -c 20 -n 20 -u localhost:8080/api/limit
服务日志:
Before ------ IP : ::1 count: 15
After ------ IP : ::1 count: 5
Before ------ IP : ::1 count: 5
After ------ IP : ::1 count: 0
Before ------ IP : ::1 count: 0
After ------ IP : ::1 count: 0
[GIN] 2021/04/09 - 22:02:52 | 200 | 52.975949ms | ::1 | GET "/api/limit"
[GIN] 2021/04/09 - 22:02:52 | 200 | 33.017938ms | ::1 | GET "/api/limit"
[GIN] 2021/04/09 - 22:02:52 | 200 | 32.982102ms | ::1 | GET "/api/limit"
[GIN] 2021/04/09 - 22:02:52 | 200 | 49.785125ms | ::1 | GET "/api/limit"
[GIN] 2021/04/09 - 22:02:52 | 200 | 33.019011ms | ::1 | GET "/api/limit"
Before ------ IP : ::1 count: 0
After ------ IP : ::1 count: 0
[GIN] 2021/04/09 - 22:02:52 | 200 | 33.045255ms | ::1 | GET "/api/limit"
Before ------ IP : ::1 count: 0
After ------ IP : ::1 count: 0
---- Go-http-client/1.123 <nil>
---- Go-http-client/1.123 <nil>
[GIN] 2021/04/09 - 22:02:52 | 200 | 65.433351ms | ::1 | GET "/api/limit"
[GIN] 2021/04/09 - 22:02:52 | 200 | 50.847047ms | ::1 | GET "/api/limit"
Before ------ IP : ::1 count: 1
[GIN] 2021/04/09 - 22:02:52 | 200 | 506.656µs | ::1 | GET "/api/limit"
[GIN] 2021/04/09 - 22:02:52 | 200 | 9.930572ms | ::1 | GET "/api/limit"
---- Go-http-client/1.123 <nil>
Before ------ IP : ::1 count: 3
After ------ IP : ::1 count: 0
[GIN] 2021/04/09 - 22:02:52 | 200 | 1.372397ms | ::1 | GET "/api/limit"
[GIN] 2021/04/09 - 22:02:52 | 200 | 602.793µs | ::1 | GET "/api/limit"
[GIN] 2021/04/09 - 22:02:52 | 200 | 1.377011ms | ::1 | GET "/api/limit"
---- Go-http-client/1.123 <nil>
---- Go-http-client/1.123 <nil>
---- Go-http-client/1.123 <nil>
---- Go-http-client/1.123 <nil>
---- Go-http-client/1.123 <nil>
---- Go-http-client/1.123 <nil>
---- Go-http-client/1.123 <nil>
[GIN] 2021/04/09 - 22:02:52 | 200 | 72.990133ms | ::1 | GET "/api/limit"
[GIN] 2021/04/09 - 22:02:52 | 200 | 53.24303ms | ::1 | GET "/api/limit"
[GIN] 2021/04/09 - 22:02:52 | 200 | 52.235207ms | ::1 | GET "/api/limit"
[GIN] 2021/04/09 - 22:02:52 | 200 | 53.10638ms | ::1 | GET "/api/limit"
---- Go-http-client/1.123 <nil>
[GIN] 2021/04/09 - 22:02:52 | 200 | 50.019364ms | ::1 | GET "/api/limit"
[GIN] 2021/04/09 - 22:02:52 | 200 | 53.408403ms | ::1 | GET "/api/limit"
[GIN] 2021/04/09 - 22:02:52 | 200 | 51.147636ms | ::1 | GET "/api/limit"
[GIN] 2021/04/09 - 22:02:52 | 200 | 53.404998ms | ::1 | GET "/api/limit"
---- Go-http-client/1.123 <nil>
Before ------ IP : ::1 count: 4
After ------ IP : ::1 count: 0
Before ------ IP : ::1 count: 0
After ------ IP : ::1 count: 0
Before ------ IP : ::1 count: 0
After ------ IP : ::1 count: 0
[GIN] 2021/04/09 - 22:02:52 | 200 | 12.975392ms | ::1 | GET "/api/limit"
---- Go-http-client/1.123 <nil>
---- Go-http-client/1.123 <nil>
Before ------ IP : ::1 count: 2
After ------ IP : ::1 count: 0
Before ------ IP : ::1 count: 0
After ------ IP : ::1 count: 0
[GIN] 2021/04/09 - 22:02:52 | 200 | 13.068885ms | ::1 | GET "/api/limit"
Before ------ IP : ::1 count: 1
After ------ IP : ::1 count: 0
Before ------ IP : ::1 count: 0
After ------ IP : ::1 count: 0
[GIN] 2021/04/09 - 22:02:52 | 200 | 13.5707ms | ::1 | GET "/api/limit"
---- Go-http-client/1.123 <nil>
---- Go-http-client/1.123 <nil>
---- Go-http-client/1.123 <nil>
---- Go-http-client/1.123 <nil>
---- Go-http-client/1.123 <nil>
[GIN] 2021/04/09 - 22:02:52 | 200 | 55.673286ms | ::1 | GET "/api/limit"
---- Go-http-client/1.123 <nil>
---- Go-http-client/1.123 <nil>
Before ------ IP : ::1 count: 6
After ------ IP : ::1 count: 0
Before ------ IP : ::1 count: 0
After ------ IP : ::1 count: 0
[GIN] 2021/04/09 - 22:02:52 | 200 | 53.126323ms | ::1 | GET "/api/limit"
---- Go-http-client/1.123 <nil>
---- Go-http-client/1.123 <nil>
[GIN] 2021/04/09 - 22:02:52 | 200 | 8.151735ms | ::1 | GET "/api/limit"
---- Go-http-client/1.123 <nil>
[GIN] 2021/04/09 - 22:02:52 | 200 | 53.223775ms | ::1 | GET "/api/limit"
---- Go-http-client/1.123 <nil>
Before ------ IP : ::1 count: 8
After ------ IP : ::1 count: 0
---- Go-http-client/1.123 <nil>
Before ------ IP : ::1 count: 1
After ------ IP : ::1 count: 0
---- Go-http-client/1.123 <nil>
Before ------ IP : ::1 count: 2
After ------ IP : ::1 count: 0
Before ------ IP : ::1 count: 0
After ------ IP : ::1 count: 0
[GIN] 2021/04/09 - 22:02:52 | 200 | 826.491µs | ::1 | GET "/api/limit"
[GIN] 2021/04/09 - 22:02:52 | 200 | 1.040488ms | ::1 | GET "/api/limit"
---- Go-http-client/1.123 <nil>
Before ------ IP : ::1 count: 2
After ------ IP : ::1 count: 0
[GIN] 2021/04/09 - 22:02:52 | 200 | 9.693082ms | ::1 | GET "/api/limit"
---- Go-http-client/1.123 <nil>
[GIN] 2021/04/09 - 22:02:52 | 200 | 27.692747ms | ::1 | GET "/api/limit"
Before ------ IP : ::1 count: 1
After ------ IP : ::1 count: 0
Before ------ IP : ::1 count: 0
After ------ IP : ::1 count: 0
Before ------ IP : ::1 count: 0
After ------ IP : ::1 count: 0
Before ------ IP : ::1 count: 0
After ------ IP : ::1 count: 0
Before ------ IP : ::1 count: 0
After ------ IP : ::1 count: 0
Before ------ IP : ::1 count: 0
After ------ IP : ::1 count: 0
---- Go-http-client/1.123 <nil>
[GIN] 2021/04/09 - 22:02:52 | 200 | 838.92µs | ::1 | GET "/api/limit"
---- Go-http-client/1.123 <nil>
---- Go-http-client/1.123 <nil>
---- Go-http-client/1.123 <nil>
[GIN] 2021/04/09 - 22:02:52 | 200 | 41.30448ms | ::1 | GET "/api/limit"
[GIN] 2021/04/09 - 22:02:52 | 200 | 41.646191ms | ::1 | GET "/api/limit"
[GIN] 2021/04/09 - 22:02:52 | 200 | 33.808401ms | ::1 | GET "/api/limit"
---- Go-http-client/1.123 <nil>
[GIN] 2021/04/09 - 22:02:52 | 200 | 42.36363ms | ::1 | GET "/api/limit"
[GIN] 2021/04/09 - 22:02:52 | 200 | 38.868172ms | ::1 | GET "/api/limit"
[GIN] 2021/04/09 - 22:02:52 | 200 | 465.389µs | ::1 | GET "/api/limit"
[GIN] 2021/04/09 - 22:02:52 | 200 | 492.525µs | ::1 | GET "/api/limit"
Before ------ IP : ::1 count: 2
After ------ IP : ::1 count: 0
Before ------ IP : ::1 count: 0
After ------ IP : ::1 count: 0
Before ------ IP : ::1 count: 0
After ------ IP : ::1 count: 0
Before ------ IP : ::1 count: 0
After ------ IP : ::1 count: 0
Before ------ IP : ::1 count: 1
After ------ IP : ::1 count: 0
[GIN] 2021/04/09 - 22:02:52 | 200 | 278.108µs | ::1 | GET "/api/limit"
[GIN] 2021/04/09 - 22:02:52 | 200 | 121.357µs | ::1 | GET "/api/limit"
Before ------ IP : ::1 count: 1
After ------ IP : ::1 count: 0
[GIN] 2021/04/09 - 22:02:52 | 200 | 341.215µs | ::1 | GET "/api/limit"
---- Go-http-client/1.123 <nil>
Before ------ IP : ::1 count: 2
After ------ IP : ::1 count: 0
[GIN] 2021/04/09 - 22:02:52 | 200 | 444.966µs | ::1 | GET "/api/limit"
[GIN] 2021/04/09 - 22:02:52 | 200 | 956.363µs | ::1 | GET "/api/limit"
Before ------ IP : ::1 count: 1
After ------ IP : ::1 count: 0
[GIN] 2021/04/09 - 22:02:52 | 200 | 237.543µs | ::1 | GET "/api/limit"
Before ------ IP : ::1 count: 0
After ------ IP : ::1 count: 0
Before ------ IP : ::1 count: 0
After ------ IP : ::1 count: 0
[GIN] 2021/04/09 - 22:02:52 | 200 | 22.753412ms | ::1 | GET "/api/limit"
[GIN] 2021/04/09 - 22:02:52 | 200 | 62.916527ms | ::1 | GET "/api/limit"
[GIN] 2021/04/09 - 22:02:52 | 200 | 63.053594ms | ::1 | GET "/api/limit"
[GIN] 2021/04/09 - 22:02:52 | 200 | 63.170451ms | ::1 | GET "/api/limit"
[GIN] 2021/04/09 - 22:02:52 | 200 | 63.097772ms | ::1 | GET "/api/limit"
[GIN] 2021/04/09 - 22:02:52 | 200 | 41.751858ms | ::1 | GET "/api/limit"
---- Go-http-client/1.123 <nil>
[GIN] 2021/04/09 - 22:02:52 | 200 | 63.128788ms | ::1 | GET "/api/limit"
[GIN] 2021/04/09 - 22:02:52 | 200 | 15.458979ms | ::1 | GET "/api/limit"
[GIN] 2021/04/09 - 22:02:52 | 200 | 60.814893ms | ::1 | GET "/api/limit"
[GIN] 2021/04/09 - 22:02:52 | 200 | 63.177745ms | ::1 | GET "/api/limit"
23 <nil>
Before ------ IP : ::1 count: 1
After ------ IP : ::1 count: 0
[GIN] 2021/04/09 - 22:02:52 | 200 | 63.054733ms | ::1 | GET "/api/limit"
[GIN] 2021/04/09 - 22:02:52 | 200 | 63.346652ms | ::1 | GET "/api/limit"
[GIN] 2021/04/09 - 22:02:52 | 200 | 367.146µs | ::1 | GET "/api/limit"
Before ------ IP : ::1 count: 1
Before ------ IP : ::1 count: 1
After ------ IP : ::1 count: 0
[GIN] 2021/04/09 - 22:02:52 | 200 | 132.156µs | ::1 | GET "/api/limit"
[GIN] 2021/04/09 - 22:02:52 | 200 | 105.188µs | ::1 | GET "/api/limit"
这里的日志不太全，但是我们可以看出每次处理的请求不超过10个，而且如果数组大于10个，也只会处理10个。大家可以自行看下自己的控制台
gin框架的进阶 （文件上传、静态文件、重定向...）
- 上传文件
func(c *gin.Context) {
// 单文件
file, _ := c.FormFile("file")
log.Println(file.Filename)
// 上传文件到指定的路径
// c.SaveUploadedFile(file, dst)
c.String(http.StatusOK, fmt.Sprintf("'%s' uploaded!", file.Filename))
}
func(c *gin.Context) {
// 多文件
form, _ := c.MultipartForm()
files := form.File["upload[]"]
for _, file := range files {
log.Println(file.Filename)
// 上传文件到指定的路径
// c.SaveUploadedFile(file, dst)
}
c.String(http.StatusOK, fmt.Sprintf("%d files uploaded!", len(files)))
}
- 静态文件
router := gin.Default()
router.Static("/assets", "./assets")
router.StaticFS("/more_static", http.Dir("my_file_system"))
router.StaticFile("/favicon.ico", "./resources/favicon.ico")
- 重定向
func(c *gin.Context) {
c.Redirect(http.StatusMovedPermanently, "http://www.google.com/")
}
func(c *gin.Context) {
c.Request.URL.Path = "/test2"
r.HandleContext(c)
}
总结
-
其实限流的中间件还有很多可以进步的地方，在这里地方主要是想让大家联系一下go语言的特性和拓展大家的编程思路。
-
gin框架的常用代码我也已经介绍完毕
-
额。可能是因为我没有人关注的原因吧，关于gin框架也没有收到同学的反馈，我也不知道自己写得咋样，由于没有同学来问问题，那么gin框架的文章就到此为一段落
想获取工程的同学可以关注
超级英雄吉姆，在公众号里发送gin，获取工程。
gin相关的文章就到此结束，如果还有任何问题，可以关注我的公众号超级英雄吉姆，在公众号留言，我看到后第一时间回复。
后话
求求大家给个关注。要吃不上饭了~
