golang 设置任务并行度

需求： 有5个http请求任务需要执行，设置并行度为2执行，防止对http服务器造成大的压力

不考虑并行度，5个任务启动5个worker，且同时执行：

package main

import (
	"fmt"
	"math/rand"
	"time"
)


func main() {
	// 模拟每个任务一个goroutine 执行，
	linar_test()

	for {
	}
}

func linar_test() {
	arrs := []int{1, 2, 3, 4, 5}
	for {
		if pop, end := linar_consumer(&arrs); !end {
			ch := make(chan string)
			go worker(pop, ch)
			//fmt.Println("worker res:", <-ch)
		} else {
			break
		}
	}

}

func linar_consumer(arrs *[]int) (int, bool) {
	if len(*arrs) == 0 {
		return 0, true
	}

	pop := (*arrs)[0]
	*arrs = (*arrs)[1:]

	return pop, false
}

func randomSleep() {
	sec := rand.Intn(3)

	fmt.Println("time sleep:", sec)
	time.Sleep(time.Second * time.Duration(sec))

}

func worker(id int, ch chan string) {
	fmt.Println("worker processing id:", id)

	randomSleep()
	res := fmt.Sprintf("worker handler job:%d", id)
	ch <- res
	return
}

考虑2并行度: 5个任务启5个goroutine,通过缓冲队列长度为2的token chan来控制任务的并发执行

package main

import (
	"fmt"
	"math/rand"
	"net/http"
	"time"
)

var (
	TokenQ = make(chan int, 3)
)

func main() {

	// 模拟最高2并发之行
	concurrent := 2
	h := NewHandler(concurrent)
	h.Run()

	for {
	}
}

func randomSleep() {
	sec := rand.Intn(3)

	//fmt.Println("time sleep:", sec)
	time.Sleep(time.Second * time.Duration(sec))

}

type TaskHandler struct {
	TokenQ chan int

	TaskQ chan int
}

func NewHandler(concurrent int) *TaskHandler {
	t := &TaskHandler{
		TaskQ:  make(chan int, concurrent),
		TokenQ: make(chan int, concurrent),
	}

	// 根据并行度放置可用的token 个数
	for i := 0; i < concurrent; i++ {
		t.TokenQ <- i
	}

	return t
}

func (t *TaskHandler) PushTask(arrs *[]int) bool {

	if len(*arrs) == 0 {
		return false
	}

	pop := (*arrs)[0]
	*arrs = (*arrs)[1:]

	t.TaskQ <- pop
	return true
}

func (t *TaskHandler) GetWorker(task int) {
	token := <-t.TokenQ //取一个可用的token

	//fmt.Println("-->get one token:", token)

	getTaskTest()
	fmt.Println("<--worker handler job finished, token:%d, task:%d", token, task)
	t.TokenQ <- token //执行完毕token放回
}

func (t *TaskHandler) Run() {
	arrs := []int{1, 2, 3, 4, 5}
	go func() {
		for {
			t.PushTask(&arrs)
		}
	}()

	for {
		task := <-t.TaskQ
		go t.GetWorker(task)
	}
}

func getTaskTest() {
	randomSleep()
	resp, err := http.Get("https://www.baidu.com")
	if err != nil {
		// handle error
		fmt.Println("err:", err)
		return
	}
	defer resp.Body.Close()

	fmt.Println(resp.Status)
}

微调，考虑并行度的同时，限制goroutine 启动个数：

package main

import (
	"fmt"
	"math/rand"
	"net/http"
	"time"
)

var (
	TokenQ = make(chan int, 3)
)

func main() {

	// 模拟最高2并发之行
	concurrent := 2
	h := NewHandler(concurrent)
	h.Run()

	for {
	}
}

func randomSleep() {
	sec := rand.Intn(3)

	//fmt.Println("time sleep:", sec)
	time.Sleep(time.Second * time.Duration(sec))

}

type TaskHandler struct {
	TokenQ chan int

	TaskQ chan int
}

func NewHandler(concurrent int) *TaskHandler {
	t := &TaskHandler{
		TaskQ:  make(chan int, concurrent),
		TokenQ: make(chan int, concurrent),
	}

	// 根据并行度放置可用的token 个数
	for i := 0; i < concurrent; i++ {
		t.TokenQ <- i
	}

	return t
}

func (t *TaskHandler) PushTask(arrs *[]int) bool {

	if len(*arrs) == 0 {
		return false
	}

	pop := (*arrs)[0]
	*arrs = (*arrs)[1:]

	t.TaskQ <- pop
	return true
}

func (t *TaskHandler) GetWorker(token, task int) {
	getTaskTest()
	fmt.Println("<--worker handler job finished, token:%d, task:%d", token, task)

	t.TokenQ <- token //在运行任务的goroutine 最后，将token放回
}

func (t *TaskHandler) Run() {
	arrs := []int{1, 2, 3, 4, 5}
	go func() {
		for {
			t.PushTask(&arrs)
		}
	}()

	for {
		task := <-t.TaskQ
		token := <-t.TokenQ //在启动一个新goroutine 前，取一个可用的token
		go t.GetWorker(token, task)

	}
}

func getTaskTest() bool {
	randomSleep()
	resp, err := http.Get("https://www.baidu.com")
	if err != nil {
		// handle error
		fmt.Println("err:", err)
		return false
	}
	defer resp.Body.Close()

	fmt.Println(resp.Status)
	return true
}