一次用goroutine多线程提速800%的检索文件的例子记录

334 阅读1分钟

下面是一个用goroutine提速检索文件例子,希望可以帮助大家理解golang的多线程

//这是一个单线程搜索文件的例子

package main

import (
 "fmt"
 "io/ioutil"
 "time"
)

var query = "test"
var matches int

func main()  {
 start := time.Now()
 search("/Users/damon/")
 fmt.Println("matches",matches)
 fmt.Println(time.Since(start))
}

func search(path string)  {
 files,errs := ioutil.ReadDir(path)
 if errs == nil {
  for _,file := range files{
   name := file.Name()
   if name == query {
    matches++
   }
   if file.IsDir() {
    search(path + name + "/")
   }
  }
 }
 
//结果
//matches 484
//33.96989532s
package main

import (
 "fmt"
 "io/ioutil"
 "time"
)

var query = "test"
var matches int

//处于工作状态的工人数
var workerCount = 0
//同时允许的最大工人数,即限制goroutine的个数,goroutine 的数量是不受控制的,但是每个 CPU 核上同一时间只能执行一个 goroutine,当 job 很多且生成了相应数目的 goroutine 后,会出现很多等待执行的 goroutine,从而造成资源上的浪费。
var maxWorkerCount = 32
//需要继续向下搜索的目录
var searchRequest = make(chan string)
//工人工作已完成
var workerDone = make(chan bool)
//搜索结果,即匹配成功
var foundMatch = make(chan bool)

func main() {
 start := time.Now()
 workerCount = 1
 go search("/Users/damon/", true)
 //wait
 waitForWorker()
 fmt.Println("matches", matches)
 fmt.Println(time.Since(start))
}

func waitForWorker() {
 for {
  select {
  case path := <-searchRequest:
   workerCount++
   go search(path, true)
  case <-workerDone:
   workerCount--
   if workerCount == 0 {
    return
   }
  case <-foundMatch:
   matches++
  }
 }
}

func search(path string, master bool) {
 //fmt.Println(runtime.NumGoroutine())  观察goroutine数
 files, errs := ioutil.ReadDir(path)
 if errs == nil {
  for _, file := range files {
   name := file.Name()
   if name == query {
    foundMatch <- true
   }
   if file.IsDir() {
    if workerCount <= maxWorkerCount {
     searchRequest <- path + name + "/"
    } else {
     search(path + name + "/", false)
    }
   }
  }
 }
 if master {
  workerDone <- true
 }
}

//结果
//matches 484
//6.435415949s