生产者消费者模型

Golang

package main

import (
	"fmt"
	"math/rand"
	"time"
)

//生产者
func produce(ch chan<- int) {
	for {
		time.Sleep(time.Millisecond * 1500)
		ranNum := rand.Intn(10)
		fmt.Println("produce end:", ranNum)
		ch <- ranNum
	}
}

//消费者
func consume(ch <-chan int) {
	for n := range ch {
		fmt.Println("consumer:", n)
		time.Sleep(time.Second)
	}
}

//生产者和消费者模型
func main() {
	ch := make(chan int)

	//三个生产者
	for i := 0; i < 3; i++ {
		go produce(ch)
	}
	//两个消费者
	for i := 0; i < 2; i++ {
		go consume(ch)
	}

	select {}
}

生产者消费者模型下载图片

package main

import (
   "bufio"
   "fmt"
   "github.com/imroc/req/v3"
   "io"
   "io/ioutil"
   "log"
   "os"
   "path"
   "path/filepath"
   "strings"
)

var start = 0

func main() {
   ch := make(chan string, 100)
   finish := make(chan bool)

   for i:=0;i<1;i++ {
      go func() {
         for line := range ch {
            download(fmt.Sprintf("https://artlib.cn/%s", line))
         }

         if len(ch) ==0 {
            finish <- true
         }
      }()
   }

   go func() {
      ReadLineForFile("urls.txt", func(line string) {
         ch <- line
      })

      close(ch)
   }()

   <- finish
   fmt.Println("finish.")
}

func download(urlAddr string) {
   response, err := req.Get(urlAddr)
   if err != nil {
      panic(err)
   }

   path1 := strings.TrimLeft(urlAddr, "https://artlib.cn/")
   filename := filepath.Base(path1)
   dirname := filepath.Dir(path1)

   _, err = os.Stat(path.Join(dirname, filename))
   if !os.IsNotExist(err) {
      fmt.Println(path.Join(dirname, filename), "文件已存在, 跳过...")
      return
   }

   _, err = os.Stat(dirname)
   if os.IsNotExist(err) {
      err = os.MkdirAll(dirname, 0777)
      if err != nil {
         panic(err)
      }
   }

   err = ioutil.WriteFile(path.Join(dirname, filename), response.Bytes(), 0777)
   if err != nil {
      panic(err)
   }

   start ++
   fmt.Println("download->", start, urlAddr)
}


func ReadLineForFile(filename string, callback func(line string) ) {
   f, err := os.Open(filename)
   if err != nil {
      log.Fatal(err)
   }
   defer f.Close()

   reader := bufio.NewReader(f)
   for {
      line, _, err := reader.ReadLine()
      if err == io.EOF {
         break
      }

      if len(line) > 0 {
         callback(strings.TrimSpace(string(line)))
      }
   }
}

Python

import threading
from queue import Queue
import re

q = Queue(1000)


def product():
    f = open('data.txt')
    for line in f:
        line_slice = re.split(r"\s+", line.strip())
        if len(line_slice) == 2:
            work_id, imgs = line_slice
            q.put(f'{work_id}|{imgs}')


def consumer():
    while not q.empty():
        data = q.get()
        print(f"data: {data}, empty: {q.empty()}: qsize: {q.qsize()} thread name: {threading.current_thread().name}")
    print('empty.', threading.current_thread().name)


if __name__ == '__main__':
    p1 = threading.Thread(target=product)
    p1.start()

    consumers = []
    for i in range(100):
        t = threading.Thread(target=consumer)
        consumers.append(t)

    for thread in consumers:
        thread.start()

    for thread in consumers:
        thread.join()

    p1.join()
    print('finish.')