多进程的两种实现方式

Process函数传递target参数，该参数为新的进程需要执行的代码。

或者通过继承Process,实现run方法来达到多进程的目的。

import time
from multiprocessing import Process

def task(name):
    print(f"{name} begin")
    time.sleep(3)
    print(f"{name} finish")
    
class MyProcess(Process):
    def run(self):
        print("自定义进程")
        time.sleep(3)
        print("自定义进程结束")


if __name__ == "__main__":
    p1 = Process(target=task, args=("hello",))
    p2 = Process(target=task, args=("world",))
    p3 = MyProcess()
    p1.start()
    p2.start()
    p3.start()

join等待进程结束

import time
from multiprocessing import Process

def task():
    for i in range(10):
        print(f"download with {i*10}%")

def show():
    print("文件下载完成")

if __name__ == "__main__":
    p1 = Process(target=task)
    p2 = Process(target=show)
    p1.start()
    p1.join()  # 在进程p1完成之前阻塞。p1完成之后才会之后后续的任务
    p2.start()

进程之间的数据隔离

进程之间的数据是隔离的，进程创建的时候会复制一份当前的数据。

from multiprocessing import Process

age = 10


def task():
    global age;
    age += 1
    print(f"task age is {age}")


class MyProcess(Process):
    def run(self):
        global age
        age += 100
        print(f"Myprocess age is {age}")


if __name__ == "__main__":
    p1 = Process(target=task)
    p2 = MyProcess()
    p1.start()
    p2.start()
    p1.join()
    p2.join()
    print(f"main age is {age}")

进程通信

第一种方式：使用管道,macOS测试无效，Cenos7测试有效

from multiprocessing import Process, Pipe

fd1, fd2 = Pipe()


def app1():
    print("app1：发送消息 hello world")
    fd1.send("hello world")
    data = fd1.recv()
    print(f"app1: 收到消息 {data}")


def app2():
    data = fd2.recv()
    print("app2: 收到消息", data)
    print('app2: 发送消息 ("hello", 2022)')
    fd2.send(("hello", 2022))
    print('app2: 发送消息 3.14')
    fd2.send(3.14)


if __name__ == '__main__':
    p1 = Process(target=app1)
    p2 = Process(target=app2)
    p1.start()
    p2.start()
    print(f"main {fd1.recv()}")

程序输出

app1：发送消息 hello world

app2: 收到消息 hello world

app2: 发送消息 ("hello", 2022)

app2: 发送消息 3.14

main ('hello', 2022)

app1: 收到消息 3.14

第二种方式：使用消息队列Queue

from multiprocessing import Queue

q = Queue(5)  # 队列的数据容量

q.put(123)
q.put("Hello world")
print(q.get())
print(q.get())

# 此时队列中没有数据，会阻塞，等待填入数据之后再读取
# print(q.get())

# 如果没有值会报错
# print(q.get_nowait())

# 设置超时时间，时间到了还没有值，则会报错
# print(q.get(timeout=3))

print(q.empty())
print(q.full())

'''
q.full()
q.empty()
q.get_nowait()
多进程场景下数据不够精确
'''

多进程使用Queue通信，macOS无效。Centos7有效

from multiprocessing import Process, Queue

q = Queue()

def app1():
    print(f"app1 {q.get()}")
    q.put(("Hello", 2022))


def app2():
    print(f"app2 {q.put(123)}")
    print(f"app2 {q.get()}")


if __name__ == '__main__':
    Process(target=app1).start()
    Process(target=app2).start()
# 输出
app2 None

app1 123

app2 ('Hello', 2022)

共享内存的方式进行进程之间的通信

# 共享内存的方式进行进程之间的通信
import time
from multiprocessing import Value, Array, Process

data = Value("i", 100)


def app1():
    print(data.value)
    time.sleep(3)
    print(data.value)


def app2():
    time.sleep(2)
    data.value = 3


if __name__ == '__main__':
    Process(target=app1).start()
    Process(target=app2).start()

# 共享内存的方式进行进程之间的通信2,数据的方式
import time
from multiprocessing import Value, Array, Process

'''
i int 
I Long
l int
L long
f float
d double
'''

# data = Array("i", 5)  # 5个0
data = Array("i", [100, 200, 300, 400, 500])


def app1():
    print(data[1])
    time.sleep(3)
    print(data[3])


def app2():
    time.sleep(2)
    data[3] = 1000


if __name__ == '__main__':
    Process(target=app1).start()
    Process(target=app2).start()

信号量：该方式在不同系统表现不一致，Semaphore(2)在macOS可以获取3次锁，在CentOS只能获取两次锁，因此是macOS的bug

# 进程之间的通信，信号量
'''

from multiprocessing import Semaphore

# 创建信号量
sem = Semaphore(10)

sem.acquire()  # 信号量减1，

sem.release()  # 信号量加1

sem.get_value() # 获取信号量

print(sem.get_value())
'''

import os
from multiprocessing import Semaphore, Process
from time import sleep

sem = Semaphore(2)


def app1():
    sem.acquire()  # 信号量减1
    print("app1 start")
    sleep(3)
    sem.release()  # 信号量加1
    print("app1 finished")


def app2():
    sem.acquire()  # 信号量减1
    print("app2 start")
    sleep(3)
    sem.release()  # 信号量加1
    print("app2 finished")


def app3():
    sem.acquire()  # 信号量减1
    print("app3 start")
    sleep(3)
    sem.release()  # 信号量加1
    print("app3 finished")


def app4():
    sem.acquire()  # 信号量减1
    print("app4 start")
    sleep(3)
    sem.release()  # 信号量加1
    print("app4 finished")


if __name__ == '__main__':
    Process(target=app1).start()
    Process(target=app2).start()
    Process(target=app3).start()
    Process(target=app4).start()

进程属性

from multiprocessing import Process, current_process
import os
'''
os.getpid()获取当前的进程号
os.getppid()获取当前进程的父进程号
current_process().name 获取进程的名称，主线程是MainProcess,子线程为Process-1,Process-2
Process(name="helloworld")可以指定进程的名称

'''
def app1():
    print(current_process())
    print(current_process().name)
    print(current_process().pid)
    print(os.getpid())
    print(os.getppid())

def app2():
    print(current_process())
    print(current_process().name)


if __name__ == '__main__':
    p = Process(target=app1)
    p.start()
    p.join()
    print("main", current_process())
    print("main", current_process().name)
    print("main", current_process().pid)
    Process(target=app2, name="hello world").start()

进程传递参数

from multiprocessing import Process


def app1(a, b):
    print("app1:", a, b)


def app2(name, age=18, ):
    print(f"app2:name is {name} age is {age}")


if __name__ == '__main__':
    Process(target=app1, args=("hello", 2022)).start()
    Process(target=app2, kwargs={"name": "张三"}).start()
    Process(target=app2, kwargs={"name": "lisi", "age": 12}).start()

进程方法

'''
is_alive判断进程是否在运行
terminate结束进程，并不立刻结束，只是通知操作系统结束掉进程。因此如果需要某个进程退出后再执行，可以使用join

'''
import time
from multiprocessing import Process, current_process

def app1():
    print("app1 begin", current_process(), current_process().pid)
    time.sleep(10)
    print("app1 finished")


if __name__ == '__main__':
    p = Process(target=app1)
    p.start()

    time.sleep(2)
    print(p.is_alive())  # True
    p.terminate()
    print(p.is_alive()) # True
    time.sleep(3)
    print(p.is_alive())
    print("time finished")

守护进程

# 默认主线程执行结束，会等待子线程执行结束。
# 守护进程，设置子进程为守护进程，在主线程执行结束的时候自动结束子进程

from multiprocessing import Process
import time


def app1():
    while True:
        print("子进程1。。。")
        time.sleep(1)


def app2():
    while True:
        print("子进程2.。。。")
        time.sleep(1)


if __name__ == '__main__':
    # 创建进程的时候指定为守护进程
    p1 = Process(target=app1, daemon=True)
    p1.start()

    # 创建进程之后，再设置为守护进程
    p2 = Process(target=app2)
    p2.daemon = True  # 必须在start之前
    p2.start()
    # p2.daemon = False  # 如果在start之后会报错

    time.sleep(3)
    print("主进程执行结束")

僵尸进程，孤儿进程

僵尸进程，子进程执行结束，但是主进程还没执行结束，继续占用资源，python会自动回收系统资源，仅仅保留少量的数据用于父进程判断子进程的状态，比如pid,结束时间等。（其实这个过程，我认为不应该算是僵尸进程，因为占用的内存非常少）

孤儿进程，父进程结束，子进程没有结束，此时子进程会被系统init(进程号为1)进程回收。

互斥锁

# 该代码在macOS下运行失败，在CentOS7上运行成功
import time
from multiprocessing import Process, Lock

def child(l):
    print("child1 尝试获取锁")
    l.acquire()
    print("child1 获取锁")
    time.sleep(3)
    l.release()
    print("child1 释放锁")


def app(l):
    print("app 尝试获取锁")
    l.acquire()
    print("app 获取锁")
    time.sleep(3)
    l.release()
    print("app 释放锁")


if __name__ == '__main__':
    lock = Lock()
    p1 = Process(target=child, args=(lock,))
    p2 = Process(target=app, args=(lock,))

    p1.start()
    p2.start()

输出如下

child1 尝试获取锁

child1 获取锁

app 尝试获取锁  # 此时阻塞，一直到锁被释放，app进程才可以继续执行。

child1 释放锁

app 获取锁

app 释放锁

_thread多线程

import _thread
import time


def my_print(name):
    count = 0
    while True:
        time.sleep(2)
        count += 1
        print(f"{name} {count}")


_thread.start_new_thread(my_print, ("thread1",))
_thread.start_new_thread(my_print, ("thread2",))

time.sleep(50)