Kafka生产和消费

生产者

from kafka import KafkaProducer
import config_k
import json

# 使用gzip压缩
producer = KafkaProducer(bootstrap_servers=config_k.SERVER, compression_type='gzip')
topic = config_k.TOPIC
topic2 = config_k.TOPIC2

def kafka(content):
    try:
        # 数据使用json格式，保留中文
        f = producer.send(topic, value=json.dumps(content, ensure_ascii=False).encode('utf-8'))
        f.get(timeout=1000)
        print('kafka写入成功：{}'.format(content['title']))

        images = {}
        images['images'] = img = content['images']
        images['url'] = content['url']
        images['gather_time'] = content['gather_time']
        if len(img) > 0:
            f2 = producer.send(topic2, value=json.dumps(images, ensure_ascii=False).encode('utf-8'))
            f2.get(timeout=100)
    except Exception as e:
        print('写入失败，数据回滚：{}'.format(e))

if __name__ == '__main__':
    kafka()

消费者

# -*-coding:utf-8-*-
from kafka import KafkaConsumer
import json
import config_k

consumer = KafkaConsumer(
    # SERVER = '192.168.1.1:9092, 192.168.1.2:9092, 192.168.1.3:9092'
    bootstrap_servers=config_k.SERVER,
    # 随机值
    group_id="1231",
    enable_auto_commit=True,
    auto_commit_interval_ms=500,
)

consumer.subscribe(["news_v1"])
for msg in consumer:
    data = (msg.value.decode("utf-8"))

    data = json.loads(data)
    print(data)