kafak怎么快速批量造一百万数据_怎么向kafka中造数据，大数据开发高级工程师进阶学习

用户33914928060
41 阅读2分钟

img img

网上学习资料一大堆，但如果学到的知识不成体系，遇到问题时只是浅尝辄止，不再深入研究，那么很难做到真正的技术提升。

需要这份系统化资料的朋友，可以戳这里获取

一个人可以走的很快，但一群人才能走的更远！不论你是正从事IT行业的老鸟或是对IT行业感兴趣的新人，都欢迎加入我们的的圈子（技术交流、学习资源、职场吐槽、大厂内推、面试辅导），让我们一起学习成长！

{ "version": { "version": "1.0.0", "versionParts": [ 1 ] }, "msg_compression_kind": "1", "msg_split_idx": 1, "msg_source_ip": "10.82.13.74", "msg_created_by": "DELL", "msg_creation_time": 1711525684104, "message": { "type": "ENTITY_CREATE_V2", "user": "DELL", "entities": { "entities": [ { "typeName": "ceshifuzadu2", "attributes": { "qualifiedName": "1711525684104", "tenantId": "tenantId12", "name": "百万消息测试复杂度2024-03-27 07:48:04.104966+00", "userId": "userId12", "version": 1 }, "guid": "-1711525684104", "isIncomplete": false, "provenanceType": 0, "version": 0, "proxy": false } ] } } }


1. 我在pg数据库当中建立对应的表

create table test_b ( version json, msg_compression_kind varchar, msg_split_idx integer, msg_source_ip varchar, msg_created_by varchar, msg_creation_time bigint, message json );


2. 创建造数脚本 造 100万条

DO DECLAREkinteger:=1;DECLAREmessagejson:=′′;beginWHILEk<=1000000loopWITHdynamicjsonAS(SELECTjsonbuildobject(′type′,′ENTITY_CREATE_V2′,′user′,′DELL′,′entities′,jsonbuildobject(′entities′,jsonbuildarray(jsonbuildobject(′typeName′,′ceshifuzadu2′,′attributes′,jsonbuildobject(′qualifiedName′,′′∣∣FLOOR(EXTRACT(EPOCHFROMNOW())\*1000)+k,′tenantId′,′tenantId12′,′name′,′百万消息测试复杂度′∣∣FLOOR(EXTRACT(EPOCHFROMNOW())\*1000)+k,′userId′,′userId12′,′version′,1),′guid′,′−′∣∣FLOOR(EXTRACT(EPOCHFROMNOW())\*1000)+k,′isIncomplete′,false,′provenanceType′,0,′version′,0,′proxy′,false))))ASmessageb)SELECTmessagebINTOmessageFROMdynamicjson;INSERTINTOpublic.testb("version","msg_compression_kind","msg_split_idx","msg_source_ip","msg_created_by","msg_creation_time","message")VALUES(′"version":"1.0.0","versionParts":[1]′,1,1,′10.82.13.74′,′DELL′,FLOOR(EXTRACT(EPOCHFROMNOW())\*1000),message);k=k+1;endloop;END DECLARE k integer := 1; DECLARE message json :='{}'; begin WHILE k <= 1000000 loop WITH dynamic_json AS ( SELECT json_build_object( 'type', 'ENTITY\_CREATE\_V2', 'user', 'DELL', 'entities', json_build_object( 'entities', json_build_array( json_build_object( 'typeName', 'ceshifuzadu2', 'attributes', json_build_object( 'qualifiedName', ''||FLOOR(EXTRACT(EPOCH FROM NOW()) \* 1000)+k, 'tenantId', 'tenantId12', 'name', '百万消息测试复杂度' || FLOOR(EXTRACT(EPOCH FROM NOW()) \* 1000)+k, 'userId', 'userId12', 'version', 1 ), 'guid', '-' || FLOOR(EXTRACT(EPOCH FROM NOW()) \* 1000)+k, 'isIncomplete', false, 'provenanceType', 0, 'version', 0, 'proxy', false ) ) ) ) AS messageb ) SELECT messageb INTO message FROM dynamic_json; INSERT INTO public.test_b ("version", "msg\_compression\_kind", "msg\_split\_idx", "msg\_source\_ip", "msg\_created\_by", "msg\_creation\_time", "message") VALUES('{"version":"1.0.0","versionParts":[1]}', 1, 1, '10.82.13.74','DELL',FLOOR(EXTRACT(EPOCH FROM NOW()) \* 1000), message); k = k + 1; end loop; END;


3. 使用psql 将数据导出


![img](https://p3-xtjj-sign.byteimg.com/tos-cn-i-73owjymdk6/3ba7f475d6ec43e8b9ebfe7078f67a86~tplv-73owjymdk6-jj-mark-v1:0:0:0:0:5o6Y6YeR5oqA5pyv56S-5Yy6IEAg55So5oi3MzM5MTQ5MjgwNjA=:q75.awebp?rk3s=f64ab15b&x-expires=1773130094&x-signature=UDlXisuLXmyCFK1rYTMYdpv%2B13g%3D)
![img](https://p3-xtjj-sign.byteimg.com/tos-cn-i-73owjymdk6/1146eaabb7374666a2891f65764df872~tplv-73owjymdk6-jj-mark-v1:0:0:0:0:5o6Y6YeR5oqA5pyv56S-5Yy6IEAg55So5oi3MzM5MTQ5MjgwNjA=:q75.awebp?rk3s=f64ab15b&x-expires=1773130094&x-signature=ngn4DbF8G1cn2QvVy1JwxkYp9Wg%3D)

**网上学习资料一大堆，但如果学到的知识不成体系，遇到问题时只是浅尝辄止，不再深入研究，那么很难做到真正的技术提升。**

**[需要这份系统化资料的朋友，可以戳这里获取](https://gitee.com/vip204888)**


**一个人可以走的很快，但一群人才能走的更远！不论你是正从事IT行业的老鸟或是对IT行业感兴趣的新人，都欢迎加入我们的的圈子（技术交流、学习资源、职场吐槽、大厂内推、面试辅导），让我们一起学习成长！**
avatar
文章
阅读
粉丝