MySQL生成数据清洗SQL脚本

143 阅读1分钟

数据清洗SQL脚本 由于数据库导入数据存在null字符串的脏数据,通过查询MySQL的information_schema表结构,来生成清洗SQL

全自动生成

select CONCAT(' UPDATE ',TABLE_NAME, ' SET ' , GROUP_CONCAT(temp.ql) ) from (
SELECT
	TABLE_SCHEMA,
	TABLE_NAME,
	COLUMN_NAME,
	COLUMN_TYPE,
	-- , asset_name = IF(asset_name="null",null,asset_name)
	CONCAT( COLUMN_NAME,' = IF(',COLUMN_NAME,'="null",null,',COLUMN_NAME,')' ) AS 'ql'
FROM
	`COLUMNS` 
WHERE
	`TABLE_SCHEMA` IN ( "studio" )
	AND TABLE_NAME = 'data_assets_dir_info_struct_queue'
	AND (COLUMN_TYPE LIKE 'varchar%' OR COLUMN_TYPE = 'text') )temp

半自动生成

SELECT
	TABLE_SCHEMA,
	TABLE_NAME,
	COLUMN_NAME,
	COLUMN_TYPE,
	CONCAT(', ',COLUMN_NAME,' = IF(',COLUMN_NAME,'="null",null,',COLUMN_NAME,')' )
FROM
	`COLUMNS` 
WHERE
	`TABLE_SCHEMA` IN ( "your_db_name" )
	AND TABLE_NAME = 'your_table_name'
	AND (COLUMN_TYPE LIKE 'varchar%' OR COLUMN_TYPE = 'text')
	

-- 组装SQL
update your_table_name set
 from_info = IF(from_info="null",null,from_info)
, level_rule = IF(level_rule="null",null,level_rule)
, class_rule = IF(class_rule="null",null,class_rule)

image.png

image.png