影刀RPA 文件编码问题：UTF-8、GBK、BOM
署名：林焱
什么情况用什么
RPA流程中读写文件时最头疼的就是编码问题——采集的网页是UTF-8，保存到CSV后Excel打开乱码；读取同事用Windows记事本保存的文件报UnicodeDecodeError；处理旧系统导出的GBK编码数据。
| 编码 | 使用场景 | 特点 |
|---|---|---|
| UTF-8 | 通用、网页、API | 国际标准，变长编码 |
| UTF-8 with BOM | Windows兼容 | 带BOM头，Excel可正确识别 |
| GBK/GB2312 | Windows中文系统 | 中文专用，不兼容国际 |
| latin-1 | 兜底 | 不会报错但可能乱码 |
怎么做
一：读写文件指定编码
# 写入文件
with open(r"D:\data\output.txt", "w", encoding="utf-8") as f:
f.write("中文内容")
# 读取文件
with open(r"D:\data\input.txt", "r", encoding="utf-8") as f:
content = f.read()
# CSV文件用utf-8-sig（带BOM，Excel兼容）
import csv
with open(r"D:\data\data.csv", "w", encoding="utf-8-sig", newline="") as f:
writer = csv.writer(f)
writer.writerow(["姓名", "年龄"])
二：自动检测文件编码
[video(video-Tv68IEDv-1784312237807)(type-csdn)(url-live.csdn.net/v/embed/526…)]
def detect_encoding(file_path, sample_size=1024):
"""尝试检测文件编码"""
encodings = ["utf-8-sig", "utf-8", "gbk", "gb2312", "gb18030", "big5", "latin-1"]
with open(file_path, "rb") as f:
raw = f.read(sample_size)
for enc in encodings:
try:
raw.decode(enc)
return enc
except (UnicodeDecodeError, LookupError):
continue
return "utf-8" # 默认回退
# 使用
encoding = detect_encoding(r"D:\data\unknown.txt")
print(f"检测到编码: {encoding}")
with open(r"D:\data\unknown.txt", "r", encoding=encoding) as f:
content = f.read()
三：编码转换
def convert_encoding(input_path, output_path, from_enc=None, to_enc="utf-8"):
"""转换文件编码"""
if from_enc is None:
from_enc = detect_encoding(input_path)
with open(input_path, "r", encoding=from_enc) as f:
content = f.read()
with open(output_path, "w", encoding=to_enc) as f:
f.write(content)
print(f"转换完成: {from_enc} → {to_enc}")
# GBK转UTF-8
convert_encoding(
r"D:\data\gbk文件.txt",
r"D:\data\utf8文件.txt",
from_enc="gbk",
to_enc="utf-8"
)
四：处理BOM头
# 检查文件是否有BOM
def has_bom(file_path):
with open(file_path, "rb") as f:
return f.read(3) == b'\xef\xbb\xbf'
# 去掉BOM头
def remove_bom(file_path):
with open(file_path, "rb") as f:
content = f.read()
if content[:3] == b'\xef\xbb\xbf':
content = content[3:]
with open(file_path, "wb") as f:
f.write(content)
# 读取时自动处理BOM
# 用utf-8-sig读取会自动去掉BOM
with open("file.txt", "r", encoding="utf-8-sig") as f:
content = f.read()
# content不会有BOM字符
完整流程：编码兼容读写工具
import os
class EncodingSafeFile:
"""编码安全的文件读写工具"""
# 常见编码优先级
ENCODINGS = ["utf-8-sig", "utf-8", "gbk", "gb18030", "latin-1"]
@classmethod
def read(cls, file_path, encoding=None):
"""安全读取文件，自动检测编码"""
if encoding:
with open(file_path, "r", encoding=encoding) as f:
return f.read()
# 自动检测
for enc in cls.ENCODINGS:
try:
with open(file_path, "r", encoding=enc) as f:
return f.read()
except UnicodeDecodeError:
continue
# 兜底：用latin-1（不会报错，但中文会乱码）
with open(file_path, "r", encoding="latin-1") as f:
return f.read()
@classmethod
def write(cls, file_path, content, encoding="utf-8-sig"):
"""安全写入文件"""
os.makedirs(os.path.dirname(file_path) if os.path.dirname(file_path) else ".", exist_ok=True)
with open(file_path, "w", encoding=encoding) as f:
f.write(content)
@classmethod
def read_lines(cls, file_path, encoding=None):
"""逐行读取"""
if encoding:
enc = encoding
else:
# 先检测编码
for e in cls.ENCODINGS:
try:
with open(file_path, "r", encoding=e) as f:
f.read(100)
enc = e
break
except UnicodeDecodeError:
continue
else:
enc = "latin-1"
with open(file_path, "r", encoding=enc) as f:
for line in f:
yield line.rstrip("\n\r")
# 使用
content = EncodingSafeFile.read(r"D:\data\未知编码.txt")
print(f"读取成功: {len(content)}字符")
EncodingSafeFile.write(r"D:\data\输出.txt", "测试中文内容")
有什么坑
坑一：UTF-8文件在Excel中乱码
现象：Python用UTF-8编码写的CSV/TXT文件，用Excel打开后中文全是乱码。
原因：Excel在Windows中文系统下默认用GBK编码打开文本文件，UTF-8文件没有BOM头时Excel无法识别。
解决：写入时用utf-8-sig（带BOM的UTF-8）：
# CSV文件
import csv
with open("data.csv", "w", encoding="utf-8-sig", newline="") as f:
writer = csv.writer(f)
writer.writerow(["姓名", "年龄", "部门"])
# 普通文本
with open("data.txt", "w", encoding="utf-8-sig") as f:
f.write("中文内容")
# UTF-8-sig和UTF-8的区别：
# UTF-8: 不带BOM头，文件开头直接是内容
# UTF-8-sig: 带BOM头（3字节：EF BB BF），Excel能正确识别编码
坑二：读取GBK文件报UnicodeDecodeError
现象：读取Windows记事本保存的文件，报UnicodeDecodeError: 'utf-8' codec can't decode byte...。
原因：Windows记事本默认用ANSI编码（中文系统下是GBK），用UTF-8读取会失败。
解决：指定GBK编码，或自动检测：
[video(video-tywgmDBh-1784312244497)(type-csdn)(url-live.csdn.net/v/embed/526…)]
# 方案1：直接用GBK
with open("file.txt", "r", encoding="gbk") as f:
content = f.read()
# 方案2：用errors参数忽略错误（可能丢失字符）
with open("file.txt", "r", encoding="utf-8", errors="ignore") as f:
content = f.read()
# 方案3：用errors="replace"替换无法解码的字符
with open("file.txt", "r", encoding="utf-8", errors="replace") as f:
content = f.read()
# 方案4：自动检测（推荐）
def safe_read(file_path):
for enc in ["utf-8-sig", "utf-8", "gbk", "gb18030", "latin-1"]:
try:
with open(file_path, "r", encoding=enc) as f:
return f.read()
except UnicodeDecodeError:
continue
return ""
content = safe_read("file.txt")
坑三：BOM头导致字符串开头多出不可见字符
现象：读取UTF-8 with BOM文件后，字符串开头多了\ufeff字符，导致字符串比较失败。
原因：utf-8编码读取带BOM的文件时，BOM不会被自动去掉，变成\ufeff字符。
解决：用utf-8-sig编码读取，或手动去掉：
# 正确：用utf-8-sig读取（自动去BOM）
with open("file.txt", "r", encoding="utf-8-sig") as f:
content = f.read()
# content开头没有\ufeff
# 手动去掉
content = content.lstrip("\ufeff")
# 检查是否有BOM字符
if content.startswith("\ufeff"):
content = content[1:]
# 常见错误：用utf-8读取BOM文件
# with open("file.txt", "r", encoding="utf-8") as f: ❌ content开头有\ufeff
# content = f.read()
# if content == "hello": ❌ 永远不等，因为实际是"\ufeffhello"
坑四：混合编码文件读取失败
现象：一个文件大部分是UTF-8编码，但中间有少部分GBK字符，导致读取中途报错。
原因：文件编码不统一，可能是从不同来源拼接的。
解决：用errors参数或逐段处理：
# 方案1：忽略错误字符
with open("file.txt", "r", encoding="utf-8", errors="ignore") as f:
content = f.read()
# 丢弃无法解码的字节
# 方案2：替换错误字符
with open("file.txt", "r", encoding="utf-8", errors="replace") as f:
content = f.read()
# 无法解码的字节替换为�
# 方案3：逐字节处理，混合编码解码
def read_mixed_encoding(file_path):
result = []
with open(file_path, "rb") as f:
data = f.read()
i = 0
while i < len(data):
# 尝试UTF-8解码
try:
char = data[i:i+4].decode("utf-8")
result.append(char)
i += len(char.encode("utf-8"))
continue
except UnicodeDecodeError:
pass
# 回退到GBK
try:
char = data[i:i+2].decode("gbk")
result.append(char)
i += 2
continue
except UnicodeDecodeError:
pass
# 跳过无法解码的字节
i += 1
return "".join(result)
