PythonのUnicodeEncodeError/UnicodeDecodeError 原因と対処法まとめ

# 正しい組み合わせ: CP932でエンコードしたものをCP932でデコード
b_cp932 = "あ".encode("cp932")
print(b_cp932)  # b'\x82\xa0'
print(b_cp932.decode("cp932"))  # "あ"

# 間違った組み合わせ: CP932のバイト列をUTF-8としてデコード -> 失敗
try:
    print(b_cp932.decode("utf-8"))
except UnicodeDecodeError as e:
    print("Decode error:", e)

text = "日本"
print(type(text), len(text))  # Unicodeとして2文字

b_utf8 = text.encode("utf-8")
b_cp932 = text.encode("cp932")
print(type(b_utf8), len(b_utf8), b_utf8)   # UTF-8は可変長 -> 6バイト
print(type(b_cp932), len(b_cp932), b_cp932)  # CP932では4バイト

# 例: 実際はCP932で保存されたファイルをUTF-8として読む
try:
    with open("data_cp932.txt", "r", encoding="utf-8") as f:
        print(f.read())
except UnicodeDecodeError as e:
    print("Decode error:", e)

# 正しくCP932で読む
with open("data_cp932.txt", "r", encoding="cp932") as f:
    content = f.read()
    print("OK:", content[:20])

# どうしても不明なら一時回避
with open("unknown.txt", "r", encoding="utf-8", errors="replace") as f:
    content = f.read()
    print("REPLACED:", content[:50])

text = "寿司🍣"  # 絵文字を含む
try:
    with open("out_cp932.txt", "w", encoding="cp932") as f:
        f.write(text)  # CP932では🍣が表現できない
except UnicodeEncodeError as e:
    print("Encode error:", e)

# 推奨: UTF-8で書く
with open("out_utf8.txt", "w", encoding="utf-8") as f:
    f.write("寿司🍣")  # OK

# やむを得ずCP932のとき
with open("out_cp932_safe.txt", "w", encoding="cp932", errors="replace") as f:
    f.write("寿司🍣")  # 🍣は '?' に置換される

import sys, locale
print("stdout encoding:", sys.stdout.encoding)
print("preferred encoding:", locale.getpreferredencoding(False))
print("テスト: こんにちは🌏")

import json

# JSONはUTF-8で読むのが原則
with open("data.json", "r", encoding="utf-8") as f:
    data = json.load(f)
print(type(data))

obj = {"title": "こんにちは世界", "emoji": "🌏"}
with open("out.json", "w", encoding="utf-8") as f:
    json.dump(obj, f, ensure_ascii=False)

import csv

# Excel想定ならCP932で出力する場合がある
rows = [["商品名", "価格"], ["寿司🍣", 1200]]
with open("out_excel.csv", "w", newline="", encoding="cp932", errors="replace") as f:
    writer = csv.writer(f)
    writer.writerows(rows)

import requests

resp = requests.get("https://example.com/data.csv", timeout=10)
# サーバが宣言したエンコードを使う。なければ推定値を使う
resp.encoding = resp.encoding or resp.apparent_encoding
text = resp.text  # 自動デコード済みのstr
print(text[:200])

b = "あ".encode("cp932")  # b'\x82\xa0'
print(b.decode("utf-8"))  # 間違い

"🍣".encode("cp932")

try:
    b = b"\x82\xa0ABC"  # CP932の「あ」+ ASCII
    b.decode("utf-8")   # 間違い
except UnicodeDecodeError as e:
    print("encoding:", e.encoding)
    print("reason:", e.reason)
    print("start-end:", e.start, e.end)
    print("problem bytes:", e.object[e.start:e.end])

def hexdump(bs, around=8):
    # bytesを16進表記で可視化するヘルパ
    return " ".join(f"{b:02X}" for b in bs[:around])

try:
    b = "価格：1200円".encode("cp932")  # CP932で生成
    b_bad = b.replace(b"\x46", b"\x20")  # 壊れたバイトを混入
    b_bad.decode("cp932")  # デコード失敗
except UnicodeDecodeError as e:
    frag = e.object[max(0, e.start-4):e.end+4]
    print("pos:", e.start, "hex around:", hexdump(frag))

# 読み書きともにUTF-8を明示
with open("input.txt", "r", encoding="utf-8") as f:
    data = f.read()

with open("output.txt", "w", encoding="utf-8") as f:
    f.write(data)

# 置換して読む
with open("unknown.bin", "r", encoding="utf-8", errors="replace") as f:
    preview = f.read(200)
print(preview)

# 無視して読む
with open("unknown.bin", "r", encoding="utf-8", errors="ignore") as f:
    preview2 = f.read(200)
print(preview2)

s = "こんにちは"
b = s.encode("utf-8")     # OK
s2 = b.decode("utf-8")    # OK

try:
    s.decode("utf-8")     # NG: strにdecodeはない
except AttributeError as e:
    print("AttributeError:", e)

try:
    b.encode("utf-8")     # NG: bytesにencodeはない
except AttributeError as e:
    print("AttributeError:", e)

import sys
print("sys.stdout.encoding:", sys.stdout.encoding)
print("こんにちは🌏")

REM cmd.exeの例
chcp 65001
set PYTHONUTF8=1
python script.py

# pandas
import pandas as pd
df = pd.read_csv("in.csv", encoding="cp932")  # Excel由来ならcp932が多い
df.to_csv("out.csv", index=False, encoding="utf-8")

# json
import json
json.loads('{"title":"こんにちは"}')  # 文字列の場合は既にUnicode
with open("in.json", encoding="utf-8") as f:
    obj = json.load(f)

# requests
import requests
r = requests.get("https://example.com/page", timeout=10)
r.encoding = r.encoding or r.apparent_encoding
html = r.text  # デコード済みのstr

import logging

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s %(levelname)s %(message)s"
)
logger = logging.getLogger(__name__)

def read_text(path, encoding="utf-8"):
    try:
        with open(path, "r", encoding=encoding) as f:
            return f.read()
    except UnicodeError as e:
        # 詳細を拾って調査に活かす
        if hasattr(e, "start"):
            logger.error("UnicodeError at %s [%s:%s] %s",
                         path, getattr(e, "start", "?"),
                         getattr(e, "end", "?"),
                         getattr(e, "reason", str(e)))
        logger.exception("Stack:")
        # 状況に応じて再スローまたは代替処理
        return None

text = read_text("mystery.txt", encoding="utf-8")

def strict_read(path, encoding="utf-8"):
    try:
        with open(path, "r", encoding=encoding) as f:
            return f.read()
    except UnicodeDecodeError as e:
        # 独自の例外に包んで再スロー
        raise RuntimeError(f"Failed to decode {path} with {encoding}") from e

import csv, json

# CSVをUTF-8で
with open("data.csv", "w", newline="", encoding="utf-8") as f:
    writer = csv.writer(f)
    writer.writerow(["name", "note"])
    writer.writerow(["寿司", "海鮮"])

# JSONをUTF-8で、非ASCIIをそのまま
obj = {"title": "寿司", "emoji": "🍣"}
with open("data.json", "w", encoding="utf-8") as f:
    json.dump(obj, f, ensure_ascii=False)