Windowsの文字化け対策！cp932とencodingの使い分け

# 環境の文字エンコーディングを確認するサンプル
import locale
import sys
import os

print("locale.getpreferredencoding(False) =", locale.getpreferredencoding(False))
print("sys.getdefaultencoding()          =", sys.getdefaultencoding())
print("sys.stdout.encoding               =", sys.stdout.encoding)
print("PYTHONUTF8 env                    =", os.environ.get("PYTHONUTF8"))
print("UTF-8 mode                        =", sys.flags.utf8_mode)  # 1ならUTF-8モード

# UTF-8での基本的な読み書き
text = "Hello, こんにちは 👋"

# UTF-8で保存
with open("utf8_example.txt", "w", encoding="utf-8") as f:
    f.write(text)

# UTF-8で読み込み
with open("utf8_example.txt", "r", encoding="utf-8") as f:
    loaded = f.read()

print(loaded)

# cp932のファイルを作って、UTF-8読み込みで失敗→cp932で再読み込み
data = "売上合計,1000,円\n内訳,消耗品,500,円\n"

# cp932で保存(仮に社内システムが出力した想定)
with open("cp932_example.csv", "w", encoding="cp932") as f:
    f.write(data)

# 間違ってUTF-8で読むとエラーになることがある
try:
    with open("cp932_example.csv", "r", encoding="utf-8") as f:
        print(f.read())
except UnicodeDecodeError as e:
    print("UTF-8での読込に失敗:", e)

# cp932で再読み込み
with open("cp932_example.csv", "r", encoding="cp932") as f:
    print("cp932での再読込に成功:\n" + f.read())

# BOM付きUTF-8の違いを確認
s = "商品名,価格\nりんご,120\n"

# BOM付きで保存
with open("bom_utf8.csv", "w", encoding="utf-8-sig") as f:
    f.write(s)

# 通常のutf-8で読むとBOM(\ufeff)が先頭に残る場合がある
with open("bom_utf8.csv", "r", encoding="utf-8") as f:
    head = f.read(1)
    print("先頭1文字(utf-8):", repr(head))  # 先頭に'\ufeff'が見える

# utf-8-sigならBOMをスキップして読み取る
with open("bom_utf8.csv", "r", encoding="utf-8-sig") as f:
    head = f.read(1)
    print("先頭1文字(utf-8-sig):", repr(head))

# cp932で書けない文字(例: 絵文字)を含むケース
text = "顧客名: 山田太郎 🍣"

try:
    with open("out_cp932.txt", "w", encoding="cp932") as f:
        f.write(text)
except UnicodeEncodeError as e:
    print("cp932への保存に失敗:", e)

# 回避策1: UTF-8で保存(推奨)
with open("out_utf8.txt", "w", encoding="utf-8") as f:
    f.write(text)
print("UTF-8での保存は成功")

# 回避策2: 最終手段としてerrors='replace'で代替文字に置換
with open("out_cp932_replace.txt", "w", encoding="cp932", errors="replace") as f:
    f.write(text)
print("cp932(errors='replace')での保存は成功(文字は'?'等に置換)")

# cp932のファイルをあえてUTF-8で開いて失敗を観察
with open("cp932_example.csv", "r", encoding="utf-8") as f:
    print(f.read())

# 簡易な自動判定: UTF-8 → cp932 → UTF-8(BOM付き)の順に試す
from typing import Tuple

CANDIDATES = ("utf-8", "cp932", "utf-8-sig")

def read_text_with_guess(path: str) -> Tuple[str, str]:
    """
    ファイルを複数エンコーディングで試し読みして、成功したテキストと使用エンコーディングを返す。
    優先順位: utf-8 → cp932 → utf-8-sig
    """
    last_error = None
    for enc in CANDIDATES:
        try:
            with open(path, "r", encoding=enc) as f:
                return f.read(), enc
        except UnicodeDecodeError as e:
            last_error = e
    # すべて失敗した場合は例外を投げ直す
    raise UnicodeDecodeError(last_error.encoding, last_error.object, last_error.start, last_error.end, last_error.reason)

# 使い方例
content, used_enc = read_text_with_guess("unknown_file.txt")
print("判定エンコーディング:", used_enc)
print(content[:80], "...")

# 壊れたバイト列をcp932として読み、置換/無視の違いを確認
bad_bytes = b"\x82\xa0\xff\x82\xa2"  # cp932に不正なバイト(0xFF)を混入

# replace: 不正箇所をU+FFFD( )に置換して返す
print(bad_bytes.decode("cp932", errors="replace"))

# ignore: 不正箇所を丸ごと無視
print(bad_bytes.decode("cp932", errors="ignore"))

# 簡易BOMチェックと、先頭行の可視化
def probe(path: str) -> None:
    with open(path, "rb") as fb:
        head = fb.read(4)  # BOM判定に十分
    if head.startswith(b"\xef\xbb\xbf"):
        print("BOM: UTF-8-SIG っぽい")
    elif head.startswith(b"\xff\xfe"):
        print("BOM: UTF-16-LE っぽい")
    elif head.startswith(b"\xfe\xff"):
        print("BOM: UTF-16-BE っぽい")
    else:
        print("BOM: なし(UTF-8かcp932等の可能性)")

    # テキストとして読み取って先頭行の見た目を確認(失敗は握りつぶして可視化)
    for enc in ("utf-8", "cp932", "utf-8-sig"):
        try:
            with open(path, "r", encoding=enc) as f:
                first = f.readline().rstrip("\n")
            print(f"[{enc}] 先頭行: {first!r}")
        except UnicodeDecodeError:
            print(f"[{enc}] 読み取り失敗")

probe("unknown_file.txt")