Pandasのメモリ不足を解消！数値型を最適化して処理を高速化する手法

import pandas as pd
import numpy as np

# サンプルデータの作成（100万行）
df = pd.DataFrame({
    'id': range(1000000),
    'score': np.random.randint(0, 100, 1000000),
    'ratio': np.random.rand(1000000)
})

# メモリ使用量の表示
df.info(memory_usage='deep')

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 1000000 entries, 0 to 999999
Data columns (total 3 columns):
 #   Column  Non-Null Count    Dtype  
---  ------  --------------    -----  
 0   id      1000000 non-null  int64  
 1   score   1000000 non-null  int64  
 2   ratio   1000000 non-null  float64
dtypes: float64(1), int64(2)
memory usage: 22.9 MB

# 整数列のダウンキャスト
df['id'] = pd.to_numeric(df['id'], downcast='integer')
df['score'] = pd.to_numeric(df['score'], downcast='integer')

# 浮動小数点列のダウンキャスト
df['ratio'] = pd.to_numeric(df['ratio'], downcast='float')

# 最適化後の情報を表示
df.info(memory_usage='deep')

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 1000000 entries, 0 to 999999
Data columns (total 3 columns):
 #   Column  Non-Null Count    Dtype  
---  ------  --------------    -----  
 0   id      1000000 non-null  int32  
 1   score   1000000 non-null  int8   
 2   ratio   1000000 non-null  float32
dtypes: float32(1), int32(1), int8(1)
memory usage: 8.6 MB

def reduce_mem_usage(df):
    start_mem = df.memory_usage().sum() / 1024**2
    
    for col in df.columns:
        col_type = df[col].dtype
        
        if col_type != object:
            c_min = df[col].min()
            c_max = df[col].max()
            
            if str(col_type)[:3] == 'int':
                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:
                    df[col] = df[col].astype(np.int8)
                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:
                    df[col] = df[col].astype(np.int16)
                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:
                    df[col] = df[col].astype(np.int32)
                else:
                    df[col] = df[col].astype(np.int64)  
            else:
                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:
                    df[col] = df[col].astype(np.float16)
                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:
                    df[col] = df[col].astype(np.float32)
                else:
                    df[col] = df[col].astype(np.float64)
                    
    end_mem = df.memory_usage().sum() / 1024**2
    print(f'メモリ消費量が {start_mem:.2f} MB から {end_mem:.2f} MB に減少しました。')
    return df

# 関数の適用
df_optimized = reduce_mem_usage(df)

# 読み込み時に型を指定
dtype_dict = {
    'id': 'int32',
    'score': 'int8',
    'ratio': 'float32'
}

df_fast = pd.read_csv('large_data.csv', dtype=dtype_dict)

# 欠損値を含む整数列の定義
s = pd.Series([1, 2, None], dtype="Int8")
print(s)

0       1
1       2
2    <NA>
dtype: Int8