機械学習でツイートが炎上するか予測してみた

import pandas as pd
import csv
import glob

# csvディレクトリのパスを取得
DATA_PATH = "/csv_directory"
All_Files = glob.glob('{}tweet*'.format(DATA_PATH))

# ディレクトリ内の全csvをマージ
tweets = []
for file in All_Files:
    tweets.append(pd.read_csv(file,engine='python'))
tweet_df = pd.concat(tweets, sort=False)

tweet_df.dropna()

import emoji
import nagisa
import unicodedata
import re

# 絵文字削除用関数
def remove_emoji(src_str):
    return ''.join(c for c in src_str if c not in emoji.UNICODE_EMOJI)

# 顔文字削除関数
def remove_kaomoji(text):
    KAOMOJI_LEN = 5
    results = nagisa.extract(text, extract_postags=['補助記号'])
    words = results.words
    kaomoji_words = []
    kaomoji_idx = [i for i, w in enumerate(words) if len(w) >= KAOMOJI_LEN]
    kaomoji_hands = ['ノ', 'ヽ', '∑', 'm', 'O', 'o', '┐', '/', '\\', '┌'] 
    # 顔文字と手を検索
    for i in kaomoji_idx:
        kaomoji = words[i]
        try:
            # 顔文字の左手
            if words[i-1] in kaomoji_hands and 0 < i:
                kaomoji = words[i-1] + kaomoji
            # 顔文字の右手
            if words[i+1] in kaomoji_hands:
                 kaomoji = kaomoji + words[i+1]
        except IndexError:
            pass
        finally:
            kaomoji_words.append(kaomoji)
    
    # 抽出した顔文字を削除
    for j in kaomoji_words:
        text = text.replace(j, '')

    return text

# 整形したテキストをDataFrameに追加
tweet_text = []
for text in tweet_df["text"]:
    text = remove_emoji(text) #絵文字削除
    text = remove_kaomoji(text) #顔文字削除
    text = re.sub(r'@[0-9a-zA-Z_:]*', "", text) #@ユーザー名削除
    tweet_text.append(text)
tweet_df["text_modified"] = tweet_text

reply_groupby_df = reply_df.groupby('to_id').sum()

tweet_reply_df = pd.merge(tweet_df, reply_groupby_df, left_on='id',right_on='to_id', how='inner')

tweet_reply_df["angry_mmn"] = (tweet_reply_df["angry"]-tweet_reply_df["angry"].min()) / (tweet_reply_df["angry"].max()-tweet_reply_df["angry"].min())
tweet_reply_df["disgust_mmn"] = (tweet_reply_df["disgust"]-tweet_reply_df["disgust"].min()) / (tweet_reply_df["disgust"].max()-tweet_reply_df["disgust"].min())

from sklearn.cluster import KMeans
distortions = []

for i  in range(1,11): # 1~10クラスタまで計算 
    km = KMeans(n_clusters=i,
                init='k-means++',
                n_init=10,
                max_iter=300,
                random_state=0)
    km.fit(tweet_df[['disgust_mmn', 'angry_mmn']])
    distortions.append(km.inertia_)

plt.plot(range(1,11),distortions,marker='o')
plt.xlabel('Number of clusters')
plt.ylabel('Distortion')
plt.show()

# k-meansでdisgust,angryをもとに分類
kmeans_model = KMeans(n_clusters=5, random_state=0).fit(tweet_df[['disgust_mmn', 'angry_mmn']])
# 分類結果のラベルをDataFrameに追加
tweet_df["label"] = kmeans_model.labels_

BSV = BertSequenceVectorizer()
tweet_df['text_feature'] = tweet_df['text_modified'].progress_apply(lambda x: BSV.vectorize(x))

from sklearn.model_selection import train_test_split
feature_train, feature_test,label_train, label_test, weight_train, weight_test = train_test_split(tweet_df["text_feature"],tweet_df["label"], weight_list, test_size=0.1, shuffle=True)

import xgboost as xgb
from sklearn.model_selection import GridSearchCV
from sklearn.datasets import load_digits
# モデル定義
xgb_mc = xgb.XGBClassifier()
# ハイパーパラメータ探索
xgb_model_cv = GridSearchCV(xgb_mc, {'eta': [0.01,0.1,0.3], 'max_depth': [5,6,7,8,9,10], 'gamma': [0.01,0.1,1.0,10.0]}, verbose=1, n_jobs=-1, cv=5)
# 学習
xgb_model_cv.fit(feature_train, label_train, sample_weight=weight_train)
print(xgb_model_cv.best_params_, xgb_model_cv.best_score_)

from sklearn.metrics import confusion_matrix, classification_report

pred = xgb_model_cv.predict(feature_test)
print(classification_report(label_test, pred))

result = sum(1*(p == t or p == t+1 or p==t-1) for p, t in zip(pred.tolist(), label_test)) / len(label_test)
print("Accuracy: {:.2f}".format(result))