tldr

KggleのTop Personality DatasetをPredicting Movie Preferences - Data Every Day #049に沿ってやっていきます。

実行環境はGoogle Colaboratorです。

インポート

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

import sklearn.preprocessing as sp
from sklearn.model_selection import train_test_split
import sklearn.linear_model as slm
import sklearn.svm as svm
import sklearn.neural_network as snn

import tensorflow as tf

データのダウンロード

Google Driveをマウントします。

from google.colab import drive
drive.mount('/content/drive')

Mounted at /content/drive

KaggleのAPIクライアントを初期化し、認証します。
認証情報はGoogle Drive内（/content/drive/My Drive/Colab Notebooks/Kaggle）にkaggle.jsonとして置いてあります。

import os
kaggle_path = "/content/drive/My Drive/Colab Notebooks/Kaggle"
os.environ['KAGGLE_CONFIG_DIR'] = kaggle_path

from kaggle.api.kaggle_api_extended import KaggleApi
api = KaggleApi()
api.authenticate()

Kaggle APIを使ってデータをダウンロードします。

dataset_id = 'arslanali4343/top-personality-dataset'
dataset = api.dataset_list_files(dataset_id)
file_name = dataset.files[0].name
file_path = os.path.join(api.get_default_download_dir(), file_name)
file_path

'/content/2018-personality-data.csv'

api.dataset_download_file(dataset_id, file_name, force=True, quiet=False)

100%|██████████| 608k/608k [00:00<00:00, 74.4MB/s]

Downloading 2018-personality-data.csv to /content









True


api.dataset_download_file(dataset_id, file_name, force=True, quiet=False)

データの読み込み

data = pd.read_csv(file_path)

data

	userid	openness	agreeableness	emotional_stability	conscientiousness	extraversion	assigned metric	assigned condition	movie_1	predicted_rating_1	movie_2	predicted_rating_2	movie_3	predicted_rating_3	movie_4	predicted_rating_4	movie_5	predicted_rating_5	movie_6	predicted_rating_6	movie_7	predicted_rating_7	movie_8	predicted_rating_8	movie_9	predicted_rating_9	movie_10	predicted_rating_10	movie_11	predicted_rating_11	movie_12	predicted_rating_12	is_personalized	enjoy_watching
0	8e7cebf9a234c064b75016249f2ac65e	5.0	2.0	3.0	2.5	6.5	serendipity	high	77658	4.410466	95858	4.271995	115713	4.611922	26674	4.459407	93040	4.147292	117533	4.098206	108979	4.064843	112582	4.149100	120138	4.244817	121372	4.396004	127152	4.120456	95311	4.053847	4	4
1	77c7d756a093150d4377720abeaeef76	7.0	4.0	6.0	5.5	4.0	all	default	94959	4.207280	1247	4.266540	953	4.211322	2010	4.408341	1234	4.090358	5291	4.202424	106642	4.113912	1209	4.094422	56782	4.019599	5618	3.963953	969	4.174188	1232	4.334877	2	3
2	b7e8a92987a530cc368719a0e60e26a3	4.0	3.0	4.5	2.0	2.5	serendipity	medium	110501	4.868064	77658	4.710444	101895	5.029360	1260	4.698602	5971	4.660769	98491	4.962319	926	4.706864	1204	4.645191	2288	4.823212	3307	4.676756	1172	4.649281	1212	4.744990	2	2
3	92561f21446e017dd6b68b94b23ad5b7	5.5	5.5	4.0	4.5	4.0	popularity	medium	2905	4.526371	2843	4.456451	3629	4.668444	3022	4.676067	3307	4.530360	1211	4.292660	3462	4.341634	5291	4.261166	3030	4.425689	1281	4.479921	940	4.355061	905	4.317927	3	3
4	030001ac2145a938b07e686a35a2d638	5.5	5.5	3.5	4.5	2.5	popularity	medium	2905	4.526371	2843	4.456451	3629	4.668444	3022	4.676067	3307	4.530360	1211	4.292660	3462	4.341634	5291	4.261166	3030	4.425689	1281	4.479921	940	4.355061	905	4.317927	2	3
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
1829	cff910b71f09b3120289ff6b461a9e03	5.5	3.5	2.5	4.0	5.5	popularity	low	108979	4.246346	33779	4.002232	96728	3.981009	77307	4.059373	111759	3.888507	7762	4.368495	26082	4.554531	89753	4.145526	6643	4.409108	115122	3.960470	7700	4.178546	67997	4.085300	3	3
1830	1ab3a4c2921d8da640854819b0f6cfce	4.0	3.5	4.5	4.0	2.5	serendipity	high	93040	4.227140	1199	4.069527	83134	4.336292	5971	4.091813	68237	4.217929	745	4.101192	4973	4.244278	47099	4.473696	5618	4.149697	903	4.116152	38061	4.155210	1197	4.045751	3	4
1831	a06386edadf3bc614dadb7044708c46c	6.0	3.0	5.5	3.5	6.0	serendipity	low	106173	3.935297	126397	4.006561	109710	3.947412	99171	4.003978	119194	4.002666	69483	4.235115	45691	4.191760	58530	3.965657	26519	3.998642	89707	4.144870	2571	3.860041	108709	3.899857	3	4
1832	bad56d9506832cd79d874a6b66b3d813	5.0	3.5	1.5	3.5	2.5	serendipity	medium	6874	4.241766	38061	4.350788	46578	4.399071	4848	4.749688	44195	4.493639	4979	4.397887	7438	4.207513	3897	4.212995	92259	4.819710	32	4.059369	3730	4.427336	3435	4.844386	4	4
1833	721ea658e148fc0f76ddd6e2b0e02422	6.5	6.5	2.5	6.5	2.0	popularity	high	140737	4.456842	110366	4.283821	89315	4.287975	112006	4.292487	113190	4.345421	26614	4.218514	50872	4.256004	4447	4.187709	8360	4.247956	8961	4.321693	597	4.278465	457	4.352895	2	2

1834 rows × 34 columns

下準備

data.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 1834 entries, 0 to 1833
Data columns (total 34 columns):
 #   Column                Non-Null Count  Dtype  
---  ------                --------------  -----  
 0   userid                1834 non-null   object 
 1    openness             1834 non-null   float64
 2    agreeableness        1834 non-null   float64
 3    emotional_stability  1834 non-null   float64
 4    conscientiousness    1834 non-null   float64
 5    extraversion         1834 non-null   float64
 6    assigned metric      1834 non-null   object 
 7    assigned condition   1834 non-null   object 
 8    movie_1              1834 non-null   int64  
 9    predicted_rating_1   1834 non-null   float64
 10   movie_2              1834 non-null   int64  
 11   predicted_rating_2   1834 non-null   float64
 12   movie_3              1834 non-null   int64  
 13   predicted_rating_3   1834 non-null   float64
 14   movie_4              1834 non-null   int64  
 15   predicted_rating_4   1834 non-null   float64
 16   movie_5              1834 non-null   int64  
 17   predicted_rating_5   1834 non-null   float64
 18   movie_6              1834 non-null   int64  
 19   predicted_rating_6   1834 non-null   float64
 20   movie_7              1834 non-null   int64  
 21   predicted_rating_7   1834 non-null   float64
 22   movie_8              1834 non-null   int64  
 23   predicted_rating_8   1834 non-null   float64
 24   movie_9              1834 non-null   int64  
 25   predicted_rating_9   1834 non-null   float64
 26   movie_10             1834 non-null   int64  
 27   predicted_rating_10  1834 non-null   float64
 28   movie_11             1834 non-null   int64  
 29   predicted_rating_11  1834 non-null   float64
 30   movie_12             1834 non-null   int64  
 31   predicted_rating_12  1834 non-null   float64
 32   is_personalized      1834 non-null   int64  
 33   enjoy_watching       1834 non-null   int64  
dtypes: float64(17), int64(14), object(3)
memory usage: 487.3+ KB

data.isna().sum()

userid                  0
 openness               0
 agreeableness          0
 emotional_stability    0
 conscientiousness      0
 extraversion           0
 assigned metric        0
 assigned condition     0
 movie_1                0
 predicted_rating_1     0
 movie_2                0
 predicted_rating_2     0
 movie_3                0
 predicted_rating_3     0
 movie_4                0
 predicted_rating_4     0
 movie_5                0
 predicted_rating_5     0
 movie_6                0
 predicted_rating_6     0
 movie_7                0
 predicted_rating_7     0
 movie_8                0
 predicted_rating_8     0
 movie_9                0
 predicted_rating_9     0
 movie_10               0
 predicted_rating_10    0
 movie_11               0
 predicted_rating_11    0
 movie_12               0
 predicted_rating_12    0
 is_personalized        0
 enjoy_watching         0
dtype: int64

data = data.drop([
                  'userid', 
                  ' movie_1',  ' predicted_rating_1',
                  ' movie_2',  ' predicted_rating_2',
                  ' movie_3',  ' predicted_rating_3',
                  ' movie_4',  ' predicted_rating_4',
                  ' movie_5',  ' predicted_rating_5',
                  ' movie_6',  ' predicted_rating_6',
                  ' movie_7',  ' predicted_rating_7',
                  ' movie_8',  ' predicted_rating_8',
                  ' movie_9',  ' predicted_rating_9',
                  ' movie_10',  ' predicted_rating_10',
                  ' movie_11',  ' predicted_rating_11',
                  ' movie_12',  ' predicted_rating_12'], axis=1)

エンコード

{column: list(data[column].unique()) for column in data.columns if data.dtypes[column] == 'object'}

{' assigned condition': [' high', ' default', ' medium', ' low'],
 ' assigned metric': [' serendipity', ' all', ' popularity', ' diversity']}

condition_ordering = [' low', ' medium', ' default', ' high']

def ordinal_encode(df, column, ordering):
    df = df.copy()
    df[column] = df[column].apply(lambda x: ordering.index(x))
    return df

def onehot_encode(df, column, prefix):
    df = df.copy()
    dummies = pd.get_dummies(df[column], prefix=prefix)
    df = pd.concat([df, dummies], axis=1)
    df = df.drop(column, axis=1)
    return df

data = ordinal_encode(data, ' assigned condition', condition_ordering)
data = onehot_encode(data, ' assigned metric', 'm')

分割とスケーリング

y = data[' enjoy_watching ']
X = data.drop([' enjoy_watching '], axis=1)

scaler = sp.StandardScaler()
X = scaler.fit_transform(X)

X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.7)

トレーニング

log_model = slm.LogisticRegression()
svm_model = svm.SVC()
ann_model = snn.MLPClassifier(hidden_layer_sizes=(16, 16))

log_model.fit(X_train, y_train)
svm_model.fit(X_train, y_train)
ann_model.fit(X_train, y_train)

log_acc = log_model.score(X_test, y_test)
svm_acc = svm_model.score(X_test, y_test)
ann_acc = ann_model.score(X_test, y_test)

/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/_multilayer_perceptron.py:571: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.
  % self.max_iter, ConvergenceWarning)

import plotly.express as px

fig = px.bar(
    x=['Logistic Regression', 'SVM', 'NN'],
    y=[log_acc, svm_acc, ann_acc],
    color=['Logistic Regression', 'SVM', 'NN'],
    labels={'x': 'Model', 'y': 'Accuracy'},
    title='Model Accuracy',
)
fig.show()