!apt install aptitude
!aptitude install mecab libmecab-dev mecab-ipadic-utf8 git make curl xz-utils file -y
!pip install mecab-python3==0.7

import json
import pandas as pd
import numpy as np
import re
import csv
import MeCab
import matplotlib.pyplot as plt
import traceback
from google.colab import drive 
drive.mount('/content/drive')
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import MinMaxScaler

# 岩波国語辞書のダウンロード
import urllib.request
import zipfile
# URLを指定
url = "https://storage.googleapis.com/tutor-contents-dataset/6050_stock_price_prediction_data.zip"
save_name = url.split('/')[-1]
# ダウンロードする
mem = urllib.request.urlopen(url).read()
# ファイルへ保存
with open(save_name, mode='wb') as f:
    f.write(mem)
# zipファイルをカレントディレクトリに展開する
zfile = zipfile.ZipFile(save_name)
zfile.extractall('.')

# MeCabインスタンスの作成．引数を無指定にするとIPA辞書になります．
m = MeCab.Tagger('')

# テキストを形態素解析し辞書のリストを返す関数
def get_diclist(text):
    parsed = m.parse(text)      # 形態素解析結果（改行を含む文字列として得られる）
    lines = parsed.split('\n')  # 解析結果を1行（1語）ごとに分けてリストにする
    lines = lines[0:-2]         # 後ろ2行は不要なので削除
    diclist = []
    for word in lines:
        l = re.split('\t|,',word)  # 各行はタブとカンマで区切られてるので
        d = {'Surface':l[0], 'POS1':l[1], 'POS2':l[2], 'BaseForm':l[7]}
        diclist.append(d)
    return(diclist)

# 形態素解析結果の単語ごとのdictデータにPN値を追加する関数
def add_pnvalue(diclist_old, pn_dict):
    diclist_new = []
    for word in diclist_old:
        base = word['BaseForm']        # 個々の辞書から基本形を取得
        if base in pn_dict:
            pn = float(pn_dict[base]) 
        else:
            pn = 'notfound'            # その語がPN Tableになかった場合
        word['PN'] = pn
        diclist_new.append(word)
    return(diclist_new)

# 各ツイートのPN平均値
def get_mean(dictlist):
    pn_list = []
    for word in dictlist:
        pn = word['PN']
        if pn!='notfound':
            pn_list.append(pn)
    if len(pn_list)>0:
        pnmean = np.mean(pn_list)
    else:
        pnmean=0
    return pnmean

df_tw = pd.read_csv('/content/drive/MyDrive/aidemy/ブログ＿食べログ/tweet/csv after/(216)銀座ウエスト.csv',
                 usecols=['text', 'name','created_at'])
df_tw = pd.DataFrame(df_tw)
df_tw.index +=1
df_tw = df_tw[["text"]]

# 岩波国語辞書の読み込み
pn_df = pd.read_csv('./6050_stock_price_prediction_data/pn_ja.csv', encoding='utf-8', names=('Word','Reading','POS', 'PN'))
#word_listにリスト型でWordを格納
word_list = list(pn_df['Word'])
#pn_listにリスト型でPNを格納
pn_list   = list(pn_df['PN'])
#pn_dictとしてword_list, pn_listを格納した辞書を作成
pn_dict   = dict(zip(word_list, pn_list))

# means_listという空のリストを作りそこにツイートごとの平均値を求めてください。
means_list = []
for tweet in df_tw["text"]:
    dl_old = get_diclist(tweet)
    dl_new = add_pnvalue(dl_old, pn_dict)
    pnmean = get_mean(dl_new)
    means_list.append(pnmean)
df_tw['pn'] = means_list

!apt install aptitude
!aptitude install mecab libmecab-dev mecab-ipadic-utf8 git make curl xz-utils file -y
!pip install mecab-python3==0.7

import json
import pandas as pd
import numpy as np
import re
import csv
import MeCab
import matplotlib.pyplot as plt
import traceback
from google.colab import drive 
drive.mount('/content/drive')
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import MinMaxScaler

# カラム名と値の位置ずれを制御
pd.set_option('display.unicode.east_asian_width', True)

pndic_1 = pd.read_csv('/content/drive/MyDrive/aidemy/ブログ＿食べログ/tweet/pn.csv.m3.120408.trim', names=['word_pn_oth'])
#区切り文字で展開
pndic_2 = pndic_1['word_pn_oth'].str.split('\t', expand=True)
#感情極性値のノイズを削除
pndic_3 = pndic_2[(pndic_2[1] == 'p') | (pndic_2[1] == 'e') | (pndic_2[1] == 'n')]
#感情極性値を数値に置換
pndic_4 = pndic_3.drop(pndic_3.columns[2], axis=1)
pndic_4[1] = pndic_4[1].replace({'p':1, 'e':0, 'n':-1})

word_list = list(pndic_4[0].tolist())
pn_list   = list(pndic_4[1].tolist())
pn_dict   = dict(zip(word_list, pn_list))

# MeCabインスタンスの作成
m = MeCab.Tagger('')

# テキストを形態素解析し辞書のリストを返す
def get_diclist(text):
    parsed = m.parse(text)     
    lines = parsed.split('\n')  
    lines = lines[0:-2]        
    diclist = []
    for word in lines:
        l = re.split('\t|,',word) 
        d = {'Surface':l[0], 'POS1':l[1], 'POS2':l[2], 'BaseForm':l[7]}
        diclist.append(d)
    return(diclist)

# 形態素解析結果の単語ごとのdictデータにPN値を追加
def add_pnvalue(diclist_old, pn_dict):
    diclist_new = []
    for word in diclist_old:
        base = word['BaseForm']       
        if base in pn_dict:
            pn = float(pn_dict[base]) 
        else:
            pn = 'notfound'            
        word['PN'] = pn
        diclist_new.append(word)
    return(diclist_new)

# 各ツイートのPN平均値
def get_mean(dictlist):
    pn_list = []
    for word in dictlist:
        pn = word['PN']
        if pn!='notfound':
            pn_list.append(pn)
    if len(pn_list)>0:
        pnmean = np.mean(pn_list)
    else:
        pnmean=0
    return pnmean

#ツイートの読み込み
df_tw = pd.read_csv('/content/drive/MyDrive/aidemy/ブログ＿食べログ/tweet/csv after/(786)ピエールエルメ.csv',
                 usecols=['text', 'name','created_at'])
df_tw = pd.DataFrame(df_tw)
df_tw.index +=1
df_tw = df_tw[["text"]]

#ツイートごとの平均値
means_list = []
for tweet in df_tw["text"]:
    dl_old = get_diclist(tweet)
    dl_new = add_pnvalue(dl_old, pn_dict)
    pnmean = get_mean(dl_new)
    means_list.append(pnmean)
df_tw['pn'] = means_list

# 標準化
st = StandardScaler()
df_tw["pn_st"] = st.fit_transform(df_tw[["pn"]])

# 正規化(0-5)
sc = MinMaxScaler([0,5])
df_tw["pn_sc"] = sc.fit_transform(df_tw[["pn_st"]])

#並び替え
df_tw = df_tw.sort_values(by='pn_sc', ascending=False)