#-*-coding:utf-8-*-

from tweet_analyze_app import tweet_analyze as ta

if __name__ == "__main__":
#filename
    tweet_filename = 'tweet.txt'
    mecab_filename = 'tweet.txt.mecab'
    analyze_filename = 'tweet.txt.mecab.analyze'
    double_noun_filename = 'tweet.txt.mecab.analyze.noun'

#APIparamete
    list_name = 'list名'
    list_owner = 'listの所有者'

#main()
    api = ta.tweet_api()
    ta.tweet_get(api,tweet_filename,list_owner,list_name)
    ta.tweet_mecab_analyze(tweet_filename, mecab_filename)
    ta.tweet_mecab_analyze_dict(mecab_filename,analyze_filename )
    ta.tweet_double_noun(analyze_filename,double_noun_filename)
    ta.tweet_frequecy(double_noun_filename)

#-*-coding:utf-8-*-

import tweepy
import re
import codecs
import ast
import collections
import operator
import MeCab

#twitterのapiを取得
def tweet_api():
    # API_parameter
    consumer_key = "*****"
    consumer_secret = "*****"
    access_token = "*****"
    access_token_secret = "*****"

    # API_OAuth
    auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
    auth.set_access_token(access_token, access_token_secret)
    api = tweepy.API(auth)
    return api

#tweetをGET
def tweet_get(api,filename,owner,list_name):

    list_tweets = api.list_timeline(owner,list_name,count=300)
    pattern = re.compile(r"^http(s)://.*")

    with codecs.open(filename,"w","utf-8") as f:
        for tweets in list_tweets:
            for tweet_text in tweets.text.split('\n'):
                if pattern.match(tweet_text):
                    continue
                else:
                    f.write(tweet_text+'\n')

# mecabで形態素分析
def tweet_mecab_analyze(input_filename, output_filename):

    with codecs.open(input_filename, 'r', 'utf-8') as f:
        text = f.read()

    m = MeCab.Tagger("mecabrc")
    wt = m.parse(text)

    with codecs.open(output_filename, 'w', 'utf-8') as wf:
        wf.write(wt)

#mecab分析を名詞や動詞で分類しやすいように辞書へ格納
def tweet_mecab_analyze_dict(input_filename,output_filename):
    with codecs.open(input_filename,'r','utf-8') as f:
        data = f.readlines()

    mecab_list=[]
    temp_dict ={}
    for temp_word in data:
        temp_word = temp_word.replace('\t', ',')
        temp_word = temp_word.replace('\n', '')
        if(temp_word.count(',')==9 or temp_word.count(',')==7):
            temp_list = temp_word.split(',')
            temp_dict={'surface':temp_list[0],'base':temp_list[7],'pos':temp_list[1],'pos1':temp_list[2]}
            mecab_list.append(temp_dict)
        else:
            continue

    with codecs.open(output_filename,'w','utf-8') as wf:
        for line in mecab_list:
            wf.write(str(line)+'\n')

#名詞が連続している部分を抽出
def tweet_double_noun(input_filename,output_filename):
    with codecs.open(input_filename,'r','utf-8') as f:
        temp_lines = f.readlines()

    temp_dict={}
    temp_word = ''
    temp_list = []
    flag=0

    for temp_line in temp_lines:
        temp_dict = ast.literal_eval(temp_line)
        if(temp_dict['pos']=='名詞' and flag ==0):
            temp_word = temp_dict['surface']
            flag = 1
            continue

        elif(temp_dict['pos']=='名詞' and flag >= 1):
            temp_word += temp_dict['surface']
            flag += 1
            continue

        elif(temp_dict['pos']!='名詞' and flag >= 2):
            temp_list.append(temp_word)
            temp_word = ''
            flag = 0
            continue

        else:
            temp_word=''
            flag =0
            continue

    with codecs.open(output_filename,'w','utf-8') as wf:
        for line in temp_list:
            wf.write(str(line)+'\n')

#頻度分析
def tweet_frequecy(input_filename):

    with codecs.open(input_filename, 'r', 'utf-8') as f:
        temp_lines = f.readlines()

    temp_list = []
    for temp_line in temp_lines:
        temp_word = temp_line
        temp_list.append(temp_word)

    count_dict = collections.Counter(map(str,temp_list))

    pattern = re.compile(r"^http(s)://.*")
    for value,count in sorted(count_dict.items(),key=operator.itemgetter(1),reverse=True):
        if(pattern.match(value)):
            continue
        else:
            print(count,str(value).rstrip('\n'))

4 衆院選
2 Webサイト
2 朝日新聞デジタル
2 SYNスキャン部分
2 byteflip
2 向け人気メディアプレーヤー
2 マルウェア
2 情報流出
2 東京五輪サイバー対策
2 ビットコイン
2 HTTPS化
2 進捗状況
2 採用率
2 31%
2 55%
2 さそう
2 脳内独り言
2 人たち
2 連携協定
2 都立産技高専
2 ２社
2 安倍首相
2 ビットコイン開発者
2 特殊部隊
2 ハードフォーク対立
2 メール停止
(長いので以下略)