機械学習でツイート主を推測する¶

# テキストを行毎に分割
def tw_list_from_text(filepath):
    f = open(filepath)
    text = f.read()
    f.close()
    return text.split('\n')

nardtree_list = tw_list_from_text('nardtree.txt')
NekoAntarctica_list = tw_list_from_text('NekoAntarctica.txt')

print("nardtree: {0} tweets, last: {1}".format(len(nardtree_list), nardtree_list[0]))
print("南極にゃんこ: {0} tweets, last: {1}".format(len(NekoAntarctica_list), NekoAntarctica_list[0]))

# 2人のツイートを連結
df = pd.DataFrame({'uid': 'NekoAntarctica' , 'line' : NekoAntarctica_list})
df = df.append(pd.DataFrame({'uid': 'nardtree', 'line': nardtree_list}), ignore_index=True)

df.shape

# コンテンツを本文と日付に分割
df['dt'] = [line[-31:] for line in df['line']]
df['tweet'] = [line[:len(line)-31] for line in df['line']]

# 分割前のコンテンツはもう要らないので除外
df = df.iloc[:, df.columns != 'line']

# ツイートの長さの列を追加
df['tw_len'] = [ len(tw) for tw in df['tweet'] ]

# リツイートかどうかの列を追加
df['RT'] = [s[0:2] == 'RT' for s in df['tweet']]

df.head()

# ツイートの長さの分布を確認
df.hist(column='tw_len')

# ユーザー、RT別に確認
df.hist(column='tw_len', by=['uid', 'RT'], figsize=(12,10), sharex=True, sharey=True)

# リツイート率を確認するデータフレームを作成
df_pv = pd.DataFrame()
df_pv['tw_count'] = df.groupby('uid')['tweet'].count()
df_pv['RT_count'] = df.groupby('uid')['RT'].sum()
df_pv['RT_rate'] = df_pv['RT_count'] / df_pv['tw_count']
df_pv['RT_rate'].plot(kind='bar')

# 分割表の作成
df_pv['noRT_count'] = df_pv['tw_count'] - df_pv['RT_count'] 
df_cross = df_pv[['RT_count', 'noRT_count']]
df_cross

from scipy import stats

chsq, p, dof, ef = stats.chi2_contingency(df_cross)
print("p-value: {:1.6f}".format(p))

# MeCabで1ツイートだけ解析してみる
tagger = MeCab.Tagger()
output = tagger.parse(df.loc[500, 'tweet'])
tagger_test_dt = pd.DataFrame( [line.replace('\t', ',').split(',') for line in output.split('\n')] )
tagger_test_dt

# MeCabで分割して単語のリストを取得する関数
def split_by_mecab(content, tagger):
    parsed = tagger.parse(content)
    word_list = [ line.split('\t')[0] for line in parsed.split('\n') ]
    return word_list[:-2]

# 関数のテスト
split_by_mecab(df.loc[500,'tweet'], tagger)

# 全ツイートをMeCabで分割
whole_tweets = [split_by_mecab(tweet, tagger) for tweet in df['tweet'].tolist() ]
len(whole_tweets)

# gensimで辞書を作成
dictionary = corpora.Dictionary(whole_tweets)
len(dictionary)

# 極端に頻度が少ない単語と多い単語を除外
dictionary.filter_extremes(no_below=2, no_above=0.8)
len(dictionary)

# 特徴量を作成
bows = [dictionary.doc2bow(tweet) for tweet in whole_tweets]
x = [list(matutils.corpus2dense([bow], num_terms=len(dictionary)).T[0])  for bow in bows]
x = np.array(x, dtype=np.float32)
x.shape

y = df['uid'].as_matrix()
y.shape

# 訓練データとテストデータを作成
from sklearn.model_selection import train_test_split

X_train, X_test, Y_train, Y_test = train_test_split(x, y, random_state=0)
print(X_train.shape, X_test.shape)

# ロジスティック回帰モデルでグリッドサーチ
from sklearn.model_selection import GridSearchCV
from sklearn.linear_model import LogisticRegression

logistic_param_grid = { 'C' : [0.1, 1, 10], 'penalty' : ['l1', 'l2'] }
lgs = GridSearchCV( LogisticRegression(), logistic_param_grid, cv=5 )
lgs.fit(X_train, Y_train)

lgs.best_params_

lgs.best_score_

lgs.score(X_test, Y_test)

# ロジスティック回帰モデルのテストセットに対する混同行列
logreg_pred = lgs.predict(X_test)

from sklearn.metrics import confusion_matrix
confusion = confusion_matrix(Y_test, logreg_pred)
confusion_df = pd.DataFrame(confusion, columns= ['Negative', 'Positive'], index=['True', 'False'])
confusion_df

# 分類に失敗したツイートを確認する
df['pred'] = lgs.predict( x )
df['match'] = (df['uid'] == df['pred'])
df_fail = df[df['match'] == False]
df_fail.head()

df_fail.tail()

# リツイートかどうか？別に失敗したツイート長の分布を確認
df_fail.hist(column='tw_len', by=['uid', 'RT'] ,figsize=(12,10), sharex=True, sharey=True)

# 分類に失敗したツイート中のリツイートの割合
df_fail_pv = pd.DataFrame()
df_fail_pv['tw_count'] = df_fail.groupby('uid')['tweet'].count()
df_fail_pv['RT_count'] = df_fail.groupby('uid')['RT'].sum()
df_fail_pv['RT_rate'] = df_fail_pv['RT_count'] / df_fail_pv['tw_count']
df_fail_pv['RT_rate'].plot(kind='bar')

# 10文字以下のツイートとRTを除く
df_sub = df[df['tw_len'] > 10]
df_sub = df_sub[df_sub['RT'] == False]

X_train, X_test, Y_train, Y_test = df_to_train_test_split(df_sub)
grid_search_sub = logistic_grid_search(X_train, Y_train)

print(grid_search_sub.best_score_)
print(grid_search_sub.score(X_test, Y_test))

tjo_list = tw_list_from_text('TJO.txt')
print(tjo_list[0])

# 南極にゃんこ - TJO ペアのデータフレームを作成
df_n_tjo = tw_text_to_df(['NekoAntarctica', 'TJO'], ['NekoAntarctica.txt', 'TJO.txt'])

# 10文字以下のツイートとRTを除外
df_n_tjo = df_n_tjo[df_n_tjo['tw_len'] > 10]
df_n_tjo = df_n_tjo[df_n_tjo['RT'] == False]

# ロジスティック回帰で学習
X_train, X_test, Y_train, Y_test = df_to_train_test_split(df_n_tjo)
grid_search_ntjo = logistic_grid_search(X_train, Y_train)

print(grid_search_ntjo.best_score_)
print(grid_search_ntjo.score(X_test, Y_test))