나이즈 베이즈(Naive Bayes)를 사용한 유사문서와 스팸필터

from sklearn.datasets import fetch_20newsgroups

categories = ['alt.atheism', 'soc.religion.christian', 'comp.graphics', 'sci.med']
twenty_train = fetch_20newsgroups(subset='train', categories=categories, shuffle=True, random_state=13)
twenty_train.keys()

print(twenty_train['DESCR'])

print(len(twenty_train.data))
twenty_train.target_names

print(twenty_train['data'][0])

print(twenty_train.target_names[twenty_train['target'][0]])

print(twenty_train.target_names)
twenty_train.target[:10]

from sklearn.feature_extraction.text import CountVectorizer

count_vect = CountVectorizer()
X_train_counts = count_vect.fit_transform(twenty_train.data)
X_train_counts.shape

X_train_counts

print(len(X_train_counts.toarray()[0]))
X_train_counts.toarray()[0]

import numpy as np

print('전체 데이터의 합 : ', np.sum(X_train_counts.toarray()[0]))
print('0이 아닌 데이터의 갯수 : ', len(X_train_counts.toarray()[0][X_train_counts.toarray()[0] != 0]))

from sklearn.feature_extraction.text import TfidfTransformer

tf_transformer = TfidfTransformer(use_idf=False).fit(X_train_counts)
X_train_tf = tf_transformer.transform(X_train_counts)
X_train_tf.shape

from sklearn.naive_bayes import MultinomialNB
clf = MultinomialNB().fit(X_train_tf, twenty_train.target)

docs_new =['God is love', 'OpenGL on the GPU is fast']
X_new_counts = count_vect.transform(docs_new)
X_new_tfidf = tf_transformer.transform(X_new_counts)

predicted = clf.predict(X_new_tfidf)

for doc, categories in zip(docs_new, predicted):
    print(f'{doc} => {twenty_train.target_names[categories]}')

from sklearn.pipeline import Pipeline

text_clf = Pipeline([
    ('vect', CountVectorizer()),
    ('tfidf', TfidfTransformer()),
    ('clf', MultinomialNB()),
])

text_clf.fit(twenty_train.data, twenty_train.target)

categories = ['alt.atheism', 'soc.religion.christian', 'comp.graphics', 'sci.med']
twenty_test = fetch_20newsgroups(subset='test', categories=categories, shuffle=True, random_state=13)

docs_test = twenty_test.data
predicted = text_clf.predict(docs_test)
np.mean(predicted == twenty_test.target)

from sklearn.metrics import classification_report

print(classification_report(twenty_test.target, predicted, target_names= twenty_test.target_names))

from sklearn.metrics import confusion_matrix

confusion_matrix(twenty_test.target, predicted)

import pandas as pd

messages = pd.read_csv('https://raw.githubusercontent.com/hmkim312/datas/main/spam/spam.csv', encoding='latin-1')
messages.drop(['Unnamed: 2','Unnamed: 3','Unnamed: 4'], axis=1, inplace=True)
messages = messages.rename(columns={'v1' : 'class', 'v2' : 'text'})
messages.head()

messages.groupby('class').describe()

messages['length'] = messages['text'].apply(len)
messages.head()

messages.hist(column= 'length', by = 'class', bins = 50, figsize=(15, 6))
plt.show()

import string
from nltk.corpus import stopwords

def process_text(text):
    nopunc = [char for char in text if char not in string.punctuation]
    nopunc = ''.join(nopunc)
    
    clean_words = [word for word in nopunc.split() if word.lower() not in stopwords.words('english')]
    
    return clean_words

messages['text'].apply(process_text).head()

from sklearn.model_selection import train_test_split

msg_train, msg_test, class_train, class_test = train_test_split(messages['text'], messages['class'], test_size = 0.2)

pipeline = Pipeline([
    ('vect', CountVectorizer(analyzer=process_text)),
    ('tfidf', TfidfTransformer()),
    ('clf', MultinomialNB())
])

pipeline.fit(msg_train, class_train)

predictions = pipeline.predict(msg_test)
print(classification_report(class_test, predictions))

import seaborn as sns
sns.heatmap(confusion_matrix(class_test, predictions), annot= True, fmt = 'd')
plt.show()