# -*- coding: utf-8 -*-
from sklearn.metrics import accuracy_score
from sklearn.naive_bayes import MultinomialNB
from sklearn.model_selection import train_test_split
from sklearn.externals import joblib
from sklearn.metrics import confusion_matrix
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import json
import pandas as pd
import jieba
import re
import os
topWordNum = 500#保留前n个高频词形成词典
testRate = 0.2#测试集比例
seed = 0#随机种子
#路径目录
baseDir = ''#根目录
staticDir = os.path.join(baseDir,'Static')#静态文件目录
resultDir = os.path.join(baseDir,'Result')#结果文件目录

#载入停用词
with open(staticDir+'/中文停用词.txt','r',encoding='utf-8')as f:
    stopWords = [line.strip('\n') for line in f.readlines()]
    stopWords += '\n'
stopWords[300:305]

['乘', '乘势', '乘机', '乘胜', '乘虚']

#载入语料
data = pd.read_excel(staticDir+'/新闻分类语料.xlsx')
print('载入语料，获得%d条语料'%data.shape[0])
data.head()

载入语料，获得18000条语料

def clearpara(para):
    '''
    清理段落，分词，删除停用词
    '''
    words = jieba.cut(para)#分词
    #过滤
    words = [word for word in words if len(word)>1]
    words = [word for word in words if word not in stopWords]
    return words
print('正在对语料分词，请耐心...')
data['words'] = data['content'].apply(clearpara)
data.head()

Building prefix dict from the default dictionary ...


正在对语料分词，请耐心...


Dumping model to file cache C:\Users\ADMINI~1\AppData\Local\Temp\jieba.cache
Loading model cost 2.110 seconds.
Prefix dict has been built successfully.

#词频统计，保留top N
print('正在进行词频统计')
items = data['words'].values.tolist()
words = []
for item in items:
    words.extend(item)
wordCount = pd.Series(words).value_counts()[0:topWordNum]
wordCount = wordCount.index.values.tolist()

正在进行词频统计

def wordsToVec(words):
    '''将词组转为向量，此处向量的数为高频词出现次数'''
    vec = map(lambda word:words.count(word),wordCount)
    vec = list(vec)
    return vec
print('正在将词组转为句向量，请耐心...')
data['vec'] = data['words'].apply(wordsToVec)
data = data.drop(['content'],axis=1)
data.head()

正在将词组转为句向量，请耐心...

#输出字符转数字
flags = data['flag'].value_counts().index.values.tolist()
data['flag'] = data['flag'].apply(lambda x:flags.index(x))
data.head()

#数据集分割
xTrain,xTest,yTrain,yTest = train_test_split(data['vec'].tolist(),data['flag'].tolist(),test_size=testRate,random_state=seed)
#建模
model = MultinomialNB()
model.fit(xTrain,yTrain)
#预测
yPred = model.predict(xTest)
yTest = np.array(yTest)
yPred = np.array(yPred)
print('模型测试集准确率:%.2f'%accuracy_score(yTest,yPred))

模型测试集准确率:0.79

#用来正常显示中文标签
plt.rcParams['font.sans-serif']=['SimHei'] 
#用来正常显示负号
plt.rcParams['axes.unicode_minus']=False
#计算混淆矩阵
confusion = confusion_matrix(yTest,yPred)
plt.clf()
plt.title('分类混淆矩阵')
sns.heatmap(confusion, square=True, annot=True, fmt='d', cbar=False,
            xticklabels=flags,
            yticklabels=flags,
            linewidths=0.1,cmap='YlGnBu_r')
plt.ylabel('观测')
plt.xlabel('预测')
plt.xticks(rotation=-13)
plt.savefig(resultDir+'/新闻分类混淆矩阵.png',dpi=100)

def predict(para):
    '''输入文章，输出分类标签'''
    words = clearpara(para)#分词
    vec = wordsToVec(words)#句向量
    vec = np.array(vec)[np.newaxis,:]
    flagIndex = model.predict(vec)[0]
    flag = flags[flagIndex]
    return flag
#一则体育新闻
para = '明天凌晨，欧洲杯Ｃ组将展开第二轮的争夺。亨利在为国家队征战的１００场比赛中攻入了４４个进球，是法国队当仁不让的王牌前锋。而队长维埃拉是法国中场不可或缺的核心人物。媒体猜测在伯尔尼迎战荷兰队时，他们两人将取代阿内尔卡和图拉朗先发。另外两个需要调整的是左、右边后卫阿比达尔和萨尼奥尔，两人在首场比赛的。”'
print(para)
print('\n该则新闻分类为%s'%predict(para))

明天凌晨，欧洲杯Ｃ组将展开第二轮的争夺。亨利在为国家队征战的１００场比赛中攻入了４４个进球，是法国队当仁不让的王牌前锋。而队长维埃拉是法国中场不可或缺的核心人物。媒体猜测在伯尔尼迎战荷兰队时，他们两人将取代阿内尔卡和图拉朗先发。另外两个需要调整的是左、右边后卫阿比达尔和萨尼奥尔，两人在首场比赛的。”

该则新闻分类为sports