Skip to content

Commit ce8c0e1

Browse files
crazyyanhcaocrazyyanhcao
committed
ipython helloworld!
1 parent c8fec38 commit ce8c0e1

File tree

1 file changed

+304
-0
lines changed

1 file changed

+304
-0
lines changed

‎ipython_analysis_newwordfind.ipynb

+304
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,304 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": 22,
6+
"metadata": {
7+
"collapsed": false
8+
},
9+
"outputs": [
10+
{
11+
"name": "stdout",
12+
"output_type": "stream",
13+
"text": [
14+
"原标题形势严峻这个地方书记市长纪委书记为何连续空降市委书记市长市委副书记接连落马的广东江门市政治生态修复从补齐关键岗位开始在连续迎来空降市委书记市长候选人后江门新一任纪委书记近日也到岗了值得关注的是他也是从省里空降的还是纪检部门这位新纪委书记叫项天保任职省纪委年在案例管理派驻机构巡视部门等关键岗位都工作过经验十分丰富去年底江门成立市委巡察工作机构时任省委巡视办副主任的项天保亲赴江门参加了启动仪式长安街知事此前曾介绍过江门是腐败的重灾区市委书记毛荣楷市长邓伟根市委副书记政法委书记邹家军市委常委王积俊市人大常委会副主任聂党权曾任市委副书记落马班子塌方全国罕见中央派来了一个沙瑞金省委书记又派来了一个田国富纪委书记这是人民的名义里的一个情节以此说明推动从严治党的迫切性江门的情况与此类似市委书记林应武市长候选人刘毅都是从省委组织部副部长任上调来江门的补位落马前任如今新纪委书记又从省级纪检部门调来从一个侧面也反映出地方反腐形势的严峻性就在项天保就任的会议上前任纪委书记胡钛也以新身份亮相他已经出任市委副书记政法委书记也就是说现在江门市委常委班子中有两名来自纪检系统的领导胡钛是军转干部年底刚刚调任江门市纪委书记他有两次救火经历一次是梅州一次是江门年梅州市委书记朱泽君和纪委书记李纯德相继被调离此后又相继被查媒体对两人内斗多有报道胡钛正是接替了李的梅州纪委书记职务而去年赴江门履新正是该市市委书记毛荣楷和市委副书记邹家军落马之后胡钛之前的江门市纪委书记周伟万也是一名老纪检在纪检政法战线工作了年今年初当选市政协主席面对从严治党的新形势和班子塌方的旧局面接力反腐任重道远近日召开的江门全市领导干部大会上广东省委常委组织部长邹铭根据省委书记胡春华同志的指示对全市领导干部提出三点要求其中特别指出——要进一步严明政治纪律和政治规矩营造良好的政治生态要保持干部队伍思想稳定和改革发展大局稳定积极引导广大干部群众把违纪违法的个人问题与江门整体工作区分开来不因人废事不因案划线不因此否定江门的工作影响江门的发展稳定营造良好的政治生态更好地推动发展无疑是江门工作当下的重中之重来源长安街知事责任编辑初晓慧文章关键词纪委书记市长纪检我要反馈保存网页\n"
15+
]
16+
}
17+
],
18+
"source": [
19+
"# 测试代码(读取的数据是一篇新闻)\n",
20+
"import numpy as np\n",
21+
"import pandas as pd\n",
22+
"import re\n",
23+
"from numpy import log, min\n",
24+
"import pymysql\n",
25+
"f = open('/home/yanchao/PyCharmProject/TextSummary/news.txt', 'r')\n",
26+
"s = f.read()\n",
27+
"drop_dict = [u'，', u'\\n', u'。', u'、', u'：', u'(', u')', u'[', u']', u'.', u',', u' ', u'\\u3000', u'”', u'“', u'？', u'?',\n",
28+
" u'！', u'‘', u'’',u'(',u')',u'《',u'》', u'（',u'）',u'…',u'-',u'0',u'1',u'2',u'3',u'4',u'5',u'6',u'7',u'8',u'9',\n",
29+
" u':',u'q',u'w',u'e',u'r',u't',u'y',u'u',u'i',u'o',u'u',u'p',u'a',u's',u'd',u'f',u'g',u'h',u'j',u'k',u'l',u'z',\n",
30+
" u'x',u'c',u'v',u'b',u'n',u'm',u'<',u'>',u'@',u'!',u'#',u'$',u'%',u'^',u'&',u'*',u'/',u'?',u'~',u'Q',u'W',u'E',\n",
31+
" u'R',u'T',u'Y',u'U',u'I',u'O',u'P',u'A',u'S',u'D',u'F',u'G','H',u'J',u'K',u'L',u'Z',u'X',u'C',u'V',u'B',u'N',u'M',\n",
32+
" u'【',u'】',u'|',u'à',u'╰',u'{',u'=',u';',u',',u'\\ ',u' \\ ',u'\\\\',u'[',u']',u'﹌﹌﹌']\n",
33+
"for i in drop_dict: # 去掉标点字或者字段\n",
34+
" s = s.replace(i, '')\n",
35+
"print(s)"
36+
]
37+
},
38+
{
39+
"cell_type": "code",
40+
"execution_count": 23,
41+
"metadata": {
42+
"collapsed": false
43+
},
44+
"outputs": [
45+
{
46+
"name": "stdout",
47+
"output_type": "stream",
48+
"text": [
49+
"[委 37\n",
50+
"市 28\n",
51+
"记 27\n",
52+
"书 27\n",
53+
"的 25\n",
54+
"门 20\n",
55+
"纪 20\n",
56+
"江 17\n",
57+
"任 14\n",
58+
"是 14\n",
59+
"部 11\n",
60+
"长 10\n",
61+
"政 9\n",
62+
"一 9\n",
63+
"副 8\n",
64+
"来 8\n",
65+
"省 8\n",
66+
"年 7\n",
67+
"治 7\n",
68+
"从 7\n",
69+
"人 7\n",
70+
"检 6\n",
71+
"作 6\n",
72+
"新 6\n",
73+
"工 6\n",
74+
"也 6\n",
75+
"个 6\n",
76+
"了 6\n",
77+
"严 5\n",
78+
"保 5\n",
79+
" ..\n",
80+
"类 1\n",
81+
"思 1\n",
82+
"金 1\n",
83+
"老 1\n",
84+
"央 1\n",
85+
"统 1\n",
86+
"下 1\n",
87+
"见 1\n",
88+
"问 1\n",
89+
"规 1\n",
90+
"武 1\n",
91+
"式 1\n",
92+
"身 1\n",
93+
"据 1\n",
94+
"级 1\n",
95+
"例 1\n",
96+
"毅 1\n",
97+
"馈 1\n",
98+
"自 1\n",
99+
"源 1\n",
100+
"系 1\n",
101+
"迫 1\n",
102+
"如 1\n",
103+
"王 1\n",
104+
"转 1\n",
105+
"点 1\n",
106+
"媒 1\n",
107+
"俊 1\n",
108+
"修 1\n",
109+
"标 1\n",
110+
"dtype: int64]\n"
111+
]
112+
}
113+
],
114+
"source": [
115+
"min_count = 10\n",
116+
"min_support = 30\n",
117+
"min_s = 3\n",
118+
"max_sep = 4\n",
119+
"\n",
120+
"t = []\n",
121+
"t.append(pd.Series(list(s)).value_counts())\n",
122+
"print(t)"
123+
]
124+
},
125+
{
126+
"cell_type": "code",
127+
"execution_count": 24,
128+
"metadata": {
129+
"collapsed": false
130+
},
131+
"outputs": [
132+
{
133+
"name": "stdout",
134+
"output_type": "stream",
135+
"text": [
136+
"915\n"
137+
]
138+
}
139+
],
140+
"source": [
141+
"tsum = t[0].sum() # 统计文本总字数\n",
142+
"print(tsum)"
143+
]
144+
},
145+
{
146+
"cell_type": "code",
147+
"execution_count": 25,
148+
"metadata": {
149+
"collapsed": false
150+
},
151+
"outputs": [
152+
{
153+
"name": "stdout",
154+
"output_type": "stream",
155+
"text": [
156+
"正在生成2字词...\n",
157+
"正在生成3字词...\n",
158+
"正在生成4字词...\n"
159+
]
160+
}
161+
],
162+
"source": [
163+
"myre = {2: '(..)', 3: '(...)', 4: '(....)', 5: '(.....)', 6: '(......)', 7: '(.......)'}\n",
164+
"rt = [] # 保存结果用\n",
165+
"for m in range(2, max_sep + 1):\n",
166+
" print(u'正在生成%s字词...' % m)\n",
167+
" t.append([])\n",
168+
" for i in range(m): # 生成所有可能的m字词\n",
169+
" t[m - 1] = t[m - 1] + re.findall(myre[m], s[i:])\n",
170+
"\n",
171+
" t[m - 1] = pd.Series(t[m - 1]).value_counts() # 逐词统计\n",
172+
" t[m - 1] = t[m - 1][t[m - 1] > min_count] # 最小次数筛选\n",
173+
" tt = t[m - 1][:]\n",
174+
" for k in range(m - 1):\n",
175+
" '''map(function,*iterables);lambda关键字支持将函数赋值给变量的一个操作符 默认是返回的,所以不用再加return关键字'''\n",
176+
" qq = np.array(list(map(lambda ms: tsum * t[m - 1][ms] / t[m - 2 - k][ms[:m - 1 - k]] / t[k][ms[m - 1 - k:]],\n",
177+
" tt.index))) > min_support # 最小支持度筛选（凝固程度(PMI的最小值)）\n",
178+
" tt = tt[qq]\n",
179+
" rt.append(tt.index)"
180+
]
181+
},
182+
{
183+
"cell_type": "code",
184+
"execution_count": 26,
185+
"metadata": {
186+
"collapsed": true
187+
},
188+
"outputs": [],
189+
"source": [
190+
"def cal_S(sl): # 信息熵计算函数;熵越大则丰富程度越高。\n",
191+
" return -((sl / sl.sum()).apply(log) * sl / sl.sum()).sum()"
192+
]
193+
},
194+
{
195+
"cell_type": "code",
196+
"execution_count": 27,
197+
"metadata": {
198+
"collapsed": false
199+
},
200+
"outputs": [
201+
{
202+
"name": "stdout",
203+
"output_type": "stream",
204+
"text": [
205+
"正在进行2字词的最大熵筛选(2)...\n",
206+
"正在进行3字词的最大熵筛选(0)...\n",
207+
"正在进行4字词的最大熵筛选(0)...\n"
208+
]
209+
}
210+
],
211+
"source": [
212+
"for i in range(2, max_sep + 1): # print(i) 2,3,4\n",
213+
" print(u'正在进行%s字词的最大熵筛选(%s)...' % (i, len(rt[i - 2]))) # len()返回rt[]中保存的结果 # 信息论中保留最大的不确定性，也就是说让熵达到最大\n",
214+
" pp = [] # 保存所有的左右邻结果\n",
215+
" for j in range(i):\n",
216+
" pp = pp + re.findall('(.)%s(.)' % myre[i], s[j:]) # 正则匹配i个字的词\n",
217+
" pp = pd.DataFrame(pp).set_index(1).sort_index() # 先排序，这个很重要，可以加快检索速度；DataFrame表格形结构类似与R语言的数据框\n",
218+
" index = np.sort(np.intersect1d(rt[i - 2], pp.index)) # 作交集;intersect1d()寻找交集；sort()返回数组的排序\n",
219+
" # 下面两句分别是左邻和右邻信息熵筛选\n",
220+
" # min_s录取词语的最低信息熵,信息熵越大越有可能独立成词\n",
221+
" index = index[np.array(list(map(lambda s: cal_S(pd.Series(pp[0][s]).value_counts()), index))) > min_s]\n",
222+
" rt[i - 2] = index[np.array(list(map(lambda s: cal_S(pd.Series(pp[2][s]).value_counts()), index))) > min_s]"
223+
]
224+
},
225+
{
226+
"cell_type": "code",
227+
"execution_count": 28,
228+
"metadata": {
229+
"collapsed": false
230+
},
231+
"outputs": [],
232+
"source": [
233+
"pd.DataFrame(pd.concat(t[1:])).to_csv('result.txt', header=False)"
234+
]
235+
},
236+
{
237+
"cell_type": "code",
238+
"execution_count": 33,
239+
"metadata": {
240+
"collapsed": false
241+
},
242+
"outputs": [
243+
{
244+
"name": "stdout",
245+
"output_type": "stream",
246+
"text": [
247+
"书记,27\n",
248+
"\n",
249+
"委书,21\n",
250+
"\n",
251+
"江门,17\n",
252+
"\n",
253+
"市委,14\n",
254+
"\n",
255+
"纪委,12\n",
256+
"\n",
257+
"委书记,21\n",
258+
"\n",
259+
"纪委书,11\n",
260+
"\n",
261+
"纪委书记,11\n",
262+
"\n"
263+
]
264+
}
265+
],
266+
"source": [
267+
"for line in open('result.txt'):\n",
268+
" print(line)"
269+
]
270+
},
271+
{
272+
"cell_type": "code",
273+
"execution_count": null,
274+
"metadata": {
275+
"collapsed": true
276+
},
277+
"outputs": [],
278+
"source": [
279+
""
280+
]
281+
}
282+
],
283+
"metadata": {
284+
"kernelspec": {
285+
"display_name": "Python 3",
286+
"language": "python",
287+
"name": "python3"
288+
},
289+
"language_info": {
290+
"codemirror_mode": {
291+
"name": "ipython",
292+
"version": 3.0
293+
},
294+
"file_extension": ".py",
295+
"mimetype": "text/x-python",
296+
"name": "python",
297+
"nbconvert_exporter": "python",
298+
"pygments_lexer": "ipython3",
299+
"version": "3.6.0"
300+
}
301+
},
302+
"nbformat": 4,
303+
"nbformat_minor": 0
304+
}

0 commit comments

Comments
 (0)