text miner

algorithm/text_classifical/base.py

@@ -8,6 +8,7 @@ from preprocesser.filter import stopwords_filter
 from collections import Counter
 from config import config
 import os
+import codecs
 
 
 class SELECTED_CONTENT_TYPE():
@@ -28,21 +29,21 @@ class TextClassifical(object):
 
     def build_network_influencer_words(self, word_path):
         ret = {}
-        for line in open(word_path, "r", errors="ignore"):
+        for line in codecs.open(word_path, "r", errors="ignore"):
             line = line.strip()
             ret[line] = 1
         return ret
 
     def build_project_words(self, project_path):
         ret = {}
-        for line in open(project_path, "r", errors="ignore"):
+        for line in codecs.open(project_path, "r", errors="ignore"):
             line = line.strip()
             ret[line] = 1
         return ret
 
     def build_star_words(self, star_path):
         ret = {}
-        for line in open(star_path, "r", errors="ignore"):
+        for line in codecs.open(star_path, "r", errors="ignore"):
             line = line.strip()
             ret[line] = 1
         return ret

build/lib/algorithm/text_classifical/base.py

@@ -8,6 +8,7 @@ from preprocesser.filter import stopwords_filter
 from collections import Counter
 from config import config
 import os
+import codecs
 
 
 class SELECTED_CONTENT_TYPE():
@@ -28,21 +29,21 @@ class TextClassifical(object):
 
     def build_network_influencer_words(self, word_path):
         ret = {}
-        for line in open(word_path, "r", errors="ignore"):
+        for line in codecs.open(word_path, "r", errors="ignore"):
             line = line.strip()
             ret[line] = 1
         return ret
 
     def build_project_words(self, project_path):
         ret = {}
-        for line in open(project_path, "r", errors="ignore"):
+        for line in codecs.open(project_path, "r", errors="ignore"):
             line = line.strip()
             ret[line] = 1
         return ret
 
     def build_star_words(self, star_path):
         ret = {}
-        for line in open(star_path, "r", errors="ignore"):
+        for line in codecs.open(star_path, "r", errors="ignore"):
             line = line.strip()
             ret[line] = 1
         return ret

build/lib/preprocesser/filter.py

@@ -5,7 +5,7 @@
 # software: PyCharm
 import re, os
 from config import config
-
+import codecs
 
 class Filter(object):
     def __init__(self, file_path, encoding="utf-8"):
@@ -26,7 +26,7 @@ class StopwordsFilter(Filter):
         return re.sub(u"([^\u4e00-\u9fa5\u0030-\u0039\u0041-\u005a\u0061-\u007a])", "", corpus)
 
     def init(self):
-        for line in open(self.file_path, "r", encoding=self.encoding):
+        for line in codecs.open(self.file_path, "r", encoding=self.encoding):
             try:
                 line = line.strip()
                 self.stopwords.add(line)

preprocesser/filter.py

@@ -5,7 +5,7 @@
 # software: PyCharm
 import re, os
 from config import config
-
+import codecs
 
 class Filter(object):
     def __init__(self, file_path, encoding="utf-*"):
@@ -26,7 +26,7 @@ class StopwordsFilter(Filter):
         return re.sub(u"([^\u4e00-\u9fa5\u0030-\u0039\u0041-\u005a\u0061-\u007a])", "", corpus)
 
     def init(self):
-        for line in open(self.file_path, "r", encoding=self.encoding):
+        for line in codecs.open(self.file_path, "r", encoding=self.encoding):
             try:
                 line = line.strip()
                 self.stopwords.add(line)