Merge branch 'main' into feature/baidu_tieba_20240805

NanmiCoder · Aug 8, 2024 · 62ac454 · 62ac454
2 parents 3f42368 + 7e9a759
commit 62ac454
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/tools/words.py b/tools/words.py
@@ -27,7 +27,7 @@ def load_stop_words(self):
 
     async def generate_word_frequency_and_cloud(self, data, save_words_prefix):
         all_text = ' '.join(item['content'] for item in data)
-        words = [word for word in jieba.lcut(all_text) if word not in self.stop_words]
+        words = [word for word in jieba.lcut(all_text) if word not in self.stop_words and len(word.strip()) > 0]
         word_freq = Counter(words)
 
         # Save word frequency to file