pip install nltk
pip install mecab-python
La fonction de sortie TF-IDF est tfidf La fonction d'analyse morphologique est extract_words Le long gars sous le test unitaire d'importation en bas est un test
#!/usr/bin/env python
#-*- encoding: utf-8 -*-
import nltk
import MeCab
import urllib2
from urllib2 import HTTPError
from itertools import chain
def tfidf(doc,docs):
"""TF du document cible lorsque le document cible et la liste de mots analysés par morphologie de la phrase entière sont spécifiés-Renvoie IDF"""
tokens = list(chain.from_iterable(docs)) #flatten
A = nltk.TextCollection(docs)
token_types = set(tokens)
return [{"word":token_type,"tfidf":A.tf_idf(token_type, doc)} for token_type in token_types]
def extract_words(text):
"""Texte donné, retourne sous forme de liste de nomenclature"""
text = text.encode("utf-8") if isinstance(text,unicode) else text
mecab = MeCab.Tagger("")
node = mecab.parseToNode(text)
words = []
while node:
fs = node.feature.split(",")
if (node.surface is not None) and node.surface != "" and fs[0] in [u'nom']:
words.append(node.surface)
node = node.next
return words
import unittest
class MachineLearningTest(unittest.TestCase):
def test_extract_words(self):
"""Test d'analyse morphologique"""
text = "Analyse morphologique du texte et retourne une liste de nomenclature"
keywords = extract_words(text)
self.assertEqual(keywords, ["text","morphème","une analyse","nom","liste"])
def test_tfidf(self):
"""test tfidf"""
urls = ["http://qiita.com/puriketu99/items/"+str(i) for i in range(1,10)]
def url2words(url):
try:
html = urllib2.urlopen(url).read()
except HTTPError:
html = ""
plain_text = nltk.clean_html(html).replace('\n','')
words = extract_words(plain_text)
return words
docs = [url2words(url) for url in urls]
tfidfs_fizzbuzz = tfidf(docs[0],docs)
tfidfs_fizzbuzz.sort(cmp=lambda x,y:cmp(x["tfidf"],y["tfidf"]),reverse=True)
result = [e for i,e in enumerate(tfidfs_fizzbuzz) if len(e["word"]) > 2 and i < 30]
self.assertEqual(result[7]["word"],"yaotti")#Si le côté Qiita change la conception, le test peut échouer
print result
#[{'tfidf': 0.08270135278254376, 'word': 'quot'},
# {'tfidf': 0.02819364299404901, 'word': 'FizzBuzz'},
# {'tfidf': 0.02067533819563594, 'word': 'fizzbuzz'},
# {'tfidf': 0.02067533819563594, 'word': 'Buzz'},
# {'tfidf': 0.016916185796429405, 'word': 'Fizz'},
# {'tfidf': 0.016726267030018446, 'word': 'end'},
# {'tfidf': 0.015036609596826138, 'word': 'map'},
# {'tfidf': 0.015036609596826138, 'word': 'yaotti'},
# {'tfidf': 0.011277457197619604, 'word': 'def'}]
if __name__ == '__main__':
unittest.main()
Référence Calcul de TF-IDF http://everydayprog.blogspot.jp/2011/12/tf-idf.html
Recommended Posts