比较开源Python的工作-用于识别命名实体的库

介绍

公认的命名实体

语言不平等

要处理的文字

english_text = ''' I want a person available 7 days and with prompt response all most every time. Only Indian freelancer need I need PHP developer who have strong experience in Laravel and Codeigniter framework for daily 4 hours. I need this work by Monday 27th Jan. should be free from plagiarism . 
Need SAP FICO consultant for support project needs to be work on 6 months on FI AREAWe.  Want a same site to be created as the same as this https://www.facebook.com/?ref=logo, please check the site before contacting to me and i want this site to be ready in 10 days. They will be ready at noon tomorrow .'''

russian_text = '''   110     ,     .        https://www.sobyanin.ru/  , 1 .     .51 (   :  , )  ?     2107   47     24,    . 
 c        10  1970 ,     -, . ,  5/1 8 000 ( )  00  .               .              - .'''

NLTK库

import nltk
nltk.download('punkt')
nltk.download('averaged_perceptron_tagger')
nltk.download('maxent_ne_chunker')
nltk.download('words')
for sent in nltk.sent_tokenize(english_text):
   for chunk in nltk.ne_chunk(nltk.pos_tag(nltk.word_tokenize(sent))):
      if hasattr(chunk, 'label'):
         print(chunk)

斯坦福大学

from nltk.tag.stanford import StanfordNERTagger
jar = "stanford-ner-2015-04-20/stanford-ner-3.5.2.jar"
model = "stanford-ner-2015-04-20/classifiers/" 
st_3class = StanfordNERTagger(model + "english.all.3class.distsim.crf.ser.gz", jar, encoding='utf8') 
st_4class = StanfordNERTagger(model + "english.conll.4class.distsim.crf.ser.gz", jar, encoding='utf8') 
st_7class = StanfordNERTagger(model + "english.muc.7class.distsim.crf.ser.gz", jar, encoding='utf8')
for i in [st_3class.tag(english_text.split()), st_4class.tag(english_text.split()), st_7class.tag(english_text.split())]:
  for b in i:
    if b[1] != 'O':
        print(b)

空间

import spacy
model_sp = en_core_web_lg.load()
for ent in model_sp(english_text).ents:
  print(ent.text.strip(), ent.label_)

天才

from flair.models import SequenceTagger
tagger = SequenceTagger.load('ner')
from flair.data import Sentence
s = Sentence(english_text)
tagger.predict(s)
for entity in s.get_spans('ner'):
    print(entity)

迪帕夫洛夫

from deeppavlov import configs, build_model
from deeppavlov import build_model, configs

ner_model = build_model(configs.ner.ner_ontonotes_bert, download=True)
result = ner_model([english_text])
for i in range(len(result[0][0])):
     if result [1][0][i] != 'O':
         print(result[0][0][i], result[1][0][i])

deepmipt / ner

import ner
example = russian_text
def deepmint_ner(text):
  extractor = ner.Extractor()
  for m in extractor(text):
     print(m)
deepmint_ner(example)

多种语言

from polyglot.text import Text
for ent in Text(english_text).entities:
 print(ent[0],ent.tag)

!polyglot download embeddings2.ru ner2.ru
for ent in Text(russian_text).entities:
 print(ent[0],ent.tag)

AdaptNLP

from adaptnlp import EasyTokenTagger
tagger = EasyTokenTagger()
sentences = tagger.tag_text(
    text = english_text, model_name_or_path = "ner-ontonotes"
)
spans = sentences[0].get_spans("ner")
for sen in sentences:
    for entity in sen.get_spans("ner"):
        print(entity)

节

import stanza
stanza.download('en')
def stanza_nlp(text):
  nlp = stanza.Pipeline(lang='en', processors='tokenize,ner')
  doc = nlp(text)
  print(*[f'entity: {ent.text}\ttype: {ent.type}' for sent in doc.sentences for ent in sent.ents], sep='\n')
stanza_nlp(english_text)

import stanza
stanza.download('ru')
def stanza_nlp_ru(text):
  nlp = stanza.Pipeline(lang='ru', processors='tokenize,ner')
  doc = nlp(text)
  print(*[f'entity: {ent.text}\ttype: {ent.type}' for sent in doc.sentences for ent in sent.ents], sep='\n')
stanza_nlp_ru(russian_text)

艾伦普

from allennlp.predictors.predictor import Predictor
import allennlp_models.ner.crf_tagger
predictor = Predictor.from_path("https://storage.googleapis.com/allennlp-public-models/ner-model-2020.02.10.tar.gz")
allen_result = predictor.predict(
  sentence=english_text
)
for i in zip(allen_result['tags'], allen_result['words']):
    if (i[0]) != 'O':
      print(i)

汉普

recognizer = hanlp.load(hanlp.pretrained.ner.MSRA_NER_BERT_BASE_ZH)
recognizer([list('上海华安工业（集团）公司董事长谭旭光和秘书张晚霞来到美国纽约现代艺术博物馆参观。'),
                list('萨哈夫说，伊拉克将同联合国销毁伊拉克大规模杀伤性武器特别委员会继续保持合作。')])

import hanlp
tokenizer = hanlp.utils.rules.tokenize_english
testing = tokenizer('Need SAP FICO consultant for support project needs to be work on 6 months on FI AREAWe')
recognizer = hanlp.load(hanlp.pretrained.ner.CONLL03_NER_BERT_BASE_UNCASED_EN)
recognizer(testing)

PullEnti

from pullenti_wrapper.processor import (
    Processor,
    MONEY,
    URI,
    PHONE,
    DATE,
    KEYWORD,
    DEFINITION,
    DENOMINATION,
    MEASURE,
    BANK,
    GEO,
    ADDRESS,
    ORGANIZATION,
    PERSON,
    MAIL,
    TRANSPORT,
    DECREE,
    INSTRUMENT,
    TITLEPAGE,
    BOOKLINK,
    BUSINESS,
    NAMEDENTITY,
    WEAPON,
)

processor = Processor([PERSON, ORGANIZATION, GEO, DATE, MONEY])
text = russian_text
result = processor(text)
result.graph

娜塔莎（Natasha）

from natasha import (
    Segmenter,
    MorphVocab,
    
    NewsEmbedding,
    NewsMorphTagger,
    NewsSyntaxParser,
    NewsNERTagger,
    
    PER,
    NamesExtractor,

    Doc
)

segmenter = Segmenter()
morph_vocab = MorphVocab()

emb = NewsEmbedding()
morph_tagger = NewsMorphTagger(emb)
syntax_parser = NewsSyntaxParser(emb)
ner_tagger = NewsNERTagger(emb)

names_extractor = NamesExtractor(morph_vocab)

doc = Doc(russian_text)

书呆子

from nerd import ner
doc_nerd_d = ner.name(english_text)
text_label = [(X.text, X.label_) for X in doc_nerd_d]
print(text_label)