Compare o trabalho do Python de código aberto - bibliotecas para o reconhecimento de entidades nomeadas

Introdução

Entidades nomeadas reconhecidas

Desigualdade linguística

Texto a ser processado

english_text = ''' I want a person available 7 days and with prompt response all most every time. Only Indian freelancer need I need PHP developer who have strong experience in Laravel and Codeigniter framework for daily 4 hours. I need this work by Monday 27th Jan. should be free from plagiarism . 
Need SAP FICO consultant for support project needs to be work on 6 months on FI AREAWe.  Want a same site to be created as the same as this https://www.facebook.com/?ref=logo, please check the site before contacting to me and i want this site to be ready in 10 days. They will be ready at noon tomorrow .'''

russian_text = '''   110     ,     .        https://www.sobyanin.ru/  , 1 .     .51 (   :  , )  ?     2107   47     24,    . 
 c        10  1970 ,     -, . ,  5/1 8 000 ( )  00  .               .              - .'''

Biblioteca NLTK

import nltk
nltk.download('punkt')
nltk.download('averaged_perceptron_tagger')
nltk.download('maxent_ne_chunker')
nltk.download('words')
for sent in nltk.sent_tokenize(english_text):
   for chunk in nltk.ne_chunk(nltk.pos_tag(nltk.word_tokenize(sent))):
      if hasattr(chunk, 'label'):
         print(chunk)

Stanford CoreNLP

from nltk.tag.stanford import StanfordNERTagger
jar = "stanford-ner-2015-04-20/stanford-ner-3.5.2.jar"
model = "stanford-ner-2015-04-20/classifiers/" 
st_3class = StanfordNERTagger(model + "english.all.3class.distsim.crf.ser.gz", jar, encoding='utf8') 
st_4class = StanfordNERTagger(model + "english.conll.4class.distsim.crf.ser.gz", jar, encoding='utf8') 
st_7class = StanfordNERTagger(model + "english.muc.7class.distsim.crf.ser.gz", jar, encoding='utf8')
for i in [st_3class.tag(english_text.split()), st_4class.tag(english_text.split()), st_7class.tag(english_text.split())]:
  for b in i:
    if b[1] != 'O':
        print(b)

Spacy

import spacy
model_sp = en_core_web_lg.load()
for ent in model_sp(english_text).ents:
  print(ent.text.strip(), ent.label_)

Flair

from flair.models import SequenceTagger
tagger = SequenceTagger.load('ner')
from flair.data import Sentence
s = Sentence(english_text)
tagger.predict(s)
for entity in s.get_spans('ner'):
    print(entity)

Deeppavlov

from deeppavlov import configs, build_model
from deeppavlov import build_model, configs

ner_model = build_model(configs.ner.ner_ontonotes_bert, download=True)
result = ner_model([english_text])
for i in range(len(result[0][0])):
     if result [1][0][i] != 'O':
         print(result[0][0][i], result[1][0][i])

deepmipt / ner

import ner
example = russian_text
def deepmint_ner(text):
  extractor = ner.Extractor()
  for m in extractor(text):
     print(m)
deepmint_ner(example)

Poliglota

from polyglot.text import Text
for ent in Text(english_text).entities:
 print(ent[0],ent.tag)

!polyglot download embeddings2.ru ner2.ru
for ent in Text(russian_text).entities:
 print(ent[0],ent.tag)

AdaptNLP

from adaptnlp import EasyTokenTagger
tagger = EasyTokenTagger()
sentences = tagger.tag_text(
    text = english_text, model_name_or_path = "ner-ontonotes"
)
spans = sentences[0].get_spans("ner")
for sen in sentences:
    for entity in sen.get_spans("ner"):
        print(entity)

Estrofe

import stanza
stanza.download('en')
def stanza_nlp(text):
  nlp = stanza.Pipeline(lang='en', processors='tokenize,ner')
  doc = nlp(text)
  print(*[f'entity: {ent.text}\ttype: {ent.type}' for sent in doc.sentences for ent in sent.ents], sep='\n')
stanza_nlp(english_text)

import stanza
stanza.download('ru')
def stanza_nlp_ru(text):
  nlp = stanza.Pipeline(lang='ru', processors='tokenize,ner')
  doc = nlp(text)
  print(*[f'entity: {ent.text}\ttype: {ent.type}' for sent in doc.sentences for ent in sent.ents], sep='\n')
stanza_nlp_ru(russian_text)

Allennlp

from allennlp.predictors.predictor import Predictor
import allennlp_models.ner.crf_tagger
predictor = Predictor.from_path("https://storage.googleapis.com/allennlp-public-models/ner-model-2020.02.10.tar.gz")
allen_result = predictor.predict(
  sentence=english_text
)
for i in zip(allen_result['tags'], allen_result['words']):
    if (i[0]) != 'O':
      print(i)

Hanlp

recognizer = hanlp.load(hanlp.pretrained.ner.MSRA_NER_BERT_BASE_ZH)
recognizer([list('上海华安工业（集团）公司董事长谭旭光和秘书张晚霞来到美国纽约现代艺术博物馆参观。'),
                list('萨哈夫说，伊拉克将同联合国销毁伊拉克大规模杀伤性武器特别委员会继续保持合作。')])

import hanlp
tokenizer = hanlp.utils.rules.tokenize_english
testing = tokenizer('Need SAP FICO consultant for support project needs to be work on 6 months on FI AREAWe')
recognizer = hanlp.load(hanlp.pretrained.ner.CONLL03_NER_BERT_BASE_UNCASED_EN)
recognizer(testing)

PullEnti

from pullenti_wrapper.processor import (
    Processor,
    MONEY,
    URI,
    PHONE,
    DATE,
    KEYWORD,
    DEFINITION,
    DENOMINATION,
    MEASURE,
    BANK,
    GEO,
    ADDRESS,
    ORGANIZATION,
    PERSON,
    MAIL,
    TRANSPORT,
    DECREE,
    INSTRUMENT,
    TITLEPAGE,
    BOOKLINK,
    BUSINESS,
    NAMEDENTITY,
    WEAPON,
)

processor = Processor([PERSON, ORGANIZATION, GEO, DATE, MONEY])
text = russian_text
result = processor(text)
result.graph

Natasha

from natasha import (
    Segmenter,
    MorphVocab,
    
    NewsEmbedding,
    NewsMorphTagger,
    NewsSyntaxParser,
    NewsNERTagger,
    
    PER,
    NamesExtractor,

    Doc
)

segmenter = Segmenter()
morph_vocab = MorphVocab()

emb = NewsEmbedding()
morph_tagger = NewsMorphTagger(emb)
syntax_parser = NewsSyntaxParser(emb)
ner_tagger = NewsNERTagger(emb)

names_extractor = NamesExtractor(morph_vocab)

doc = Doc(russian_text)

ner-d

from nerd import ner
doc_nerd_d = ner.name(english_text)
text_label = [(X.text, X.label_) for X in doc_nerd_d]
print(text_label)