提升职业技能：了解并掌握这十大NLP技术

import nltk 
from nltk.sentiment import SentimentIntensityAnalyzer 

# 初始化情感分析器
sia = SentimentIntensityAnalyzer() 

# 定义要分析的文本
text = "非常棒，小孩子很喜欢" 

# 获取情感分数
sentiment_score = sia.polarity_scores(text) 

# 打印情感得分
print (sentiment_score) 

# 将情绪分类为正面、负面或中性
ifentiment_score [ 'compound' ] > 0.5 : 
    print ( "Positiveentiment" ) 
elif情感_score[ 'compound'] < - 0.5 : 
    print ( "Negative sentiment" ) 
else : 
    print ( "Neutral sentiment" )

import spacy

# Load the pre-trained model
nlp = spacy.load("en_core_web_sm")

# Define text to be analyzed
text = "Barack Obama visited the White House today"

# Process the text with the model
doc = nlp(text)

# Extract named entities
for ent in doc.ents:
    print(ent.text, ent.label_)

Barack Obama PERSON
White House FAC

from sklearn.datasets import fetch_20newsgroups
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import accuracy_score

# Load the 20 Newsgroups dataset
newsgroups_train = fetch_20newsgroups(subset='train')
newsgroups_test = fetch_20newsgroups(subset='test')

# Transform the texts into TF-IDF vectors
vectorizer = TfidfVectorizer()
X_train = vectorizer.fit_transform(newsgroups_train.data)
X_test = vectorizer.transform(newsgroups_test.data)
y_train = newsgroups_train.target
y_test = newsgroups_test.target

# Train a Multinomial Naive Bayes classifier
clf = MultinomialNB()
clf.fit(X_train, y_train)

# Predict the newsgroup of the test texts
y_pred = clf.predict(X_test)

# Evaluate the classifier's accuracy
accuracy = accuracy_score(y_test, y_pred)
print("Accuracy: {:.2f}%".format(accuracy * 100))

from opennmt import tokenizers
from opennmt import models
import torch

# Tokenize the source and target text.
source_tokenizer = tokenizers.new("text", "en")
source_text = "Hello, how are you?"
source_tokens = source_tokenizer.tokenize(source_text)

target_tokenizer = tokenizers.new("text", "fr")
target_text = "Bonjour, comment vas-tu?"
target_tokens = target_tokenizer.tokenize(target_text)

# Build the translation model.
model = models.Transformer(
    source_vocab_size=len(source_tokenizer.vocab),
    target_vocab_size=len(target_tokenizer.vocab),
    num_layers=6,
    hidden_size=512,
    dropout=0.1,
    attention_dropout=0.1,
    relu_dropout=0.1)
model.eval()

# Convert the tokens to a tensor.
source_tokens = torch.tensor(source_tokenizer.encode(source_text)).unsqueeze(0)

# Generate a translation.
with torch.no_grad():
    log_probs, _, _ = model(source_tokens, None, None)
    tokens = log_probs.argmax(-1)

# Decode the translation.
translation = target_tokenizer.decode(tokens[0])
print(translation)

from gensim.summarization import summarize

text = "In computing, stop words are words which are filtered out before or after processing of text. Though stop words usually refer to the most common words in a language, there is no single universal list of stop words used by all natural language processing tools, and indeed not all tools even use such a list. Some tools specifically avoid removing these stop words to support phrase search."

print(summarize(text, ratio=0.2))

import nltk 
from nltk import word_tokenize, pos_tag, ne_chunk 

# 示例文本
text = "Barack Obama 是美国第 44 任总统，任期从 2009 年到 2017 年。" 

# 对文本进行分词
tokens = word_tokenize(text) 

# POS 标记
tagged_tokens = pos_tag(tokens) 

# 命名实体识别
Entity = ne_chunk(tagged_tokens) 
print(entities)

(S
  (PERSON Barack/NNP)
  Obama/NNP
  was/VBD
  the/DT
  44th/JJ
  (ORGANIZATION President/NNP)
  of/IN
  the/DT
  (GPE United/NNP States/NNPS)
  ,/,
  serving/VBG
  from/IN
  2009/CD
  to/TO
  2017/CD
  ./.)

from transformers import GPT2Tokenizer, GPT2LMHeadModel

# Load the GPT-2 model and tokenizer
tokenizer = GPT2Tokenizer.from_pretrained("gpt2")
model = GPT2LMHeadModel.from_pretrained("gpt2")

# Define the prompt and generate text
prompt = "Once upon a time in a land far, far away"
generated_text = model.generate(input_ids=tokenizer.encode(prompt))

# Decode the generated text
generated_text = tokenizer.decode(generated_text)
print(generated_text)

import nltk
from nltk import word_tokenize, pos_tag, ne_chunk

# Sample text
text = "Barack Obama was the 44th President of the United States, serving from 2009 to 2017."

# Tokenize the text
tokens = word_tokenize(text)

# POS tagging
tagged_tokens = pos_tag(tokens)

# Named entity recognition
entities = ne_chunk(tagged_tokens)
print(entities)

import speech_recognition as sr

# create a recognizer object
r = sr.Recognizer()

# create a microphone object
mic = sr.Microphone()

# listen for speech and transcribe it
with mic as source:
    r.adjust_for_ambient_noise(source)
    audio = r.listen(source)
    transcribed_text = r.recognize_google(audio)
    print(transcribed_text)

with sr.AudioFile('audio_file.wav') as source:
    audio = r.record(source)
    transcribed_text = r.recognize_google(audio)
    print(transcribed_text)

from gtts import gTTS
import os

text = "Hello, this is an example of text to speech using the gTTS library in Python."

# Language in which you want to convert
language = 'en'

# Passing the text and language to the engine, 
# here we have marked slow=False. Which tells 
# the module that the converted audio should 
# have a high speed
myobj = gTTS(text=text, lang=language, slow=False)

# Saving the converted audio in a mp3 file named
# welcome 
myobj.save("welcome.mp3")

# Playing the converted file
os.system("mpg321 welcome.mp3")