利用 Python 进行自然语言处理的 12 个实用案例

import re
import string

def preprocess_text(text):
    # 去除标点符号
    text = text.translate(str.maketrans('', '', string.punctuation))
    # 转换为小写
    text = text.lower()
    # 分词
    words = text.split()
    return words

text = "Hello, World! This is a test."
preprocessed_text = preprocess_text(text)
print(preprocessed_text)  # 输出: ['hello', 'world', 'this', 'is', 'a', 'test']

from nltk.stem import PorterStemmer, WordNetLemmatizer

stemmer = PorterStemmer()
lemmatizer = WordNetLemmatizer()

words = ["running", "jumps", "better", "worse"]

stemmed_words = [stemmer.stem(word) for word in words]
lemmatized_words = [lemmatizer.lemmatize(word) for word in words]

print(stemmed_words)  # 输出: ['run', 'jump', 'better', 'worst']
print(lemmatized_words)  # 输出: ['run', 'jump', 'better', 'worse']

from nltk.corpus import stopwords

stop_words = set(stopwords.words('english'))

def remove_stopwords(words):
    filtered_words = [word for word in words if word not in stop_words]
    return filtered_words

words = ["this", "is", "a", "test", "of", "stop", "words"]
filtered_words = remove_stopwords(words)
print(filtered_words)  # 输出: ['test', 'stop', 'words']

from collections import Counter

words = ["apple", "banana", "apple", "orange", "banana", "banana"]
word_counts = Counter(words)

print(word_counts)  # 输出: Counter({'banana': 3, 'apple': 2, 'orange': 1})

from sklearn.feature_extraction.text import TfidfVectorizer

documents = [
    "The cat in the hat.",
    "A cat is a fine pet.",
    "Dogs and cats make good pets."
]

vectorizer = TfidfVectorizer()
tfidf_matrix = vectorizer.fit_transform(documents)

print(tfidf_matrix.toarray())
# 输出: [[0.         0.57735027 0.57735027 0.57735027 0.         0.        ]
#        [0.57735027 0.57735027 0.57735027 0.         0.         0.        ]
#        [0.40824829 0.40824829 0.         0.         0.57735027 0.57735027]]from sklearn.feature_extraction.text import TfidfVectorizer

documents = [
    "The cat in the hat.",
    "A cat is a fine pet.",
    "Dogs and cats make good pets."
]

vectorizer = TfidfVectorizer()
tfidf_matrix = vectorizer.fit_transform(documents)

print(tfidf_matrix.toarray())
# 输出: [[0.         0.57735027 0.57735027 0.57735027 0.         0.        ]
#        [0.57735027 0.57735027 0.57735027 0.         0.         0.        ]
#        [0.40824829 0.40824829 0.         0.         0.57735027 0.57735027]]

import spacy

nlp = spacy.load("en_core_web_sm")

text = "Apple is looking at buying U.K. startup for $1 billion"
doc = nlp(text)

for ent in doc.ents:
    print(ent.text, ent.label_)

# 输出:
# Apple ORG
# U.K. GPE
# $1 billion MONEY

from textblob import TextBlob

text = "I love this movie!"
blob = TextBlob(text)

sentiment = blob.sentiment.polarity
print(sentiment)  # 输出: 0.875

from sklearn.datasets import fetch_20newsgroups
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.pipeline import Pipeline
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report

# 加载数据集
data = fetch_20newsgroups(subset='all')

# 划分训练集和测试集
X_train, X_test, y_train, y_test = train_test_split(data.data, data.target, test_size=0.2, random_state=42)

# 创建管道
pipeline = Pipeline([
    ('vectorizer', CountVectorizer()),
    ('classifier', MultinomialNB())
])

# 训练模型
pipeline.fit(X_train, y_train)

# 预测
y_pred = pipeline.predict(X_test)

# 评估
print(classification_report(y_test, y_pred, target_names=data.target_names))

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.cluster import KMeans

documents = [
    "The cat in the hat.",
    "A cat is a fine pet.",
    "Dogs and cats make good pets.",
    "I love my pet dog.",
    "My dog loves to play with cats."
]

vectorizer = TfidfVectorizer()
tfidf_matrix = vectorizer.fit_transform(documents)

kmeans = KMeans(n_clusters=2)
kmeans.fit(tfidf_matrix)

labels = kmeans.labels_
print(labels)  # 输出: [0 0 1 1 1]

from transformers import MarianMTModel, MarianTokenizer

model_name = "Helsinki-NLP/opus-mt-en-de"
tokenizer = MarianTokenizer.from_pretrained(model_name)
model = MarianMTModel.from_pretrained(model_name)

text = "I love this movie!"
translated = model.generate(**tokenizer(text, return_tensors="pt", padding=True))

translated_text = [tokenizer.decode(t, skip_special_tokens=True) for t in translated]
print(translated_text)  # 输出: ['Ich liebe diesen Film!']

from transformers import pipeline

qa_pipeline = pipeline("question-answering", model="distilbert-base-cased-distilled-squad")

context = "The Transformer is a deep learning model introduced in 2017 by Vaswani et al. that revolutionized natural language processing."
question = "What year was the Transformer model introduced?"

answer = qa_pipeline(question=question, context=context)
print(answer['answer'])  # 输出: 2017

from transformers import GPT2LMHeadModel, GPT2Tokenizer

model_name = "gpt2"
tokenizer = GPT2Tokenizer.from_pretrained(model_name)
model = GPT2LMHeadModel.from_pretrained(model_name)

prompt = "Once upon a time"
input_ids = tokenizer.encode(prompt, return_tensors="pt")

output = model.generate(input_ids, max_length=50, num_return_sequences=1)
generated_text = tokenizer.decode(output[0], skip_special_tokens=True)

print(generated_text)
# 输出: Once upon a time in a land far away, there lived a brave knight who set out on a quest to find a magical treasure.

from sklearn.datasets import fetch_20newsgroups

# 加载数据集
data = fetch_20newsgroups(subset='all', categories=['alt.atheism', 'soc.religion.christian'])

# 将数据划分为训练集和测试集
X_train, X_test, y_train, y_test = train_test_split(data.data, data.target, test_size=0.2, random_state=42)

from sklearn.feature_extraction.text import CountVectorizer

vectorizer = CountVectorizer()
X_train_vectorized = vectorizer.fit_transform(X_train)
X_test_vectorized = vectorizer.transform(X_test)

from sklearn.naive_bayes import MultinomialNB

model = MultinomialNB()
model.fit(X_train_vectorized, y_train)

from sklearn.metrics import classification_report

y_pred = model.predict(X_test_vectorized)
print(classification_report(y_test, y_pred, target_names=data.target_names))

def predict_spam(email_text):
    email_vectorized = vectorizer.transform([email_text])
    prediction = model.predict(email_vectorized)
    return "Spam" if prediction[0] == 1 else "Not Spam"

email_text = "Get rich quick! Click here to win a million dollars!"
print(predict_spam(email_text))  # 输出: Spam