解锁搜索的力量：关键词、相似性和语义解释

Python 
 import os
 import re
 from whoosh.index import create_in
 from whoosh.fields import Schema, TEXT
 from whoosh.qparser import QueryParser
 from sklearn.feature_extraction.text import TfidfVectorizer
 from sklearn.metrics.pairwise import cosine_similarity
 from transformers import pipeline
 import numpy as np

Python 
 # Sample documents used for demonstrating all three search techniques
 documents = [
 "Machine learning is a field of artificial intelligence that uses statistical techniques.",
 "Natural language processing (NLP) is a part of artificial intelligence that deals with the interaction between computers and humans using natural language. ",
 "Deep learning models are a subset of machine learning algorithms that use neural networks with many layers.",
 "AI is transforming the world by automating tasks, providing insights through data analysis, and enabling new technologies like autonomous vehicles and advanced robotics. ",
 "Natural language processing can be challenging due to the complexity and variability of human language. ",
 "The application of machine learning in healthcare is revolutionizing the way diseases are diagnosed and treated.",
 "Autonomous vehicles rely heavily on AI and machine learning to navigate and make decisions.",
 "Speech recognition technology has advanced considerably thanks to deep learning models. "
 ]

Python 

 def highlight_term(text, term)：
 return re.sub(f"({term})", r'\033[1;31m\1\033[0m', text, flags=re.IGNORECASE)

Python 
 # Function for Keyword Search using Whoosh
 def keyword_search(query_str)：
 schema = Schema(content=TEXT(stored=True))
 if not os.path.exists("index")：
 os.mkdir("index")
 index = create_in("index", schema)
 writer = index.writer()
 for doc in documents：
 writer.add_document(content=doc)
 writer.commit()

 with index.searcher() as searcher：
 query = QueryParser("content", index.schema).parse(query_str)
 results = searcher.search(query)
 highlighted_results = [(highlight_term(result['content'], query_str), result.score) for result in results]
 return highlighted_results

Python 
 # Function for Similarity Search using Scikit-learn
 def similarity_search(query_str):
 vectorizer = TfidfVectorizer()
 tfidf_matrix = vectorizer.fit_transform(documents)
 query_vec = vectorizer.transform([query_str])
 similarity = cosine_similarity(query_vec, tfidf_matrix)
 similar_docs = similarity[0].argsort()[-3:][::-1] # Top 3 similar documents
 similarities = similarity[0][similar_docs]
 highlighted_results = [(highlight_term(documents[i], query_str), similarities[idx]) for idx, i in enumerate(similar_docs)]
 return highlighted_results

Python 
 # Function for Semantic Search using Transformers

def semantic_search(query_str)：
 semantic_searcher = pipeline("feature-extraction", model="distilbert-base-uncased")
 query_embedding = semantic_searcher(query_str)[0][0]
 
 def get_similarity(query_embedding, doc_embedding)：
 return cosine_similarity([query_embedding], [doc_embedding])[0][0]
 
 doc_embeddings = [semantic_searcher(doc)[0][0] for doc in documents]
 similarities = [get_similarity(query_embedding, embedding) for embedding in doc_embeddings]
 sorted_indices = np.argsort(similarities)[-3：][：：-1]
 highlighted_results = [(highlight_term(documents[i], query_str), similarities[i]) for i in sorted_indices]
 return highlighted_results

Python 
 # Main execution
 if __name__ == "__main__"：
 query = input("Enter your search term： ")

 print("\nKeyword Search Results：")
 keyword_results = keyword_search(query)
 for result, score in keyword_results：
 print(f"{result} (Score： {score：.2f})")
 
 print("\nSimilarity Search Results：")
 similarity_results = similarity_search(query)
 for result, similarity in similarity_results：
 print(f"{result} (Similarity： {similarity * 100：.2f}%)")
 
 print("\nSemantic Search Results：")
 semantic_results = semantic_search(query)
 for result, similarity in semantic_results：
 print(f"{result} (Similarity： {similarity * 100：.2f}%)")