适用于TTS语音处理项目的召回模型：词袋模型究竟是个啥？和语义召回相比有什么优劣？ 原创

[‘also’, ‘football’, ‘games’, ‘john’, ‘likes’, ‘mary’, ‘movies’, ‘to’, ‘too’, ‘watch’]

import jieba, re
cut_all = False


def cutWords(s, cut_all=False):
    cut_words = []
    # 将所有字母、数字、冒号、逗号、句号等过滤掉
    s = re.sub("[A-Za-z0-9\：\·\—\，\。\“ \”]", "", s)
    seg_list = jieba.cut(s, cut_all=cut_all)
    cut_words.append("/".join(seg_list))

import re
from collections import Counter
import jieba


def wordsCount(s):
    # 初始化空字符串用于存储分词结果
    cut_words = ""
    
    # 使用正则表达式移除非中文字符
    s = re.sub("[A-Za-z0-9\：\·\—\，\。\“ \”]", "", s)
    
    # 使用jieba进行分词，并连接结果
    seg_list = jieba.cut(s, cut_all=False)
    cut_words += (" ".join(seg_list))
    
    # 将分词结果分割成列表
    all_words = cut_words.split()
    
    # 初始化计数器
    c = Counter()
    
    # 遍历所有单词，长度大于1且不是换行符则进行计数
    for x in all_words:
        if len(x) > 1 and x != '\r\n':
            c[x] += 1
    
    # 创建一个词汇列表
    vocab = []
    
    print('\n词频统计结果：')
    # 打印并收集出现频率最高的前5个词
    for (k, v) in c.most_common(5): 
        print("%s:%d" % (k, v))
        vocab.append(k)
    
    # 打印词汇列表
    print("词表：", vocab)

def vectorization_with_freq(s):
    # 此处接文本分词和词频统计代码，需要生成词表vocab和文本x_text
    
    # 初始化空列表用于存储向量化结果
    x_vec = []
    
    # 遍历文本中的每项
    for item in x_text:
        # 对词表中的每个词创建一个字典，初始化频率为0
        tmp = dict(zip(vocab, [0] * len(vocab)))
        
        # 遍历此项中的每个词
        for w in item:
            # 如果词在词表中，则频率加1
            if w in vocab:
                tmp[w] += 1
                
        # 将该项的向量添加到向量化结果列表
        x_vec.append(list(tmp.values()))
    
    # 打印词表、文本和向量化结果
    print("词表：", vocab)
    print("文本：", x_text)
    print(x_vec)

from sklearn.feature_extraction.text import CountVectorizer
corpus = [
    "John likes to watch movies, Mary likes movies too",
    "John also likes to watch football games",
]
bag_of_words = CountVectorizer(
            ngram_range=(1, 2), token_pattern="(?u)\\b\\w+\\b", binary=True
        ).fit(corpus)
vectors = bag_of_words.transform(corpus)
print(vectors)


"""
 (0, 5)	1
  (0, 7)	1
  (0, 8)	1
  (0, 9)	1
  (0, 10)	1
  (0, 11)	1
  (0, 12)	1
  (0, 13)	1
  (0, 14)	1
  (0, 15)	1
  (0, 16)	1
  (0, 17)	1
  (0, 18)	1
  (0, 19)	1
  (0, 21)	1
  (1, 0)	1
  (1, 1)	1
  (1, 2)	1
  (1, 3)	1
  (1, 4)	1
  (1, 5)	1
  (1, 6)	1
  (1, 8)	1
  (1, 10)	1
  (1, 16)	1
  (1, 17)	1
  (1, 19)	1
  (1, 20)	1
"""

import numpy as np
from nltk.stem import PorterStemmer
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.metrics.pairwise import check_pairwise_arrays, normalize, safe_sparse_dot




stemmer = PorterStemmer()




class Vectorizer:
    @staticmethod
    def stem(text):
        # 词干化
        words = text.split()
        stemmed_words = [stemmer.stem(word) for word in words]
        sentence = " ".join(stemmed_words)
        sentence = sentence
        return sentence


    def __init__(self, corpus):
        self.bag_of_words = CountVectorizer(
            ngram_range=(1, 2), token_pattern="(?u)\\b\\w+\\b", binary=True
        ).fit(corpus)
        self.vectors = self.bag_of_words.transform(corpus)
        self.corpus = corpus


    @staticmethod
    def cosine_similarity(X, Y=None, dense_output=True):
        X, Y = check_pairwise_arrays(X, Y)
        X_normalized = normalize(X, copy=True)
        if X is Y:
            Y_normalized = X_normalized
        else:
            Y_normalized = normalize(Y, copy=True)
        return safe_sparse_dot(
            X_normalized,
            Y_normalized.T,
            dense_output=dense_output)


    def get_relevent_docs(self, query, top_k=3, theshold=0):
        stem_query = self.stem(query)
        query_vectors = self.bag_of_words.transform([stem_query])
        cos_sim = self.cosine_similarity(self.vectors, query_vectors)
        indices = np.argpartition(cos_sim[:, 0], -top_k)[-top_k:]
        result = []
        for i in range(len(indices) - 1, -1, -1):
            idx = indices[i]
            score = cos_sim[idx, 0]
            if score > theshold:
                result.append([score, idx])
        result.sort(key=lambda x: x[0], reverse=True)  # 相似度降序排序后返回


        return result