混合相似度算法

import numpy as npfrom sklearn.feature_extraction.text import TfidfVectorizerfrom sklearn.decomposition import TruncatedSVDfrom sentence_transformers import SentenceTransformerfrom Levenshtein import ratio as levenshtein_ratiofrom phonetics import metaphoneimport torchimport torch.nn as nnclass HybridSimilarity(nn.Module):    def __init__(self):        super().__init__()        self.bert = SentenceTransformer('all-MiniLM-L6-v2')        self.tfidf = TfidfVectorizer()        self.attention = nn.MultiheadAttention(embed_dim=384, num_heads=4)        self.fc = nn.Sequential(            nn.Linear(1152, 256),            nn.ReLU(),            nn.LayerNorm(256),            nn.Linear(256, 1),            nn.Sigmoid()        )    def _extract_features(self, text1, text2):        # 多维度特征提取        features = {}        # 词汇相似度        features['levenshtein'] = levenshtein_ratio(text1, text2)        features['jaccard'] = len(set(text1.split()) & set(text2.split())) / len(set(text1.split()) | set(text2.split()))        # 语音相似度        features['metaphone'] = 1.0 if metaphone(text1) == metaphone(text2) else 0.0        # 语义嵌入 (BERT)        emb1 = self.bert.encode(text1, convert_to_tensor=True)        emb2 = self.bert.encode(text2, convert_to_tensor=True)        features['semantic_cosine'] = nn.CosineSimilarity()(emb1, emb2).item()        # 句法相似度 (LSA-TFIDF)        tfidf_matrix = self.tfidf.fit_transform([text1, text2])        svd = TruncatedSVD(n_components=1)        lsa = svd.fit_transform(tfidf_matrix)        features['lsa_cosine'] = np.dot(lsa[0], lsa[1].T)[0][0]        # 注意力机制        att_output, _ = self.attention(            emb1.unsqueeze(0).unsqueeze(0),            emb2.unsqueeze(0).unsqueeze(0),            emb2.unsqueeze(0).unsqueeze(0)        )        features['attention_score'] = att_output.mean().item()        return torch.tensor(list(features.values())).unsqueeze(0)    def forward(self, text1, text2):        features = self._extract_features(text1, text2)        return self.fc(features).item()def calculate_similarity(text1, text2):    model = HybridSimilarity()    return model(text1, text2)

self.bert = SentenceTransformer('all-MiniLM-L6-v2')self.tfidf = TfidfVectorizer()self.attention = nn.MultiheadAttention(embed_dim=384, num_heads=4)self.fc = nn.Sequential(    nn.Linear(1152, 256),    nn.ReLU(),    nn.LayerNorm(256),    nn.Linear(256, 1),    nn.Sigmoid())

text_a = "The quick brown fox jumps over the lazy dog"text_b = "A fast brown fox leaps over a sleepy hound"print(f"Similarity coefficient: {calculate_similarity(text_a, text_b):.4f}")