AI论文查重源码深度解析与开发指南

import re import jieba import hashlib from typing import List, Dict, Tuple class TextPreprocessor: def __init__(self): self.stopwords = self.load_stopwords() def load_stopwords(self) -> set: """加载停用词表""" with open('stopwords.txt', 'r', encoding='utf-8') as f: return set(line.strip() for line in f) def normalize_text(self, text: str) -> str: """文本标准化处理""" # 去除多余空白字符 text = re.sub(r'\s+', ' ', text) # 统一标点符号 text = re.sub(r'[，。！？；：]', lambda m: {'，': ',', '。': '.', '！': '!', '？': '?', '；': ';', '：': ':'}[m.group()], text) # 移除特殊字符但保留中英文和数字 text = re.sub(r'[^\u4e00-\u9fa5a-zA-Z0-9\s\.,!?;:]', '', text) return text.strip().lower() def segment_chinese(self, text: str) -> List[str]: """中文分词处理""" words = jieba.cut(text) return [word for word in words if word not in self.stopwords and len(word) > 1] def extract_features(self, text: str) -> Dict: """提取文本特征""" normalized = self.normalize_text(text) words = self.segment_chinese(normalized) features = { 'char_count': len(text), 'word_count': len(words), 'unique_words': len(set(words)), 'hash_signature': hashlib.md5(text.encode()).hexdigest(), 'word_freq': self.calculate_word_frequency(words) } return features def calculate_word_frequency(self, words: List[str]) -> Dict[str, int]: """计算词频""" freq = {} for word in words: freq[word] = freq.get(word, 0) + 1 return freq

import numpy as np from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.metrics.pairwise import cosine_similarity from gensim.models import Word2Vec import jieba.posseg as pseg class SimilarityCalculator: def __init__(self): self.tfidf_vectorizer = TfidfVectorizer( max_features=5000, ngram_range=(1, 2), stop_words=None ) self.word2vec_model = None def build_tfidf_index(self, documents: List[str]): """构建TF-IDF索引""" return self.tfidf_vectorizer.fit_transform(documents) def tfidf_similarity(self, doc1: str, doc2: str) -> float: """基于TF-IDF的余弦相似度计算""" try: tfidf_matrix = self.tfidf_vectorizer.transform([doc1, doc2]) similarity = cosine_similarity(tfidf_matrix[0:1], tfidf_matrix[1:2])[0][0] return round(similarity, 4) except Exception as e: print(f"TF-IDF计算错误: {e}") return 0.0 def jaccard_similarity(self, set1: set, set2: set) -> float: """Jaccard相似度计算""" intersection = len(set1.intersection(set2)) union = len(set1.union(set2)) return intersection / union if union != 0 else 0.0 def longest_common_subsequence(self, text1: str, text2: str) -> int: """最长公共子序列长度计算""" m, n = len(text1), len(text2) dp = [[0] * (n + 1) for _ in range(m + 1)] for i in range(1, m + 1): for j in range(1, n + 1): if text1[i-1] == text2[j-1]: dp[i][j] = dp[i-1][j-1] + 1 else: dp[i][j] = max(dp[i-1][j], dp[i][j-1]) return dp[m][n] def calculate_comprehensive_similarity(self, text1: str, text2: str) -> Dict: """综合相似度计算""" # 字符级LCS相似度 lcs_len = self.longest_common_subsequence(text1, text2) char_sim = lcs_len / max(len(text1), len(text2)) if max(len(text1), len(text2)) > 0 else 0 # 词汇级Jaccard相似度 words1 = set(jieba.lcut(text1)) words2 = set(jieba.lcut(text2)) word_sim = self.jaccard_similarity(words1, words2) # TF-IDF相似度 tfidf_sim = self.tfidf_similarity(text1, text2) # 加权综合评分 comprehensive_score = ( 0.3 * char_sim + 0.3 * word_sim + 0.4 * tfidf_sim ) return { 'character_similarity': round(char_sim, 4), 'word_similarity': round(word_sim, 4), 'tfidf_similarity': round(tfidf_sim, 4), 'comprehensive_similarity': round(comprehensive_score, 4) }

import torch from transformers import AutoTokenizer, AutoModel import torch.nn.functional as F class SemanticSimilarityModel: def __init__(self, model_name: str = "paraphrase-multilingual-MiniLM-L12-v2"): self.tokenizer = AutoTokenizer.from_pretrained(model_name) self.model = AutoModel.from_pretrained(model_name) self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') self.model.to(self.device) def mean_pooling(self, model_output, attention_mask): """均值池化获取句子嵌入""" token_embeddings = model_output[0] input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float() return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9) def get_sentence_embedding(self, sentences: List[str]) -> torch.Tensor: """获取句子向量表示""" encoded_input = self.tokenizer( sentences, padding=True, truncation=True, max_length=512, return_tensors='pt' ).to(self.device) with torch.no_grad(): model_output = self.model(**encoded_input) sentence_embeddings = self.mean_pooling(model_output, encoded_input['attention_mask']) sentence_embeddings = F.normalize(sentence_embeddings, p=2, dim=1) return sentence_embeddings def semantic_similarity(self, text1: str, text2: str) -> float: """计算语义相似度""" embeddings = self.get_sentence_embedding([text1, text2]) similarity = cosine_similarity(embeddings[0:1], embeddings[1:2])[0][0] return round(float(similarity), 4) # 使用示例 similarity_model = SemanticSimilarityModel() semantic_score = similarity_model.semantic_similarity( "人工智能技术在教育领域的应用研究", "AI技术在教育教学中的应用探讨" ) print(f"语义相似度: {semantic_score}")