如何使用Python for NLP处理含有重复文本的PDF文件？

pip install PyPDF2pip install textract

import PyPDF2def read_pdf(filename):    with open(filename, 'rb') as file:        pdf = PyPDF2.PdfFileReader(file)        text = ""        for page_num in range(pdf.getNumPages()):            page = pdf.getPage(page_num)            text += page.extractText()    return text# 调用函数读取PDF文件pdf_text = read_pdf('example.pdf')print(pdf_text)

import nltkfrom nltk.corpus import stopwordsfrom nltk.tokenize import sent_tokenizefrom sklearn.feature_extraction.text import TfidfVectorizerfrom sklearn.metrics.pairwise import cosine_similaritydef preprocess_text(text):    # 分词并删除停用词    tokens = nltk.word_tokenize(text)    stop_words = set(stopwords.words("english"))    filtered_tokens = [word.lower() for word in tokens if word.lower() not in stop_words and word.isalpha()]    return ' '.join(filtered_tokens)def remove_duplicate(text):    # 分成句子    sentences = sent_tokenize(text)    # 提取句子的特征向量    vectorizer = TfidfVectorizer()    sentence_vectors = vectorizer.fit_transform(sentences).toarray()    # 计算余弦相似度矩阵    similarity_matrix = cosine_similarity(sentence_vectors, sentence_vectors)    # 标记重复文本    marked_duplicates = set()    for i in range(len(similarity_matrix)):        for j in range(i+1, len(similarity_matrix)):            if similarity_matrix[i][j] > 0.9:                marked_duplicates.add(j)    # 去除重复文本    filtered_text = [sentences[i] for i in range(len(sentences)) if i not in marked_duplicates]    return ' '.join(filtered_text)# 预处理文本preprocessed_text = preprocess_text(pdf_text)# 去除重复文本filtered_text = remove_duplicate(preprocessed_text)print(filtered_text)