import json from DrissionPage import Chromium from DrissionPage import ChromiumOptions from DrissionPage.errors import ElementNotFoundError from log.log_manager import logger def get_content_from_meta(metas, itemprop): content = None for meta in metas: if meta.attr('itemprop') == itemprop: content = meta.attr('content') return content class Zhihu: def __init__(self): co = ChromiumOptions() self.browser = Chromium() self.tab = None def get_content(self, url): """获取话题内容数据""" global title, keywords, date_created, date_modified, follower_count, comment_count, answer_count, topic_description contents_result = [] try: self.tab = self.browser.new_tab() # 访问话题/问题页面 self.tab.get(url) for _ in range(10): # 等待内容加载 self.tab.wait.ele_displayed('.List-item') self.tab.wait(3) # 向下滚动页面,直到所有内容加载完成 self.tab.scroll.to_bottom() self.tab.wait(1) self.tab.scroll.up(100) # 获取话题/问题相关信息:话题内容、keywards、话题创建日期dateCreated、话题修改日期dateModified、回答数量answerCount、评论数量commentCount question_page = self.tab.ele('.QuestionPage') # 获取话题属性,为QuestionPage的前9个meta标签 metas = question_page.eles('tag:meta')[0:9] # print(metas) title = get_content_from_meta(metas, 'name') answer_count = get_content_from_meta(metas, 'answerCount') comment_count = get_content_from_meta(metas, 'commentCount') keywords = get_content_from_meta(metas, 'keywords') date_created = get_content_from_meta(metas, 'dateCreated') date_modified = get_content_from_meta(metas, 'dateModified') follower_count = get_content_from_meta(metas, 'zhihu:followerCount') # print(date_created, date_modified, answer_count, comment_count, keywords) topic_description = "" try: topic_description = question_page.ele('.RichText ztext css-ob6uua').text except ElementNotFoundError as e: logger.error(f"元素缺失:不存在topic_description") # 获取所有内容条目 content_items = self.tab.ele('.Question-mainColumn').eles('.List-item') total_characters = 0 for item in content_items: try: content = item.ele('.RichContent-inner').text # 计算content的字数 content_len = len(content) print(content_len) if content_len > 1000 or content_len < 100: logger.error(f"skip本条内容,内容长度:{content_len}") continue if total_characters > 5000: logger.error(f"contents_result长度超过5000,跳出循环") break total_characters += content_len contents_result.append(content) # 打印contents_result的长度 logger.info(f"contents_result长度:{len(contents_result)}") except ElementNotFoundError as e: logger.error(f"元素缺失:{str(e)}") except ValueError as e: logger.error(f"热度值转换失败:{str(e)}") except ElementNotFoundError as e: logger.error(f"热榜容器元素未找到:{str(e)}") except Exception as e: logger.error(f"获取热榜数据异常:{str(e)}") finally: if self.tab: self.tab.close() # 返回json格式的数据 return json.dumps({ 'title': title, 'answer_count': answer_count, 'comment_count': comment_count, 'topic_description': topic_description, 'keywords': keywords, 'date_created': date_created, 'date_modified': date_modified, 'follower_count': follower_count, 'contents': contents_result }, ensure_ascii=False) if __name__ == '__main__': # 测试用例 logger.info('知乎采集测试') # 执行采集任务 zhihu = Zhihu() result = zhihu.get_content('https://www.zhihu.com/question/588507809') print(len(result)) print(result) logger.info('测试完成')