117 lines
4.7 KiB
Python
117 lines
4.7 KiB
Python
import json
|
||
|
||
from DrissionPage import Chromium
|
||
from DrissionPage import ChromiumOptions
|
||
from DrissionPage.errors import ElementNotFoundError
|
||
|
||
from log.log_manager import logger
|
||
|
||
|
||
def get_content_from_meta(metas, itemprop):
|
||
content = None
|
||
for meta in metas:
|
||
if meta.attr('itemprop') == itemprop:
|
||
content = meta.attr('content')
|
||
return content
|
||
|
||
|
||
class Zhihu:
|
||
def __init__(self):
|
||
co = ChromiumOptions()
|
||
self.browser = Chromium()
|
||
self.tab = None
|
||
|
||
def get_content(self, url):
|
||
"""获取话题内容数据"""
|
||
global title, keywords, date_created, date_modified, follower_count, comment_count, answer_count, topic_description
|
||
contents_result = []
|
||
try:
|
||
self.tab = self.browser.new_tab()
|
||
# 访问话题/问题页面
|
||
self.tab.get(url)
|
||
|
||
for _ in range(10):
|
||
# 等待内容加载
|
||
self.tab.wait.ele_displayed('.List-item')
|
||
self.tab.wait(3)
|
||
# 向下滚动页面，直到所有内容加载完成
|
||
self.tab.scroll.to_bottom()
|
||
self.tab.wait(1)
|
||
self.tab.scroll.up(100)
|
||
|
||
# 获取话题/问题相关信息：话题内容、keywards、话题创建日期dateCreated、话题修改日期dateModified、回答数量answerCount、评论数量commentCount
|
||
question_page = self.tab.ele('.QuestionPage')
|
||
# 获取话题属性，为QuestionPage的前9个meta标签
|
||
metas = question_page.eles('tag:meta')[0:9]
|
||
# print(metas)
|
||
title = get_content_from_meta(metas, 'name')
|
||
answer_count = get_content_from_meta(metas, 'answerCount')
|
||
comment_count = get_content_from_meta(metas, 'commentCount')
|
||
keywords = get_content_from_meta(metas, 'keywords')
|
||
date_created = get_content_from_meta(metas, 'dateCreated')
|
||
date_modified = get_content_from_meta(metas, 'dateModified')
|
||
follower_count = get_content_from_meta(metas, 'zhihu:followerCount')
|
||
# print(date_created, date_modified, answer_count, comment_count, keywords)
|
||
topic_description = ""
|
||
try:
|
||
topic_description = question_page.ele('.RichText ztext css-ob6uua').text
|
||
except ElementNotFoundError as e:
|
||
logger.error(f"元素缺失：不存在topic_description")
|
||
|
||
# 获取所有内容条目
|
||
content_items = self.tab.ele('.Question-mainColumn').eles('.List-item')
|
||
|
||
total_characters = 0
|
||
for item in content_items:
|
||
try:
|
||
content = item.ele('.RichContent-inner').text
|
||
# 计算content的字数
|
||
content_len = len(content)
|
||
print(content_len)
|
||
if content_len > 1000 or content_len < 100:
|
||
logger.error(f"skip本条内容，内容长度：{content_len}")
|
||
continue
|
||
if total_characters > 5000:
|
||
logger.error(f"contents_result长度超过5000，跳出循环")
|
||
break
|
||
total_characters += content_len
|
||
contents_result.append(content)
|
||
# 打印contents_result的长度
|
||
logger.info(f"contents_result长度：{len(contents_result)}")
|
||
except ElementNotFoundError as e:
|
||
logger.error(f"元素缺失：{str(e)}")
|
||
except ValueError as e:
|
||
logger.error(f"热度值转换失败：{str(e)}")
|
||
|
||
except ElementNotFoundError as e:
|
||
logger.error(f"热榜容器元素未找到：{str(e)}")
|
||
except Exception as e:
|
||
logger.error(f"获取热榜数据异常：{str(e)}")
|
||
finally:
|
||
if self.tab:
|
||
self.tab.close()
|
||
# 返回json格式的数据
|
||
return json.dumps({
|
||
'title': title,
|
||
'answer_count': answer_count,
|
||
'comment_count': comment_count,
|
||
'topic_description': topic_description,
|
||
'keywords': keywords,
|
||
'date_created': date_created,
|
||
'date_modified': date_modified,
|
||
'follower_count': follower_count,
|
||
'contents': contents_result
|
||
}, ensure_ascii=False)
|
||
|
||
|
||
if __name__ == '__main__':
|
||
# 测试用例
|
||
logger.info('知乎采集测试')
|
||
# 执行采集任务
|
||
zhihu = Zhihu()
|
||
result = zhihu.get_content('https://www.zhihu.com/question/588507809')
|
||
print(len(result))
|
||
print(result)
|
||
logger.info('测试完成')
|
||
|