import peter

seek/zhihu_com/demo2.py

@@ -0,0 +1,116 @@
+import json
+
+from DrissionPage import Chromium
+from DrissionPage import ChromiumOptions
+from DrissionPage.errors import ElementNotFoundError
+
+from log.log_manager import logger
+
+
+def get_content_from_meta(metas, itemprop):
+    content = None
+    for meta in metas:
+        if meta.attr('itemprop') == itemprop:
+            content = meta.attr('content')
+    return content
+
+
+class Zhihu:
+    def __init__(self):
+        co = ChromiumOptions()
+        self.browser = Chromium()
+        self.tab = None
+
+    def get_content(self, url):
+        """获取话题内容数据"""
+        global title, keywords, date_created, date_modified, follower_count, comment_count, answer_count, topic_description
+        contents_result = []
+        try:
+            self.tab = self.browser.new_tab()
+            # 访问话题/问题页面
+            self.tab.get(url)
+
+            for _ in range(10):
+                # 等待内容加载
+                self.tab.wait.ele_displayed('.List-item')
+                self.tab.wait(3)
+                # 向下滚动页面，直到所有内容加载完成
+                self.tab.scroll.to_bottom()
+                self.tab.wait(1)
+                self.tab.scroll.up(100)
+
+            # 获取话题/问题相关信息：话题内容、keywards、话题创建日期dateCreated、话题修改日期dateModified、回答数量answerCount、评论数量commentCount
+            question_page = self.tab.ele('.QuestionPage')
+            # 获取话题属性，为QuestionPage的前9个meta标签
+            metas = question_page.eles('tag:meta')[0:9]
+            # print(metas)
+            title = get_content_from_meta(metas, 'name')
+            answer_count = get_content_from_meta(metas, 'answerCount')
+            comment_count = get_content_from_meta(metas, 'commentCount')
+            keywords = get_content_from_meta(metas, 'keywords')
+            date_created = get_content_from_meta(metas, 'dateCreated')
+            date_modified = get_content_from_meta(metas, 'dateModified')
+            follower_count = get_content_from_meta(metas, 'zhihu:followerCount')
+            # print(date_created, date_modified, answer_count, comment_count, keywords)
+            topic_description = ""
+            try:
+                topic_description = question_page.ele('.RichText ztext css-ob6uua').text
+            except ElementNotFoundError as e:
+                logger.error(f"元素缺失：不存在topic_description")
+
+            # 获取所有内容条目
+            content_items = self.tab.ele('.Question-mainColumn').eles('.List-item')
+
+            total_characters = 0
+            for item in content_items:
+                try:
+                    content = item.ele('.RichContent-inner').text
+                    # 计算content的字数
+                    content_len = len(content)
+                    print(content_len)
+                    if content_len > 1000 or content_len < 100:
+                        logger.error(f"skip本条内容，内容长度：{content_len}")
+                        continue
+                    if total_characters > 5000:
+                        logger.error(f"contents_result长度超过5000，跳出循环")
+                        break
+                    total_characters += content_len
+                    contents_result.append(content)
+                    # 打印contents_result的长度
+                    logger.info(f"contents_result长度：{len(contents_result)}")
+                except ElementNotFoundError as e:
+                    logger.error(f"元素缺失：{str(e)}")
+                except ValueError as e:
+                    logger.error(f"热度值转换失败：{str(e)}")
+
+        except ElementNotFoundError as e:
+            logger.error(f"热榜容器元素未找到：{str(e)}")
+        except Exception as e:
+            logger.error(f"获取热榜数据异常：{str(e)}")
+        finally:
+            if self.tab:
+                self.tab.close()
+        # 返回json格式的数据
+        return json.dumps({
+            'title': title,
+            'answer_count': answer_count,
+            'comment_count': comment_count,
+            'topic_description': topic_description,
+            'keywords': keywords,
+            'date_created': date_created,
+            'date_modified': date_modified,
+            'follower_count': follower_count,
+            'contents': contents_result
+        }, ensure_ascii=False)
+
+
+if __name__ == '__main__':
+    # 测试用例
+    logger.info('知乎采集测试')
+    # 执行采集任务
+    zhihu = Zhihu()
+    result = zhihu.get_content('https://www.zhihu.com/question/588507809')
+    print(len(result))
+    print(result)
+    logger.info('测试完成')
+