From ac6c88176302f8d00d4da99f0a58c38b396e9b5b Mon Sep 17 00:00:00 2001 From: konjacpotato Date: Fri, 20 Feb 2026 18:51:35 +0800 Subject: [PATCH] =?UTF-8?q?=E8=B0=83=E6=95=B4=E8=B1=86=E7=93=A3=E5=B0=8F?= =?UTF-8?q?=E7=BB=84=E5=B8=96=E5=AD=90=E8=8E=B7=E5=8F=96=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- seek/douban_com/douban_group_seek.py | 31 +++++++++++------- task/__pycache__/manager_task.cpython-312.pyc | Bin 4801 -> 4529 bytes 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/seek/douban_com/douban_group_seek.py b/seek/douban_com/douban_group_seek.py index 780b891..41c67c3 100644 --- a/seek/douban_com/douban_group_seek.py +++ b/seek/douban_com/douban_group_seek.py @@ -1,5 +1,5 @@ import json -from time import sleep +import time from DrissionPage import Chromium, ChromiumOptions from config.database import SessionLocal from models.source_content import SourceContent @@ -14,8 +14,6 @@ class DoubanGroupSeek: def seek(self): db = SessionLocal() - # 获取最近100条数据的URL列表，用于过滤掉已存在的URL，避免重复爬取和存储 - recent_contents = db.query(SourceContent).order_by(SourceContent.id.desc()).limit(100).all() group_url = f'https://www.douban.com/group/{self.group_id}' tab = self.browser.new_tab() @@ -28,20 +26,29 @@ class DoubanGroupSeek: for ele_tr in ele_trs: topic_title = ele_tr.ele('tag:a').text topic_url = ele_tr.ele('tag:a').attr('href') - topics.append((topic_title, topic_url)) + update_time = ele_tr.ele('.time').text + topics.append((topic_title, topic_url, update_time)) - # 过滤掉已存在的URL - existing_urls = set(content.link for content in recent_contents) - topics = [(title, url) for title, url in topics if url not in existing_urls] + # 去掉两个置顶的帖子，根据title包含“置顶”来判断，因为置顶的帖子一般情况是组规则和公告 + topics = [(title, url, update_time) for title, url, update_time in topics if "置顶" not in title] + + # 根据更新时间过滤，update_time格式为“10-18 12:34”，只保留24小时内的帖子 + time_str = time.strftime("%m-%d %H:%M", time.localtime(time.time() - 24 * 3600)) + topics = [(title, url, update_time) for title, url, update_time in topics if update_time >= time_str] # 打印要爬取的主题列表 - logger.info(f"Found {len(topics)} new topics to crawl:") - for topic_title, topic_url in topics: - logger.info(f"标题：{topic_title} 链接：{topic_url}\n") + logger.info(f"Found {len(topics)} potential new topics to crawl:") results = [] - for topic_title, topic_url in topics: - logger.info(f"fetch 标题：{topic_title} 链接：{topic_url}\n") + for topic_title, topic_url, update_time in topics: + # 检索数据库，根据topic_url查询是否已存在 + existing_content = db.query(SourceContent.id).filter(SourceContent.link == topic_url).first() + if existing_content: + # logger.info(f"Topic already exists in database, skipping: {topic_title}:{topic_url}") + continue + + logger.info(f"fetch 标题：{topic_title} 链接：{topic_url} 更新时间：{update_time}\n") + tab.get(topic_url) tab.wait(30) # 等待页面加载完成，时间可根据实际情况调整 try: diff --git a/task/__pycache__/manager_task.cpython-312.pyc b/task/__pycache__/manager_task.cpython-312.pyc index 4cae05a38b53a2001be3b410d04b5b2bd6357555..519ad2b061829a11eaef11db6fb2666a06a7cd4b 100644 GIT binary patch delta 232 zcmX@8x>1?;G%qg~0}x1_pO`7Pk(ZH;X#vY*0k%*^zRhWDQY!i*V~RN^f@I zZewK9l$iXQ=NXeG@8oy9VS2^9K&=YMphyHLa*Hb`KQSe~B(XTVxX2GEpasOm4x4NF z449O!33z?1WMmMP`p(Y4CEhRFDf_FK2Pog|q3_AUe3FILlXo(=pf#i4WOqRo)?iU6i JT_grn4ggI6Hf{g_ delta 412 zcmdm}d{CA5G%qg~0}xc7otSxeBQGNxlL6~w0k%-au+3>~QY>yY+*t-7c~k%tq*i1WmlW$L6qV*Flw{_nDr6QbXi81C;dv%|i#IQ|q9h(D6AzNP#Z_FASX2_9 zl30?e$v0VrH%#Cb2TZb<4``RdaEbTJcFO)L z1_eO3r@0po^C=!yFVV@r`K=iPCz}hXu!aC-@+T(