From 51d1c403f53f2b3942967ced0d8ae0bd280e0c50 Mon Sep 17 00:00:00 2001
From: konjacpotato <konjacpotato@konjacpotatodeMac-mini.local>
Date: Sun, 15 Feb 2026 12:37:48 +0800
Subject: [PATCH] seek: douban group

---
 .env                                          |   2 +-
 config/__pycache__/__init__.cpython-312.pyc   | Bin 129 -> 207 bytes
 peter.py                                      |  14 ++-
 seek/douban_com/douban_group_seek.py          |  98 ++++++++++++++++++
 task/__init__.py                              |   2 +
 task/__pycache__/__init__.cpython-312.pyc     | Bin 127 -> 292 bytes
 task/__pycache__/manager_task.cpython-312.pyc | Bin 4567 -> 4801 bytes
 task/hot_topic/douban.py                      |  15 +++
 task/manager_task.py                          |  18 ++--
 utils/__pycache__/__init__.cpython-312.pyc    | Bin 128 -> 201 bytes
 10 files changed, 131 insertions(+), 18 deletions(-)
 create mode 100644 seek/douban_com/douban_group_seek.py
 create mode 100644 task/__init__.py
 create mode 100644 task/hot_topic/douban.py

diff --git a/.env b/.env
index e7046d4..d5c246a 100644
--- a/.env
+++ b/.env
@@ -7,7 +7,7 @@ LOG_LEVEL=DEBUG
 LOG_TYPE=console
 
 # 数据库配置
-DB_HOST=192.168.1.200
+DB_HOST= 47.119.128.161                   # 192.168.1.200
 DB_PORT=19732
 DB_USER=postgres
 DB_PASS=postgres
diff --git a/config/__pycache__/__init__.cpython-312.pyc b/config/__pycache__/__init__.cpython-312.pyc
index bbef6004ce0fb1ef7cd62fec2ec9b81b021b1400..392cd15b8cd27725c894eb763a1d51acce1e53b2 100644
GIT binary patch
literal 207
zcmX@j%ge<81h;}FWU2t^#~=<2FhLog1%Qm{3@HpLj5!Rsj8Tk?43$ip%r8OGnvAzN
zic?EUGV{`l{WO_w@h9i!rDdk;!NiJ~fr?f#d<LogWvw4toLW?@pPiqVm6%+RUy@jo
zuV0>DlwDkqn4GGgoS%}aUyxdoS_EW*Owo^z&&<m#iI3MSsJz8tlbfGXnv-f*!~rym
V5r~UHj1SC=jEwgfREpSu8~`u)Hh2I4

literal 129
zcmX@j%ge<81g&@C(n0iN5P=Rpvj9b=GgLBYGWxA#C}INgK7-W!Qi!mMNzPA6jVVYi
zNi700^U^ZYW8&j8^D;}~<Mj$Ee{tC4=BJeAq}mm+0yQuKaWRPTk(rT^v4|PS0syM*
B9C`o%

diff --git a/peter.py b/peter.py
index ed57a39..e09ff91 100644
--- a/peter.py
+++ b/peter.py
@@ -1,13 +1,11 @@
 import datetime
 from functools import partial
-
 from apscheduler.schedulers.blocking import BlockingScheduler
-
-from config import config
-from log.log_manager import log, logger
-from task.manager_task import manager_task
-
 from apscheduler.events import EVENT_JOB_ERROR
+from config import config
+from task import manager_task
+from utils import logger
+
 
 def job_error_listener(event):
     if event.exception:
@@ -31,7 +29,7 @@ if __name__ == '__main__':
     scheduler.add_listener(job_error_listener, EVENT_JOB_ERROR)
 
     try:
-        log(f"started successfully.")
+        logger.info(f"started successfully.")
         scheduler.start()  # 阻塞运行
     except (KeyboardInterrupt, SystemExit):
-        log(f"Shutting down ...")
+        logger.info(f"Shutting down ...")
diff --git a/seek/douban_com/douban_group_seek.py b/seek/douban_com/douban_group_seek.py
new file mode 100644
index 0000000..780b891
--- /dev/null
+++ b/seek/douban_com/douban_group_seek.py
@@ -0,0 +1,98 @@
+import json
+from time import sleep
+from DrissionPage import Chromium, ChromiumOptions
+from config.database import SessionLocal
+from models.source_content import SourceContent
+from utils import logger
+
+
+class DoubanGroupSeek:
+    def __init__(self, group_id):
+        co = ChromiumOptions().set_local_port(9333)
+        self.browser = Chromium(addr_or_opts=co)
+        self.group_id = group_id
+
+    def seek(self):
+        db = SessionLocal()
+        # 获取最近100条数据的URL列表，用于过滤掉已存在的URL，避免重复爬取和存储
+        recent_contents = db.query(SourceContent).order_by(SourceContent.id.desc()).limit(100).all()
+
+        group_url = f'https://www.douban.com/group/{self.group_id}'
+        tab = self.browser.new_tab()
+        tab.get(group_url)
+
+        title = tab.title
+        ele_table = tab.ele('tag:table@class=olt')
+        ele_trs = ele_table.eles('tag:tr@!class=th')
+        topics = []
+        for ele_tr in ele_trs:
+            topic_title = ele_tr.ele('tag:a').text
+            topic_url = ele_tr.ele('tag:a').attr('href')
+            topics.append((topic_title, topic_url))
+
+        # 过滤掉已存在的URL
+        existing_urls = set(content.link for content in recent_contents)
+        topics = [(title, url) for title, url in topics if url not in existing_urls]
+
+        # 打印要爬取的主题列表
+        logger.info(f"Found {len(topics)} new topics to crawl:")
+        for topic_title, topic_url in topics:
+            logger.info(f"标题：{topic_title} 链接：{topic_url}\n")
+
+        results = []
+        for topic_title, topic_url in topics:
+            logger.info(f"fetch 标题：{topic_title} 链接：{topic_url}\n")
+            tab.get(topic_url)
+            tab.wait(30)  # 等待页面加载完成，时间可根据实际情况调整
+            try:
+                title = tab.title
+                ele_article = tab.ele('.article')
+
+                # 获取帖子内容、发布时间、IP地址位置、作者等信息
+                ele_topic_content = ele_article.ele('#topic-content')
+                ele_topic_doc = ele_topic_content.ele('.topic-doc')
+                content = ele_topic_doc.ele('.topic-content').text
+                post_time = ele_topic_doc.ele('.create-time').text
+                ip_location = ele_topic_doc.ele('.ip-location').text
+                author = ele_topic_doc.ele('.from').text
+
+                # 获取评论列表
+                comments = []
+                # 评论不一定存在，需先判断
+                try:
+                    ele_comments = ele_article.ele('#comments')
+                    ele_comments_list = ele_comments.eles('tag:li')
+                    for ele_comment in ele_comments_list:
+                        comment_content = ele_comment.ele('.reply-content').text
+                        comment_time = ele_comment.ele('.pubtime').text
+                        comment_author = ele_comment.ele('tag:h4').child().text
+                        comments.append({
+                            "comment_content": comment_content,
+                            "comment_time": comment_time,
+                            "comment_author": comment_author
+                        })
+                except Exception as e:
+                    logger.warning(f"No comments found for topic {topic_title}:{topic_url}: {str(e)}")
+
+                results.append((topic_url, json.dumps({
+                        "title": title,
+                        "content": content,
+                        "post_time": post_time,
+                        "ip_location": ip_location,
+                        "author": author,
+                        "comments": comments
+                    }, ensure_ascii=False)))
+            except Exception as e:
+                logger.error(f"Error processing topic {topic_title}:{topic_url}: {str(e)}")
+                continue
+
+        # 存入数据库
+        for topic_url, data in results:
+            source_content = SourceContent(
+                link=topic_url,
+                platform='douban',
+                content=data
+            )
+            db.add(source_content)
+        db.commit()
+        tab.close() 
\ No newline at end of file
diff --git a/task/__init__.py b/task/__init__.py
new file mode 100644
index 0000000..5f6b4a9
--- /dev/null
+++ b/task/__init__.py
@@ -0,0 +1,2 @@
+from task.manager_task import manager_task
+from task.hot_topic.douban import spider_task as douban_hot_topic_task
\ No newline at end of file
diff --git a/task/__pycache__/__init__.cpython-312.pyc b/task/__pycache__/__init__.cpython-312.pyc
index 1fdafe34ab56c4052725c2fecd627bad82555177..88bd7a84a6998371d6752bfbbff8963f13fd4ddf 100644
GIT binary patch
literal 292
zcmX@j%ge<81gAYFW)=eJ#~=<2FhLogWq^$73@HpLj5!Rsj8Tk?AU0DDQ!aB93nN1%
zvnI<+kWx*?TRgdmd5P(%Me!wx#o0hUcX2^x3Y6=o$$Co=B(8_7sE8S;^Oh)BJR`p(
zz9hdOGg&VszceW^uZRUK2jRrSWx<B8WcUoS^OvQ5XmM&$v3_=bURGjqL4HYMNxpu0
zeo=ODL1J>MesX?Fs(wLgNotWk$O`@V_{_Y_lK6PNg34PQHo5sJr8%i~MchD>7=gGL
V#Q4C>$jEq?LF*oaTM;La4*)cWRBZqN

literal 127
zcmX@j%ge<81Xc^f(n0iN5P=Rpvj9b=GgLBYGWxA#C}INgK7-W!l8dm4NzPA6jVVYi
zNiB*gNi5EeiI30B%PfhH*DI*}#bJ}1pHiBWYFESxRL%&*#URE<W=2NFB4!{903%2n
ACjbBd

diff --git a/task/__pycache__/manager_task.cpython-312.pyc b/task/__pycache__/manager_task.cpython-312.pyc
index 8b3d96ddd093610cda4b33ca2a48b30b88896e56..09f6d3f0feba0b8bda0e044f76a25ef1b8a7b048 100644
GIT binary patch
delta 1283
zcmaJ>UuauZ7{4dU&Amx(YAiR&mb6W55@x!{oo14`MW^ekNxFuOZMwPq@h-W|nxwhq
z+>~y1;t-JzAJ%Zd3fn`+V6G@EFd2eAboer+$cfliVjq0ylMKwLSa2_%lTgr=?r_id
zo%5aF`JL~6-}%mp?_R+AgVkzgNc%HV8~eUyeasolYN4P&B!Gx<05HrPgRX*uXa<~f
zx~m+8(pKa<no6_|_6q+qcy!+a7X7C48k>f(%BYGS>*CU@(n(E$np9FNML9R2R^_TH
zO{uV?O~^S#%Bgupnoz0=l&Z2;lF%u=-)txJ4$`%c(I@&h-t4+!xonxeFk8aUh3iQi
z9>n6|C4T64r)Pek;S9|PtNec;FV;i#zHg@QNAamMIFZ7sJQkHDe)JA&5I~etH~DO~
zB9ANCY)}uK!~jiH(12)qBD`KN6p3yM%6TO>Syi&sHRwVkYz#QiXRKehhehA9C&AmO
z+qhw=Tr}oE>3Kr&781FHtLP=0pJ;i*MSZa!<+-5m0fBRE_?N`ye*#FR#a*b*^@1S!
ziF1S9=nw9zr%_(xcLG1U%lj<v6JmRt_&8eQlY6)KW9yk$nfwmoO^6cdf3AAaLsJjB
zWU>byRy~{O?fKA6QvmEiKbnHzCG@wc|7rgAnS<c;otW7VjFcIA5wt`%hhJNYz=b@*
zJHUgcgh60KcZEYdHzmW$SfzkAguZ~AY-;NAXPYd|iq=VBgfi`;;uR|Pqy5&TyPZRG
zS(Wqc1ZwaYSsNf-n?^UR_xjy)!?of0z%9e=u7xhVD}p0O@JJdT$>7W=_7;}JbLgY4
z>;A2<*nM#&o^Hg`IF`ZQ?6UYC8hmEohuZ*Cjd%*j(%73>7PD)jYc5$!;=t>R&P5aU
zC6>kEHJfA3QnP%n%pd+Dc|D1JVH}R(p7<}e#INqY6?drN4&BmjPcKZbNJ9<sxrb)?
z)h(MO96pMVoyJGs#XV=1ZK+2e0LJdx;25*>O7G>~e>8#o`ls<oLeM`D*n}N9Z3hxG
zBtZaaP&HEa%;wD>mn$-P4>~vh1X)lif|kf7X4TL&b79G_6Ve)m$B77b!vpA=tu(?;
zR>#U3qz?u>NJNtzCGoGqPTA$;a#=oGR-lWn)1x7!USK**?R@dkaWy|#R$hZClzBFA
zVn@qh0)|Hp9u&a5uf<SZf4QR<ht}Kkmct05v**_tqFUXD07wJu+P%)SN3Dbo02l6g
L;V*{h_DTN&QoB@V

delta 1078
zcmZuwUuaWT7{4d?Cb_w}NpITQ=1-d@*oK*=n{3Sar!gkAy4Hud)ugQwO=EN0(WFLB
zuAnSJ_q2{atmA{tF_<j~c5DxU`ru;(9~56s%bG%vF`-WiBhrDOK6p-Ig^nFK-**n*
z@BF^={mysp2ex`_k8L)Iql5U@X1@Kw_FCZcg&Al;hfD^<aX)eJD>@E~=qui#n?W*s
z3ZsTKB!G$cLq9*F?wdH8nkr<q)D=zFa7xb<b1AsZ_ev_|?4Yw)f(<?vu`XO+$PssF
zHBLeUL=JC?$NqNumiyM7{R{FF@m)B$+P~Ue{<;z-Uz{a_36e+=`NF1naqXNSqcEea
ziRtv^%$$}^2YJl+v4t5{LXoG&@Au<2>T9xgO`E!^YiV{R?1mNNFlvWq#$Llc0Tfdb
zd0^V~(t4_7%A(wUN^uP4MK^q7{#eKgB_4>R!e3@*PlW=^4Zuo?pWR2P|1)L-@Q={x
zll2dF110%)DjgeHF?WbkTlFA}ihsN(7JuQgDr19BEIrm~nxVasVFV^E@jWOPMfEWh
ze^>WGm*j1Ep!=HKlzsP}SEM$0YH5QXB_}xOx*{p>S>TQ2MBX*k+KWux^d$B{L=NvR
z=jB=Cg6Hx$>V^}x6R>EDz=q8m@K9vUkiAqhGWm*4r9WfGX+0c1%*+u`?eWfr*iycb
z$u^WO;xjbXM`tkuH|+PjT{i>E!X5jHo%jxr14HCwk_?TJv5Ul&-c%=HvSr1;7ZR<g
zTe0MNEJ>nc#5G=3Cm^64{``MHiS<~5M3cldR#nHhRVVQrEYFnB5!dmm8r@dfmb9Bg
zx8gs?NoR<JqC}1TsSH9{={mAEGW^i>P$KHts*-qd199!1mu60Ke((Lx`>H6@LKcSy
zMil;$VjOXU)iH3I6&OUANmOL-yxVYdnai0O4L2KZlm-e6!8&uEdm`8uu5Su%+)lUn
zOKO6x_$Vwna^o%ZzkNh!`rUbKW{?OqHRCkZRmaV~pUUL(nM-*MyIGuF0<$E)aZS#~
zJBZ=ALiTE2i{SGx+!{#u>VgFcFFHh&M$0{Qj`{MzrmvjbY3%DACKO(}w!=|ZZyiJ^
ZiHIk(!!=g*(*}e*r0c_fIqDmk-U4yBE71S|

diff --git a/task/hot_topic/douban.py b/task/hot_topic/douban.py
new file mode 100644
index 0000000..33a5b3b
--- /dev/null
+++ b/task/hot_topic/douban.py
@@ -0,0 +1,15 @@
+from task.manager_task import execute_task
+from seek.douban_com.douban_group_seek import DoubanGroupSeek
+from utils import logger
+
+
+def spider_task():
+    logger.info(f"Douban hot topic spider task start execute......")
+    # 社畜买房共进会 小组 ID 为 677158
+    douban_group_seek = DoubanGroupSeek(group_id=677158)
+    douban_group_seek.seek()
+    logger.info(f"Douban hot topic spider task end execute......")
+    
+
+if __name__ == '__main__':
+    execute_task(spider_task)
\ No newline at end of file
diff --git a/task/manager_task.py b/task/manager_task.py
index e9fb398..d13a5bb 100644
--- a/task/manager_task.py
+++ b/task/manager_task.py
@@ -6,7 +6,7 @@ from apscheduler.schedulers.blocking import BlockingScheduler
 from config import config
 from database.database import get_session
 from database.tscheduler.crud import get_tasks_by_executor
-from log.log_manager import log
+from utils import logger
 
 """
 这是一个特殊的任务，负责管理任务，命名为管理者任务。
@@ -24,10 +24,10 @@ def log_task_execution(task_name: str, start_time: float, end_time: float = None
     start_time_str = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(start_time))
     end_time_str = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(end_time))
     if end_time is None:
-        log(f"{task_name} start execute at {start_time_str}")
+        logger.info(f"{task_name} start execute at {start_time_str}")
     else:
         elapsed_time = end_time - start_time
-        log(f"{task_name} end execute at {end_time_str}, use time {elapsed_time:.2f} seconds")
+        logger.info(f"{task_name} end execute at {end_time_str}, use time {elapsed_time:.2f} seconds")
 
 
 def execute_task(task: callable):
@@ -66,7 +66,7 @@ def load_tasks(scheduler: BlockingScheduler):
                         replace_existing=True,
                         misfire_grace_time=interval_seconds
                     )
-                    log(f"Task {task.task_name} added with interval {interval_seconds} seconds")
+                    logger.info(f"Task {task.task_name} added with interval {interval_seconds} seconds")
                 elif trigger == "cron":
                     # 解析 cron 表达式的字段
                     fields = task.cron_expression.split()
@@ -90,7 +90,7 @@ def load_tasks(scheduler: BlockingScheduler):
                         id=str(task_id),
                         replace_existing=True
                     )
-                    log(f"Task {task.task_name} added with cron {task.cron_expression}")
+                    logger.info(f"Task {task.task_name} added with cron {task.cron_expression}")
                 elif trigger == "date":
                     scheduler.add_job(
                         task_function,
@@ -99,13 +99,13 @@ def load_tasks(scheduler: BlockingScheduler):
                         id=str(task_id),
                         replace_existing=True
                     )
-                    log(f"Task {task.task_name} added with date {task.execution_date}")
+                    logger.info(f"Task {task.task_name} added with date {task.execution_date}")
                 else:
-                    log(f"Task Invalid trigger type: {trigger}")
+                    logger.warning(f"Task Invalid trigger type: {trigger}")
             else:
-                log(f"Task {task.task_name} already exists......")
+                logger.info(f"Task {task.task_name} already exists......")
                 run_time = job.next_run_time - job.trigger.start_date
-                log(f"Task {task.task_name} already exists, run time is {run_time}")
+                logger.info(f"Task {task.task_name} already exists, run time is {run_time}")
 
 # 管理者任务
 def manager_task(scheduler: BlockingScheduler):
diff --git a/utils/__pycache__/__init__.cpython-312.pyc b/utils/__pycache__/__init__.cpython-312.pyc
index e83ec45b4e00dde7fec7d710a1b093360fae6509..28f99f8fdc10e96bb6ef2d024914c0b6f298fe79 100644
GIT binary patch
literal 201
zcmX@j%ge<81PZ|uGUb8vV-N=hn4pZ$0zk%eh7^Vr#vF!R#wbQchDs()=9eI8O~zYn
zIr-`7sYQO8Ot*MSOEPnc^&kR8%s}~-44*;Dep%^<7N-^!>u2ZZWhEvT<d-Ct<m;E`
z7iAY0BqpcoC+DZ6>KCMzq!#IejnI#e&&<m#iI3MSsJz8tlbfGXnv-f*!~ryl5r~UH
Rj1SC=jEwgf<crvV8~_s2GeZCX

literal 128
zcmX@j%ge<81m8l7(n0iN5P=Rpvj9b=GgLBYGWxA#C}INgK7-W!l8>;8NzPA6jVVYi
zNiB*gEy>I&j){-Y%*!l^kJl@x{Ka9Do1apelWJGQ3RKSs#Kj=SM`lJw#v*1Q3jmC;
B95Mg^