Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix offset error and updata collect and eps #33

Merged
merged 1 commit into from
Aug 30, 2024
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions backend/src/module/database/bangumi.py
Original file line number Diff line number Diff line change
@@ -131,6 +131,7 @@ def search_url(self, rss_link: str) -> Bangumi | None:
logger.debug(f"[Database] Find bangumi id: {rss_link}.")
return self.session.exec(statement).first()


def search_id(self, _id: int) -> Bangumi | None:
statement = select(Bangumi).where(Bangumi.id == _id)
bangumi = self.session.exec(statement).first()
2 changes: 1 addition & 1 deletion backend/src/module/downloader/client/qb_downloader.py
Original file line number Diff line number Diff line change
@@ -72,7 +72,7 @@ async def get_torrent_files(self, _hash: str) -> list[str]:
files_name = [file["name"] for file in reps.json()]
return files_name

async def torrents_info(self, status_filter, category, tag=None, limit=50):
async def torrents_info(self, status_filter, category, tag=None, limit=0):
data = {
"filter": status_filter,
"category": category,
4 changes: 2 additions & 2 deletions backend/src/module/downloader/download_client.py
Original file line number Diff line number Diff line change
@@ -22,10 +22,10 @@ def __init__(self):
self._path_parser = TorrentPath

async def get_torrent_info(
self, category="Bangumi", status_filter="completed", tag=None
self, category="Bangumi", status_filter="completed", tag=None,limit = 0
):
return await self.torrents_info(
status_filter=status_filter, category=category, tag=tag
status_filter=status_filter, category=category, tag=tag,limit=limit
)

async def rename_torrent_file(self, _hash, old_path, new_path) -> bool:
40 changes: 20 additions & 20 deletions backend/src/module/manager/collector.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
import logging

from module.database import Database, engine
from module.downloader import DownloadClient
from module.models import Bangumi, ResponseModel
from module.downloader import DownloadClient, DownloadQueue
from module.models import Bangumi
from module.models.rss import RSSItem
from module.rss import RSSEngine, RSSManager
from module.searcher import SearchTorrent

@@ -29,24 +30,23 @@ async def collect_season(self, bangumi: Bangumi, link: str = None):
if not link:
torrents = await self.st.search_season(bangumi)
else:
async with self.st.req() as req:
torrents = await req.get_torrents(link)
async with DownloadClient() as client:
if await client.add_torrents(torrents, bangumi):
logger.info(
f"Collections of {bangumi.official_title} Season {bangumi.season} completed."
)
bangumi.eps_collect = True
with Database(engine) as db:
if db.bangumi.update(bangumi):
db.bangumi.add(bangumi)
db.torrent.add_all(torrents)
return True
else:
logger.warning(
f"Already collected {bangumi.official_title} Season {bangumi.season}."
)
return False
torrents = await self.st.search_torrents(RSSItem(url=link))

if torrents:
await DownloadQueue().add_torrents(torrents=torrents,bangumi=bangumi)
logger.info(
f"Collections of {bangumi.official_title} Season {bangumi.season} completed."
)
bangumi.eps_collect = True
with Database(engine) as db:
if db.bangumi.update(bangumi):
db.bangumi.add(bangumi)
return True
else:
logger.warning(
f"Already collected {bangumi.official_title} Season {bangumi.season}."
)
return False

@staticmethod

153 changes: 88 additions & 65 deletions backend/src/module/manager/renamer.py
Original file line number Diff line number Diff line change
@@ -77,8 +77,10 @@ async def rename_file(
renamer_info.save_path, file, file_type="subtitle"
)
if ep and renamer_info.bangumi and renamer_info.bangumi.offset:
# logging.info(f"[debug] {ep.episode=} {renamer_info.bangumi.offset}")
ep.episode += renamer_info.bangumi.offset


bangumi_name = bangumi_name

new_path = self.gen_path(ep, bangumi_name, method)
@@ -90,7 +92,7 @@ async def rename_file(
post_path = None
if renamer_info.bangumi and renamer_info.bangumi.poster_link:
post_path = renamer_info.bangumi.poster_link
if ep:
if ep and renamer_info.torrent.id:
notification_info = Notification(
title=bangumi_name,
season=ep.season,
@@ -99,7 +101,7 @@ async def rename_file(
)
asyncio.create_task(PostNotification().send(notify=notification_info))
self.count += 1
logger.info(f"[Renamer] {old_path} -> {new_path}")
# logger.info(f"[Renamer] {old_path} -> {new_path}")
return result

async def rename_files(
@@ -118,8 +120,9 @@ async def rename_files(
tasks.append(task)

results = await asyncio.gather(*tasks, return_exceptions=True)

for i, result in enumerate(results):
if isinstance(result, Exception):
if isinstance(result, BaseException):
logger.warning(f"[Renamer] {media_list[i]} rename failed")
return result
else:
@@ -129,86 +132,106 @@ async def rename_files(
# TODO: remove bad torrent
return renamer_info.torrent

async def gen_renamer_info(
self,
client: DownloadClient,
torrent_hash: str,
bangumi,
torrent_item,
save_path,
):
torrent_contents: list[str] = await client.get_torrent_files(torrent_hash)
if torrent_contents:
# 获取最浅一层的文件,若一层无文件才会到下一层,并只会穿一层
torrent_content_len = [
self._path_parser._file_depth(f) for f in torrent_contents
]
torrent_contents = [
f
for f, f_len in zip(torrent_contents, torrent_content_len)
if f_len == min(torrent_content_len)
]

return RenamerInfo(
torrent=torrent_item,
bangumi=bangumi,
hash=torrent_hash,
save_path=save_path,
content=torrent_contents,
)

async def rename_by_info(self, client, renamer_info: RenamerInfo):
rename_method = settings.bangumi_manage.rename_method

# 从save_path和settings.download.path 对比拿到 bangumi_name,season
bangumi_name, _ = self._path_parser.path_to_bangumi(renamer_info.save_path)
media_list, subtitle_list = self._path_parser.check_files(renamer_info.content)
torrent_item = await self.rename_files(
renamer_info,
media_list,
rename_method,
bangumi_name,
client,
)
await self.rename_files(
renamer_info,
subtitle_list,
f"subtitle_{rename_method}",
bangumi_name,
client,
)
if not isinstance(torrent_item, BaseException) and torrent_item and torrent_item.id:
with Database() as db:
torrent_item = db.torrent.search(torrent_item.id)
torrent_item.downloaded = True
db.torrent.update(torrent_item)

async def rename(self):
"""
每个 RenamerInfo 对应一个种子,包含种子,对应的bangumi, save_path,hash,content
"""
# 从数据库中获取downloaded=0的种子,也就不在管理非AB下载的种子
# 与其管理非AB下载的种子,不如AB提供下载的方法
# 现有的管理效果也并不好,所以放弃
# 从数据库中获取downloaded=0的种子,也就不在管理非AB下载的种子
# 与其管理非AB下载的种子,不如AB提供下载的方法
# 现有的管理效果也并不好,所以放弃
logger.debug("[Renamer] Start rename process.")
async with DownloadClient() as client:
# 获取AB 下载的种子详细信息,主要是获取下载进度和save_path
bangumi_torrent_infos: list[dict] = await client.get_torrent_info()
bangumi_torrent_infos: list[dict] = await client.get_torrent_info(limit=50)
with Database() as database:
torrent_items = database.torrent.search_all_unrenamed()
hash_list = [get_hash(link_hash.url) for link_hash in torrent_items]
name_list = [torrent_item.name for torrent_item in torrent_items]

renamer_info_list: list[RenamerInfo] = []

for bangumi_torrent_info in bangumi_torrent_infos:
torrent_hash = bangumi_torrent_info["hash"]
if torrent_hash in hash_list:
hash_idx = hash_list.index(torrent_hash)
torrent_item = torrent_items[hash_idx]
bangumi = torrent_to_bangumi(torrent_item.bangumi_id)

torrent_contents: list[str] = await client.get_torrent_files(
torrent_hash
)

if torrent_contents:
# 获取最浅一层的文件,若一层无文件才会到下一层,并只会穿一层
torrent_content_len = [
self._path_parser._file_depth(f) for f in torrent_contents
]
torrent_contents = [
f
for f, f_len in zip(torrent_contents, torrent_content_len)
if f_len == min(torrent_content_len)
]

renamer_info = RenamerInfo(
torrent=torrent_item,
bangumi=bangumi,
hash=torrent_hash,
save_path=bangumi_torrent_info["save_path"],
content=torrent_contents,
)
renamer_info_list.append(renamer_info)

rename_method = settings.bangumi_manage.rename_method
torrent_name = bangumi_torrent_info["name"]
# 部份torrent 的hash与mikan不一致
if torrent_hash in hash_list:
torrent_idx = hash_list.index(torrent_hash)
elif torrent_name in name_list:
torrent_idx = name_list.index(torrent_name)
else:
continue
torrent_item = torrent_items[torrent_idx]
bangumi = torrent_to_bangumi(torrent_item.bangumi_id)

renamer_info = await self.gen_renamer_info(
client,
torrent_hash,
bangumi,
torrent_item,
save_path=bangumi_torrent_info["save_path"],
)

renamer_info_list.append(renamer_info)

renamer_task = []
for renamer_info in renamer_info_list:
# 从save_path和settings.download.path 对比拿到 bangumi_name,season
# 当然,bangumi存在的话,直接从里面拿
bangumi_name, _ = self._path_parser.path_to_bangumi(
renamer_info.save_path
)
media_list, subtitle_list = self._path_parser.check_files(
renamer_info.content
)
renamer_task.append(
self.rename_files(
renamer_info, media_list, rename_method, bangumi_name, client
)
)
renamer_task.append(
self.rename_files(
renamer_info,
subtitle_list,
f"subtitle_{rename_method}",
bangumi_name,
client,
)
)
renamer_task.append(self.rename_by_info(client,renamer_info))
torrents = await asyncio.gather(*renamer_task, return_exceptions=True)
with Database() as db:
for torrent in torrents:
if not isinstance(torrent, Exception) and torrent:
torrent = db.torrent.search(torrent.id)
torrent.downloaded = True
db.torrent.update(torrent)

logging.info(f"[Renamer] have renamed {self.count}")

async def compare_ep_version(
50 changes: 39 additions & 11 deletions backend/src/module/manager/torrent.py
Original file line number Diff line number Diff line change
@@ -3,8 +3,8 @@

from module.database import Database, engine
from module.downloader import DownloadClient
from module.models import Bangumi, BangumiUpdate, ResponseModel
from module.manager.renamer import Renamer
from module.models import Bangumi, BangumiUpdate, ResponseModel
from module.parser import TmdbParser
from module.utils.bangumi_data import get_hash

@@ -67,8 +67,8 @@ async def delete_rule(self, _id: int | str, file: bool = False):
if file:
torrent_message = await self.delete_torrents(data, client)
logger.info(f"[Manager] Delete rule for {data.official_title}")
return data,torrent_message
return None,None
return data, torrent_message
return None, None

async def disable_rule(self, _id: str | int, file: bool = False):
with Database() as db:
@@ -118,9 +118,33 @@ async def enable_rule(self, _id: str | int):
msg_zh=f"无法找到 id {_id}",
)

async def offset_rename(self,data:Bangumi,path,hash_list):
renamer = Renamer()
# data.offset = data.offset - 20000
if data.offset<-2000:
await asyncio.sleep(30)
renamer_task = []
async with DownloadClient() as client:
for torrent_hash in hash_list:
renamer_info = await renamer.gen_renamer_info(
client=client,
torrent_hash=torrent_hash,
bangumi=data,
torrent_item="",
save_path=path,
)
renamer_task.append(
renamer.rename_by_info(
client,
renamer_info,
)
)
await asyncio.gather(*renamer_task, return_exceptions=True)

async def update_rule(self, bangumi_id: int, data: BangumiUpdate):
with Database() as db:
old_data: Bangumi | None = db.bangumi.search_id(bangumi_id)
renamer = Renamer()
if old_data:
# 当只改Filter的时候只改database
# 当offset 的时候 只改torrent
@@ -135,16 +159,20 @@ async def update_rule(self, bangumi_id: int, data: BangumiUpdate):
old_data.save_path = client._path_parser.gen_save_path(old_data)
hash_list = await self.__match_torrents_list(old_data)
path = client._path_parser.gen_save_path(data)
data.save_path = path
temp_data = Bangumi(offset=data.offset - old_data.offset)
if temp_data.offset != 0:
temp_data.offset = 10000 + temp_data.offset

if hash_list:
await client.move_torrent(hash_list, path)
data.save_path = path

torrent_list = db.torrent.search_bangumi(old_data.id)
# 有bug,这会累加 offset
for torrent in torrent_list:
if get_hash(torrent.url) in hash_list:
torrent.downloaded = False
db.torrent.add_all(torrent_list)
# save_path改动后名命名一次
await self.offset_rename(temp_data,path,hash_list)
if temp_data.offset != 0:
temp_data.offset = -10000
# 太快了会导致renamer 的数据是之前的
asyncio.create_task(self.offset_rename(temp_data,path,hash_list))

db.bangumi.update(data, bangumi_id)
return True
else:
1 change: 1 addition & 0 deletions backend/src/module/parser/analyser/raw_parser.py
Original file line number Diff line number Diff line change
@@ -442,6 +442,7 @@ def raw_parser(raw: str) -> Episode | None:
title = "[ANi] Bakemonogatari / 物语系列 第外季&第怪季 - 06.5 [1080P][Baha][WEB-DL][AAC AVC][CHT][MP4][ANi] Bakemonogatari / 物语系列 第外季&第怪季 - 06.5 [1080P][Baha][WEB-DL][AAC AVC][CHT][MP4][217.2 MB]"
title = "ANi] 我獨自升級 - 07.5 [1080P][Baha][WEB-DL][AAC AVC][CHT].mp4"
title = "[NEO·QSW]古莲泰沙U グレンダイザーU Grendizer U 02[WEBRIP AVC 1080P](搜索用:巨灵神/克雷飞天神)"
title ="地下城里的人们 (2024) S02E10005.mp4"
# title = "[LoliHouse] 2.5次元的诱惑 / 2.5-jigen no Ririsa - 01 [WebRip 1080p HEVC-10bit AAC][简繁内封字幕][LoliHouse] 2.5次元的诱惑 / 2.5-jigen no Ririsa - 01 [WebRip 1080p HEVC-10bit AAC][简繁内封字幕][609.59 MB]"
print(title)

Loading