mirror of
https://github.com/JoeanAmier/XHS-Downloader.git
synced 2026-10-07 16:38:49 +08:00
refactor(app): 更新项目 HTTP 请求库
1. 将参数 `headers` 重构为 `impersonate` 2. 使用 `curl_cffi` 替代 `httpx`
This commit is contained in:
Quan
@@ -293,7 +293,7 @@ async def example():
|
||||
work_path = "D:\\" # 作品数据/文件保存根路径，默认值：项目根路径
|
||||
folder_name = "Download" # 作品文件储存文件夹名称（自动创建），默认值：Download
|
||||
name_format = "作品标题 作品描述"
|
||||
user_agent = "" # User-Agent
|
||||
impersonate = "chrome146" # 浏览器信息，参考 curl_cffi impersonate
|
||||
cookie = "" # 小红书网页版 Cookie
|
||||
proxy = None # 网络代理
|
||||
timeout = 5 # 请求数据超时限制，单位：秒
|
||||
@@ -317,7 +317,7 @@ async def example():
|
||||
work_path=work_path,
|
||||
folder_name=folder_name,
|
||||
name_format=name_format,
|
||||
user_agent=user_agent,
|
||||
impersonate=impersonate,
|
||||
cookie=cookie,
|
||||
proxy=proxy,
|
||||
timeout=timeout,
|
||||
@@ -395,10 +395,10 @@ async def example():
|
||||
<td align="center"><code>发布时间 作者昵称 作品标题</code></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td align="center">user_agent</td>
|
||||
<td align="center">impersonate</td>
|
||||
<td align="center">str</td>
|
||||
<td align="center">浏览器 User Agent</td>
|
||||
<td align="center">内置 Chrome User Agent</td>
|
||||
<td align="center">浏览器信息，参考 <a href="https://curl-cffi.readthedocs.io/en/latest/impersonate/targets.html">curl_cffi 文档</a></td>
|
||||
<td align="center">chrome146</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td align="center">cookie</td>
|
||||
@@ -515,9 +515,6 @@ async def example():
|
||||
<p>如果 <code>author_archive</code> 参数设置为 <code>true</code>，程序会把每个作者的作品储存至单独的文件夹；当作者的昵称发生变化时，程序会自动更新已下载作品文件名称中的作者昵称部分！</p>
|
||||
<p>除此之外，你还可以通过设置 <code>mapping_data</code> 参数为某个作者设置别名；如果对某个作者设置了别名，程序会使用你设置的作者别名去替代作者昵称！</p>
|
||||
</div>
|
||||
<hr>
|
||||
<p><b>其他说明：<code>user_agent</code>参数获取示例；强烈建议根据实际浏览器信息进行设置！</b></p>
|
||||
<img src="static/screenshot/请求头示例图.png" alt="">
|
||||
<h1 id="cookie">🌐 Cookie</h1>
|
||||
<ol>
|
||||
<li>打开浏览器（可选无痕模式启动），访问 <code>https://www.xiaohongshu.com/explore</code></li>
|
||||
@@ -712,7 +709,7 @@ A: 由于权限限制，您无法直接触发主仓库的 Actions。请通过 Fo
|
||||
|
||||
# 💡 项目参考
|
||||
|
||||
* https://github.com/encode/httpx/
|
||||
* https://github.com/lexiforest/curl_cffi
|
||||
* https://github.com/tiangolo/fastapi
|
||||
* https://github.com/textualize/textual/
|
||||
* https://github.com/marcelotduarte/cx_Freeze/
|
||||
|
||||
+6
-9
@@ -297,7 +297,7 @@ async def example():
|
||||
work_path = "D:\\" # Root path for saving note data/files, default: project root path
|
||||
folder_name = "Download" # Folder name for storing note files (auto-created), default: Download
|
||||
name_format = "作品标题 作品描述"
|
||||
user_agent = "" # User-Agent
|
||||
impersonate = "chrome146" # Browser information, refer to curl_cffi impersonate
|
||||
cookie = "" # Xiaohongshu web Cookie
|
||||
proxy = None # Network proxy
|
||||
timeout = 5 # Request timeout limit, unit: seconds
|
||||
@@ -321,7 +321,7 @@ async def example():
|
||||
work_path=work_path,
|
||||
folder_name=folder_name,
|
||||
name_format=name_format,
|
||||
user_agent=user_agent,
|
||||
impersonate=impersonate,
|
||||
cookie=cookie,
|
||||
proxy=proxy,
|
||||
timeout=timeout,
|
||||
@@ -399,10 +399,10 @@ async def example():
|
||||
<td align="center"><code>发布时间 作者昵称 作品标题</code></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td align="center">user_agent</td>
|
||||
<td align="center">impersonate</td>
|
||||
<td align="center">str</td>
|
||||
<td align="center">Browser User Agent</td>
|
||||
<td align="center">Built-in Chrome User Agent</td>
|
||||
<td align="center">Browser information, refer to <a href="https://curl-cffi.readthedocs.io/en/latest/impersonate/targets.html">curl_cffi documentation</a></td>
|
||||
<td align="center">chrome146</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td align="center">cookie</td>
|
||||
@@ -538,9 +538,6 @@ async def example():
|
||||
<p>When <code>author_archive</code> is set to <code>true</code>, the program will store each author's notes in dedicated folders. If an author's nickname changes, the program automatically updates the nickname portion in existing downloaded filenames!</p>
|
||||
<p>Additionally, you can configure author aliases through the <code>mapping_data</code> parameter. When an alias is set, the program will use your custom alias instead of the original nickname in filenames!</p>
|
||||
</div>
|
||||
<hr>
|
||||
<p><b>Additional Notes: The parameters <code>user_agent</code> examples are provided for reference; Strongly recommend setting according to actual browser information!</b></p>
|
||||
<img src="static/screenshot/请求头示例图.png" alt="">
|
||||
<h1 id="cookie">🌐 Cookie</h1>
|
||||
<ol>
|
||||
<li>Open the browser (optional: start in incognito mode) and visit <code>https://www.xiaohongshu.com/explore</code></li>
|
||||
@@ -733,7 +730,7 @@ repository to execute the build process
|
||||
|
||||
# 💡 Project References
|
||||
|
||||
* https://github.com/encode/httpx/
|
||||
* https://github.com/lexiforest/curl_cffi
|
||||
* https://github.com/tiangolo/fastapi
|
||||
* https://github.com/textualize/textual/
|
||||
* https://github.com/marcelotduarte/cx_Freeze/
|
||||
|
||||
+4
-3
@@ -1,6 +1,7 @@
|
||||
from asyncio import run
|
||||
|
||||
from curl_cffi.requests import post
|
||||
from pyperclip import paste
|
||||
from httpx import post
|
||||
from rich import print
|
||||
|
||||
from source import XHS
|
||||
@@ -15,7 +16,7 @@ async def example():
|
||||
work_path = "D:\\" # 作品数据/文件保存根路径，默认值：项目根路径
|
||||
folder_name = "Download" # 作品文件储存文件夹名称（自动创建），默认值：Download
|
||||
name_format = "作品标题 作品描述"
|
||||
user_agent = "" # User-Agent
|
||||
impersonate = "chrome146" # curl_cffi impersonate
|
||||
cookie = "" # 小红书网页版 Cookie
|
||||
proxy = None # 网络代理
|
||||
timeout = 5 # 请求数据超时限制，单位：秒
|
||||
@@ -41,7 +42,7 @@ async def example():
|
||||
work_path=work_path,
|
||||
folder_name=folder_name,
|
||||
name_format=name_format,
|
||||
user_agent=user_agent,
|
||||
impersonate=impersonate,
|
||||
cookie=cookie,
|
||||
proxy=proxy,
|
||||
timeout=timeout,
|
||||
|
||||
@@ -16,7 +16,6 @@ dependencies = [
|
||||
"emoji>=2.15.0",
|
||||
"fastapi>=0.139.0",
|
||||
"fastmcp>=3.4.4",
|
||||
"httpx[http2,socks]>=0.28.1",
|
||||
"lxml>=6.1.1",
|
||||
"pyperclip>=1.11.0",
|
||||
"pywebview>=6.2.1",
|
||||
|
||||
+2
-2
@@ -6,14 +6,14 @@ aiosqlite==0.22.1
|
||||
# via xhs-downloader (pyproject.toml)
|
||||
click==8.4.2
|
||||
# via xhs-downloader (pyproject.toml)
|
||||
curl-cffi==0.15.0
|
||||
# via xhs-downloader (pyproject.toml)
|
||||
emoji==2.15.0
|
||||
# via xhs-downloader (pyproject.toml)
|
||||
fastapi==0.139.0
|
||||
# via xhs-downloader (pyproject.toml)
|
||||
fastmcp>=3.4.4
|
||||
# via xhs-downloader (pyproject.toml)
|
||||
httpx[http2,socks]==0.28.1
|
||||
# via xhs-downloader (pyproject.toml)
|
||||
lxml==6.1.1
|
||||
# via xhs-downloader (pyproject.toml)
|
||||
pyperclip==1.11.0
|
||||
|
||||
+6
-4
@@ -7,12 +7,14 @@ from typing import Callable
|
||||
from click import (
|
||||
Choice,
|
||||
Context,
|
||||
Path as ClickPath,
|
||||
command,
|
||||
echo,
|
||||
option,
|
||||
pass_context,
|
||||
)
|
||||
from click import (
|
||||
Path as ClickPath,
|
||||
)
|
||||
from rich import print
|
||||
from rich.panel import Panel
|
||||
from rich.table import Table
|
||||
@@ -140,7 +142,7 @@ class CLI:
|
||||
("--work_path", "-wp", "str", _("作品数据/文件保存根路径")),
|
||||
("--folder_name", "-fn", "str", _("作品文件储存文件夹名称")),
|
||||
("--name_format", "-nf", "str", _("作品文件名称格式")),
|
||||
("--user_agent", "-ua", "str", "User-Agent"),
|
||||
("--impersonate", "-im", "str", "curl_cffi impersonate"),
|
||||
("--cookie", "-ck", "str", _("小红书网页版 Cookie，无需登录")),
|
||||
("--proxy", "-p", "str", _("网络代理")),
|
||||
("--timeout", "-t", "int", _("请求数据超时限制，单位：秒")),
|
||||
@@ -257,8 +259,8 @@ class CLI:
|
||||
"-nf",
|
||||
)
|
||||
@option(
|
||||
"--user_agent",
|
||||
"-ua",
|
||||
"--impersonate",
|
||||
"-im",
|
||||
)
|
||||
@option(
|
||||
"--cookie",
|
||||
|
||||
@@ -634,7 +634,7 @@ class GuiBackend:
|
||||
"image_format": manager.image_format,
|
||||
"video_preference": manager.video_preference,
|
||||
"proxy": manager.proxy,
|
||||
"user_agent": manager.blank_headers.get("user-agent", ""),
|
||||
"impersonate": manager.impersonate,
|
||||
"record_data": manager.record_data,
|
||||
"folder_mode": manager.folder_mode,
|
||||
"download_record": manager.download_record,
|
||||
|
||||
@@ -5,6 +5,7 @@ from textual.containers import Container, ScrollableContainer
|
||||
from textual.screen import Screen
|
||||
from textual.widgets import Button, Checkbox, Footer, Header, Input, Label, Select
|
||||
|
||||
from ..module import IMPERSONATE
|
||||
from ..translation import _
|
||||
|
||||
__all__ = ["Setting"]
|
||||
@@ -56,14 +57,14 @@ class Setting(Screen):
|
||||
id="name_format",
|
||||
),
|
||||
Label(
|
||||
"User-Agent",
|
||||
"impersonate",
|
||||
classes="params",
|
||||
),
|
||||
Input(
|
||||
self.data["user_agent"],
|
||||
placeholder=_("内置 Chrome User Agent"),
|
||||
self.data["impersonate"],
|
||||
placeholder=IMPERSONATE,
|
||||
valid_empty=True,
|
||||
id="user_agent",
|
||||
id="impersonate",
|
||||
),
|
||||
Label(
|
||||
_("小红书网页版 Cookie"),
|
||||
@@ -253,7 +254,7 @@ class Setting(Screen):
|
||||
"work_path": self.query_one("#work_path").value,
|
||||
"folder_name": self.query_one("#folder_name").value,
|
||||
"name_format": self.query_one("#name_format").value,
|
||||
"user_agent": self.query_one("#user_agent").value,
|
||||
"impersonate": self.query_one("#impersonate").value,
|
||||
"cookie": self.query_one("#cookie").value or self.data["cookie"],
|
||||
"proxy": self.query_one("#proxy").value or None,
|
||||
"timeout": int(self.query_one("#timeout").value),
|
||||
|
||||
@@ -34,10 +34,10 @@ from ..expansion import (
|
||||
from ..module import (
|
||||
__VERSION__,
|
||||
ERROR,
|
||||
IMPERSONATE,
|
||||
INFO,
|
||||
MASTER,
|
||||
REPOSITORY,
|
||||
USERAGENT,
|
||||
VERSION_BETA,
|
||||
VERSION_MAJOR,
|
||||
VERSION_MINOR,
|
||||
@@ -138,9 +138,9 @@ class XHS:
|
||||
work_path="",
|
||||
folder_name="Download",
|
||||
name_format="发布时间 作者昵称 作品标题",
|
||||
user_agent: str = USERAGENT,
|
||||
impersonate: str = IMPERSONATE,
|
||||
cookie: str = "",
|
||||
proxy: str | dict | None = None,
|
||||
proxy: str | None = None,
|
||||
timeout=10,
|
||||
chunk=1024 * 1024,
|
||||
max_retry=5,
|
||||
@@ -170,7 +170,7 @@ class XHS:
|
||||
folder_name,
|
||||
name_format,
|
||||
chunk,
|
||||
user_agent,
|
||||
impersonate,
|
||||
cookie,
|
||||
# self.read_browser_cookie(read_cookie) or cookie,
|
||||
proxy,
|
||||
@@ -301,10 +301,12 @@ class XHS:
|
||||
progress_callback: Callable[[dict], None] | None = None,
|
||||
task_id: str | None = None,
|
||||
result_callback: Callable[[dict], None] | None = None,
|
||||
proxy: str | None = None,
|
||||
) -> list[dict]:
|
||||
if not (
|
||||
urls := await self.extract_links(
|
||||
url,
|
||||
proxy=proxy,
|
||||
)
|
||||
):
|
||||
self.logging(_("提取小红书作品链接失败"), WARNING)
|
||||
@@ -317,6 +319,7 @@ class XHS:
|
||||
download,
|
||||
index,
|
||||
check_record=check_record,
|
||||
proxy=proxy,
|
||||
count=statistics,
|
||||
progress_callback=progress_callback,
|
||||
task_id=task_id,
|
||||
@@ -390,6 +393,7 @@ class XHS:
|
||||
async def extract_links(
|
||||
self,
|
||||
url: str,
|
||||
proxy: str | None = None,
|
||||
) -> list[str]:
|
||||
urls = []
|
||||
for i in url.split():
|
||||
@@ -397,6 +401,7 @@ class XHS:
|
||||
i = await self.html.request_url(
|
||||
u.group(),
|
||||
False,
|
||||
proxy=proxy,
|
||||
)
|
||||
if u := self.SHARE_XHS.search(i):
|
||||
urls.append(u.group())
|
||||
@@ -426,8 +431,8 @@ class XHS:
|
||||
url: str,
|
||||
id_: str,
|
||||
count: SimpleNamespace,
|
||||
cookie: str = None,
|
||||
proxy: str = None,
|
||||
cookie: str | None = None,
|
||||
proxy: str | None = None,
|
||||
) -> Namespace | dict:
|
||||
self.logging(_("开始处理作品：{0}").format(id_))
|
||||
html = await self.html.request_url(
|
||||
@@ -818,6 +823,7 @@ class XHS:
|
||||
data = None
|
||||
url = await self.extract_links(
|
||||
extract.url,
|
||||
proxy=extract.proxy,
|
||||
)
|
||||
if not url:
|
||||
msg = _("提取小红书作品链接失败")
|
||||
|
||||
@@ -3,7 +3,7 @@ from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, Callable
|
||||
|
||||
from aiofiles import open
|
||||
from httpx import HTTPError
|
||||
from curl_cffi.requests.exceptions import RequestException
|
||||
|
||||
from ..expansion import CacheError
|
||||
|
||||
@@ -20,7 +20,7 @@ from ..module import retry as re_download
|
||||
from ..translation import _
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from httpx import AsyncClient
|
||||
from curl_cffi.requests import AsyncSession
|
||||
|
||||
from ..module import Manager
|
||||
|
||||
@@ -48,7 +48,7 @@ class Download:
|
||||
self.folder = manager.folder
|
||||
self.temp = manager.temp
|
||||
self.chunk = manager.chunk
|
||||
self.client: "AsyncClient" = manager.download_client
|
||||
self.client: "AsyncSession" = manager.download_client
|
||||
self.headers = manager.blank_headers
|
||||
self.retry = manager.retry
|
||||
self.folder_mode = manager.folder_mode
|
||||
@@ -249,7 +249,7 @@ class Download:
|
||||
total = completed + content_length if content_length else None
|
||||
report("downloading", total)
|
||||
async with open(temp, "ab") as f:
|
||||
async for chunk in response.aiter_bytes(self.chunk):
|
||||
async for chunk in response.aiter_content(self.chunk):
|
||||
await f.write(chunk)
|
||||
completed += len(chunk)
|
||||
report("downloading", total)
|
||||
@@ -269,7 +269,7 @@ class Download:
|
||||
report("completed", total)
|
||||
logging(self.print, _("文件 {0} 下载成功").format(real.name))
|
||||
return True
|
||||
except HTTPError as error:
|
||||
except RequestException as error:
|
||||
report("failed")
|
||||
logging(
|
||||
self.print,
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from httpx import HTTPError
|
||||
from httpx import get
|
||||
from curl_cffi.requests import get
|
||||
from curl_cffi.requests.exceptions import RequestException
|
||||
|
||||
from ..module import ERROR, Manager, logging, retry, sleep_time
|
||||
from ..module import ERROR, logging, retry, sleep_time
|
||||
from ..translation import _
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -17,50 +17,46 @@ class Html:
|
||||
self,
|
||||
manager: "Manager",
|
||||
):
|
||||
self.manager = manager
|
||||
self.print = manager.print
|
||||
self.retry = manager.retry
|
||||
self.client = manager.request_client
|
||||
self.headers = manager.blank_headers
|
||||
self.timeout = manager.timeout
|
||||
self.proxy = manager.proxy
|
||||
self.impersonate = manager.impersonate
|
||||
|
||||
@retry
|
||||
async def request_url(
|
||||
self,
|
||||
url: str,
|
||||
content=True,
|
||||
cookie: str = None,
|
||||
proxy: str = None,
|
||||
cookie: str | None = None,
|
||||
proxy: str | None = None,
|
||||
**kwargs,
|
||||
) -> str:
|
||||
if not url.startswith("http"):
|
||||
url = f"https://{url}"
|
||||
headers = self.update_cookie(
|
||||
cookie,
|
||||
)
|
||||
headers = self.manager.get_headers(url)
|
||||
try:
|
||||
match bool(proxy):
|
||||
case False:
|
||||
response = await self.__request_url_get(
|
||||
url,
|
||||
headers,
|
||||
**kwargs,
|
||||
)
|
||||
await sleep_time()
|
||||
response.raise_for_status()
|
||||
return response.text if content else str(response.url)
|
||||
case True:
|
||||
response = await self.__request_url_get_proxy(
|
||||
url,
|
||||
headers,
|
||||
proxy,
|
||||
**kwargs,
|
||||
)
|
||||
await sleep_time()
|
||||
response.raise_for_status()
|
||||
return response.text if content else str(response.url)
|
||||
case _:
|
||||
raise ValueError
|
||||
except HTTPError as error:
|
||||
if cookie:
|
||||
response = self.__request_url_with_cookie(
|
||||
url,
|
||||
headers,
|
||||
cookie,
|
||||
proxy=proxy,
|
||||
**kwargs,
|
||||
)
|
||||
else:
|
||||
response = await self.__request_url_get(
|
||||
url,
|
||||
headers,
|
||||
proxy=proxy,
|
||||
**kwargs,
|
||||
)
|
||||
await sleep_time()
|
||||
response.raise_for_status()
|
||||
return response.text if content else str(response.url)
|
||||
except RequestException as error:
|
||||
logging(
|
||||
self.print,
|
||||
_("网络异常，{0} 请求失败: {1}").format(url, repr(error)),
|
||||
@@ -72,38 +68,23 @@ class Html:
|
||||
def format_url(url: str) -> str:
|
||||
return bytes(url, "utf-8").decode("unicode_escape")
|
||||
|
||||
def update_cookie(
|
||||
self,
|
||||
cookie: str = None,
|
||||
) -> dict:
|
||||
return self.headers | {"Cookie": cookie} if cookie else self.headers.copy()
|
||||
|
||||
async def __request_url_head(
|
||||
self,
|
||||
url: str,
|
||||
headers: dict,
|
||||
**kwargs,
|
||||
):
|
||||
return await self.client.head(
|
||||
url,
|
||||
headers=headers,
|
||||
def __request_url_with_cookie(
|
||||
self,
|
||||
url: str,
|
||||
headers: dict,
|
||||
cookie: str,
|
||||
proxy: str | None = None,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
async def __request_url_head_proxy(
|
||||
self,
|
||||
url: str,
|
||||
headers: dict,
|
||||
proxy: str,
|
||||
**kwargs,
|
||||
):
|
||||
return await self.client.head(
|
||||
return get(
|
||||
url,
|
||||
headers=headers,
|
||||
proxy=proxy,
|
||||
follow_redirects=True,
|
||||
verify=False,
|
||||
cookies=self.manager.cookie_str_to_dict(cookie),
|
||||
timeout=self.timeout,
|
||||
verify=False,
|
||||
allow_redirects=True,
|
||||
proxy=self.proxy if proxy is None else proxy,
|
||||
impersonate=self.impersonate,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@@ -111,27 +92,12 @@ class Html:
|
||||
self,
|
||||
url: str,
|
||||
headers: dict,
|
||||
proxy: str | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
return await self.client.get(
|
||||
url,
|
||||
headers=headers,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
async def __request_url_get_proxy(
|
||||
self,
|
||||
url: str,
|
||||
headers: dict,
|
||||
proxy: str,
|
||||
**kwargs,
|
||||
):
|
||||
return get(
|
||||
url,
|
||||
headers=headers,
|
||||
proxy=proxy,
|
||||
follow_redirects=True,
|
||||
verify=False,
|
||||
timeout=self.timeout,
|
||||
proxy=self.proxy if proxy is None else proxy,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
from xhshow import Xhshow
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from xhshow import Xhshow
|
||||
|
||||
from ..module import retry, sleep_time
|
||||
from httpx import get
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..module import Manager
|
||||
@@ -15,8 +16,8 @@ class UserPosted:
|
||||
manager: "Manager",
|
||||
url: str,
|
||||
params: dict,
|
||||
cookies: str = None,
|
||||
proxy: str = None,
|
||||
cookies: str | None = None,
|
||||
proxy: str | None = None,
|
||||
):
|
||||
self.url = url
|
||||
self.params = params
|
||||
@@ -25,10 +26,9 @@ class UserPosted:
|
||||
self.cookies = self.get_cookie(cookies)
|
||||
self.print = manager.print
|
||||
self.retry = manager.retry
|
||||
self.timeout = manager.timeout
|
||||
self.proxy = proxy
|
||||
self.proxy = (self.client.proxies.get("all", None) if proxy is None else proxy,)
|
||||
|
||||
def get_cookie(self, cookies: str = None) -> dict | str:
|
||||
def get_cookie(self, cookies: str | None = None) -> dict | str:
|
||||
if cookies:
|
||||
self.headers["cookie"] = cookies
|
||||
return cookies
|
||||
@@ -42,22 +42,12 @@ class UserPosted:
|
||||
@retry
|
||||
async def get_data(self):
|
||||
headers = self.get_headers()
|
||||
if self.proxy:
|
||||
response = get(
|
||||
self.url,
|
||||
params=self.params,
|
||||
headers=headers,
|
||||
proxy=self.proxy,
|
||||
follow_redirects=True,
|
||||
verify=False,
|
||||
timeout=self.timeout,
|
||||
)
|
||||
else:
|
||||
response = await self.client.get(
|
||||
self.url,
|
||||
params=self.params,
|
||||
headers=headers,
|
||||
)
|
||||
response = await self.client.get(
|
||||
self.url,
|
||||
params=self.params,
|
||||
headers=headers,
|
||||
proxy=self.proxy,
|
||||
)
|
||||
await sleep_time()
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
@@ -27,8 +27,8 @@ from .static import (
|
||||
INFO,
|
||||
USERSCRIPT,
|
||||
HEADERS,
|
||||
IMPERSONATE,
|
||||
PROJECT,
|
||||
USERAGENT,
|
||||
FILE_SIGNATURES,
|
||||
FILE_SIGNATURES_LENGTH,
|
||||
MAX_WORKERS,
|
||||
@@ -38,6 +38,7 @@ from .tools import (
|
||||
compare_versions,
|
||||
retry,
|
||||
logging,
|
||||
get_site_referer,
|
||||
sleep_time,
|
||||
retry_limited,
|
||||
)
|
||||
|
||||
+49
-51
@@ -3,22 +3,16 @@ from os import utime
|
||||
from pathlib import Path
|
||||
from re import compile, sub
|
||||
from shutil import move, rmtree
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, get_args
|
||||
|
||||
from httpx import (
|
||||
AsyncClient,
|
||||
AsyncHTTPTransport,
|
||||
HTTPStatusError,
|
||||
RequestError,
|
||||
TimeoutException,
|
||||
get,
|
||||
)
|
||||
from curl_cffi.requests import AsyncSession, BrowserTypeLiteral, get
|
||||
from curl_cffi.requests.exceptions import RequestException, Timeout
|
||||
|
||||
from source.expansion import remove_empty_directories
|
||||
|
||||
from ..translation import _
|
||||
from .static import HEADERS, USERAGENT, WARNING
|
||||
from .tools import logging
|
||||
from .static import HEADERS, IMPERSONATE, WARNING
|
||||
from .tools import get_site_referer, logging
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..expansion import Cleaner
|
||||
@@ -58,9 +52,9 @@ class Manager:
|
||||
folder: str,
|
||||
name_format: str,
|
||||
chunk: int,
|
||||
user_agent: str,
|
||||
impersonate: str,
|
||||
cookie: str,
|
||||
proxy: str | dict | None,
|
||||
proxy: str | None,
|
||||
timeout: int,
|
||||
retry: int,
|
||||
record_data: bool,
|
||||
@@ -85,9 +79,8 @@ class Manager:
|
||||
self.path = self.__check_path(path)
|
||||
self.folder = self.__check_folder(folder)
|
||||
self.compatible()
|
||||
self.blank_headers = HEADERS | {
|
||||
"user-agent": user_agent or USERAGENT,
|
||||
}
|
||||
self.blank_headers = HEADERS.copy()
|
||||
self.impersonate = self.__check_impersonate(impersonate)
|
||||
self.retry = self.__check_integer(retry, 5, 0)
|
||||
self.chunk = self.__check_integer(chunk, 2 * 1024 * 1024, 1024 * 1024)
|
||||
self.name_format = self.__check_name_format(name_format)
|
||||
@@ -95,34 +88,26 @@ class Manager:
|
||||
self.image_format = self.__check_image_format(image_format)
|
||||
self.folder_mode = self.check_bool(folder_mode, False)
|
||||
self.download_record = self.check_bool(download_record, True)
|
||||
self.timeout = self.__check_integer(timeout, 10, 1)
|
||||
self.proxy_tip = None
|
||||
self.proxy = self.__check_proxy(proxy)
|
||||
self.print_proxy_tip()
|
||||
self.timeout = self.__check_integer(timeout, 10, 1)
|
||||
self.request_client = AsyncClient(
|
||||
headers=self.blank_headers
|
||||
| {
|
||||
"referer": "https://www.xiaohongshu.com/",
|
||||
},
|
||||
cookies=self.cookie_str_to_dict(cookie),
|
||||
timeout=timeout,
|
||||
verify=False,
|
||||
http2=True,
|
||||
follow_redirects=True,
|
||||
mounts={
|
||||
"http://": AsyncHTTPTransport(proxy=self.proxy),
|
||||
"https://": AsyncHTTPTransport(proxy=self.proxy),
|
||||
},
|
||||
)
|
||||
self.download_client = AsyncClient(
|
||||
self.request_client = AsyncSession(
|
||||
headers=self.blank_headers,
|
||||
timeout=timeout,
|
||||
cookies=self.cookie_str_to_dict(cookie),
|
||||
timeout=self.timeout,
|
||||
verify=False,
|
||||
follow_redirects=True,
|
||||
mounts={
|
||||
"http://": AsyncHTTPTransport(proxy=self.proxy),
|
||||
"https://": AsyncHTTPTransport(proxy=self.proxy),
|
||||
},
|
||||
allow_redirects=True,
|
||||
proxy=self.proxy,
|
||||
impersonate=self.impersonate,
|
||||
)
|
||||
self.download_client = AsyncSession(
|
||||
headers=self.blank_headers,
|
||||
timeout=self.timeout,
|
||||
verify=False,
|
||||
allow_redirects=True,
|
||||
proxy=self.proxy,
|
||||
impersonate=self.impersonate,
|
||||
)
|
||||
self.image_download = self.check_bool(image_download, True)
|
||||
self.video_download = self.check_bool(video_download, True)
|
||||
@@ -215,8 +200,8 @@ class Manager:
|
||||
return default
|
||||
|
||||
async def close(self):
|
||||
await self.request_client.aclose()
|
||||
await self.download_client.aclose()
|
||||
await self.request_client.close()
|
||||
await self.download_client.close()
|
||||
# self.__clean()
|
||||
remove_empty_directories(self.root)
|
||||
remove_empty_directories(self.folder)
|
||||
@@ -242,31 +227,26 @@ class Manager:
|
||||
|
||||
def __check_proxy(
|
||||
self,
|
||||
proxy: str,
|
||||
proxy: str | None,
|
||||
url="https://www.xiaohongshu.com/explore",
|
||||
) -> str | None:
|
||||
if proxy:
|
||||
try:
|
||||
response = get(
|
||||
url,
|
||||
timeout=self.timeout,
|
||||
impersonate=self.impersonate,
|
||||
proxy=proxy,
|
||||
timeout=10,
|
||||
headers={
|
||||
"User-Agent": USERAGENT,
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
self.proxy_tip = (_("代理 {0} 测试成功").format(proxy),)
|
||||
return proxy
|
||||
except TimeoutException:
|
||||
except Timeout:
|
||||
self.proxy_tip = (
|
||||
_("代理 {0} 测试超时").format(proxy),
|
||||
WARNING,
|
||||
)
|
||||
except (
|
||||
RequestError,
|
||||
HTTPStatusError,
|
||||
) as e:
|
||||
except RequestException as e:
|
||||
self.proxy_tip = (
|
||||
_("代理 {0} 测试失败：{1}").format(
|
||||
proxy,
|
||||
@@ -276,6 +256,24 @@ class Manager:
|
||||
)
|
||||
return None
|
||||
|
||||
def get_headers(
|
||||
self,
|
||||
url: str = "",
|
||||
) -> dict:
|
||||
headers = self.blank_headers.copy()
|
||||
headers["referer"] = get_site_referer(url)
|
||||
return headers
|
||||
|
||||
def __check_impersonate(self, impersonate: str) -> str:
|
||||
if impersonate in get_args(BrowserTypeLiteral):
|
||||
return impersonate
|
||||
logging(
|
||||
self.print,
|
||||
_("impersonate 参数错误，使用默认值: {0}").format(IMPERSONATE),
|
||||
WARNING,
|
||||
)
|
||||
return IMPERSONATE
|
||||
|
||||
def print_proxy_tip(
|
||||
self,
|
||||
) -> None:
|
||||
|
||||
@@ -5,8 +5,8 @@ class ExtractParams(BaseModel):
|
||||
url: str
|
||||
download: bool = False
|
||||
index: list[str | int] | None = None
|
||||
cookie: str = None
|
||||
proxy: str = None
|
||||
cookie: str | None = None
|
||||
proxy: str | None = None
|
||||
check_record: bool = True
|
||||
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ from pathlib import Path
|
||||
from platform import system
|
||||
from shutil import move
|
||||
|
||||
from .static import USERAGENT, VOLUME
|
||||
from .static import IMPERSONATE, VOLUME
|
||||
|
||||
__all__ = ["Settings"]
|
||||
|
||||
@@ -15,9 +15,7 @@ class Settings:
|
||||
"work_path": "", # 工作目录路径
|
||||
"folder_name": "Download", # 下载文件夹名称
|
||||
"name_format": "发布时间 作者昵称 作品标题", # 文件命名格式
|
||||
"user_agent": USERAGENT, # 请求头
|
||||
# "a_user_agent": USERAGENT, # 请求头
|
||||
# "b_user_agent": USERAGENT, # 请求头
|
||||
"impersonate": IMPERSONATE, # 浏览器指纹
|
||||
"cookie": "", # Cookie
|
||||
"proxy": None, # 代理设置
|
||||
"timeout": 10, # 超时时间(秒)
|
||||
|
||||
@@ -22,16 +22,11 @@ RELEASES = "https://github.com/JoeanAmier/XHS-Downloader/releases/latest"
|
||||
|
||||
USERSCRIPT = "https://raw.githubusercontent.com/JoeanAmier/XHS-Downloader/master/static/XHS-Downloader.js"
|
||||
|
||||
USERAGENT = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 "
|
||||
"Safari/537.36 Edg/143.0.0.0"
|
||||
)
|
||||
IMPERSONATE = "chrome146"
|
||||
|
||||
HEADERS = {
|
||||
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,"
|
||||
"application/signed-exchange;v=b3;q=0.7",
|
||||
"referer": "https://www.xiaohongshu.com/explore",
|
||||
"user-agent": USERAGENT,
|
||||
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
|
||||
"accept-language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6",
|
||||
}
|
||||
|
||||
MASTER = "#fff200"
|
||||
|
||||
@@ -51,6 +51,12 @@ def logging(log: Callable, text, style=INFO):
|
||||
)
|
||||
|
||||
|
||||
def get_site_referer(url: str | None) -> str:
|
||||
if "rednote" in (url or "").lower():
|
||||
return "https://www.rednote.com/"
|
||||
return "https://www.xiaohongshu.com/"
|
||||
|
||||
|
||||
def compare_versions(
|
||||
current_version: str,
|
||||
target_version: str,
|
||||
|
||||
+2
-2
@@ -1074,7 +1074,7 @@
|
||||
document.getElementById("settingsRecordData").checked = Boolean(settings.record_data);
|
||||
document.getElementById("settingsWriteMtime").checked = Boolean(settings.write_mtime);
|
||||
document.getElementById("settingsCookie").value = settings.cookie || "";
|
||||
document.getElementById("settingsUserAgent").value = settings.user_agent || "";
|
||||
document.getElementById("settingsImpersonate").value = settings.impersonate || "";
|
||||
document.getElementById("settingsProxy").value = settings.proxy || "";
|
||||
document.getElementById("settingsTimeout").value = String(settings.timeout);
|
||||
document.getElementById("settingsMaxRetry").value = String(settings.max_retry);
|
||||
@@ -1124,7 +1124,7 @@
|
||||
record_data: document.getElementById("settingsRecordData").checked,
|
||||
write_mtime: document.getElementById("settingsWriteMtime").checked,
|
||||
cookie: document.getElementById("settingsCookie").value.trim(),
|
||||
user_agent: document.getElementById("settingsUserAgent").value.trim(),
|
||||
impersonate: document.getElementById("settingsImpersonate").value.trim(),
|
||||
proxy: document.getElementById("settingsProxy").value.trim() || null,
|
||||
timeout: Number(document.getElementById("settingsTimeout").value || 10),
|
||||
max_retry: Number(document.getElementById("settingsMaxRetry").value || 5),
|
||||
|
||||
@@ -561,7 +561,7 @@
|
||||
<label class="field full"><span data-i18n="settings.cookie"></span><textarea
|
||||
id="settingsCookie" rows="4" spellcheck="false"
|
||||
autocomplete="off"></textarea></label>
|
||||
<label class="field full"><span>User-Agent</span><input id="settingsUserAgent"
|
||||
<label class="field full"><span>impersonate</span><input id="settingsImpersonate"
|
||||
type="text"></label>
|
||||
<label class="field"><span data-i18n="settings.proxy"></span><input id="settingsProxy"
|
||||
type="text"
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 38 KiB |
@@ -496,28 +496,6 @@ wheels = [
|
||||
{ url = "https://mirrors.ustc.edu.cn/pypi/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "h2"
|
||||
version = "4.3.0"
|
||||
source = { registry = "https://mirrors.ustc.edu.cn/pypi/simple" }
|
||||
dependencies = [
|
||||
{ name = "hpack" },
|
||||
{ name = "hyperframe" },
|
||||
]
|
||||
sdist = { url = "https://mirrors.ustc.edu.cn/pypi/packages/1d/17/afa56379f94ad0fe8defd37d6eb3f89a25404ffc71d4d848893d270325fc/h2-4.3.0.tar.gz", hash = "sha256:6c59efe4323fa18b47a632221a1888bd7fde6249819beda254aeca909f221bf1", size = 2152026, upload-time = "2025-08-23T18:12:19.778Z" }
|
||||
wheels = [
|
||||
{ url = "https://mirrors.ustc.edu.cn/pypi/packages/69/b2/119f6e6dcbd96f9069ce9a2665e0146588dc9f88f29549711853645e736a/h2-4.3.0-py3-none-any.whl", hash = "sha256:c438f029a25f7945c69e0ccf0fb951dc3f73a5f6412981daee861431b70e2bdd", size = 61779, upload-time = "2025-08-23T18:12:17.779Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hpack"
|
||||
version = "4.1.0"
|
||||
source = { registry = "https://mirrors.ustc.edu.cn/pypi/simple" }
|
||||
sdist = { url = "https://mirrors.ustc.edu.cn/pypi/packages/2c/48/71de9ed269fdae9c8057e5a4c0aa7402e8bb16f2c6e90b3aa53327b113f8/hpack-4.1.0.tar.gz", hash = "sha256:ec5eca154f7056aa06f196a557655c5b009b382873ac8d1e66e79e87535f1dca", size = 51276, upload-time = "2025-01-22T21:44:58.347Z" }
|
||||
wheels = [
|
||||
{ url = "https://mirrors.ustc.edu.cn/pypi/packages/07/c6/80c95b1b2b94682a72cbdbfb85b81ae2daffa4291fbfa1b1464502ede10d/hpack-4.1.0-py3-none-any.whl", hash = "sha256:157ac792668d995c657d93111f46b4535ed114f0c9c8d672271bbec7eae1b496", size = 34357, upload-time = "2025-01-22T21:44:56.92Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "httpcore"
|
||||
version = "1.0.9"
|
||||
@@ -546,14 +524,6 @@ wheels = [
|
||||
{ url = "https://mirrors.ustc.edu.cn/pypi/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" },
|
||||
]
|
||||
|
||||
[package.optional-dependencies]
|
||||
http2 = [
|
||||
{ name = "h2" },
|
||||
]
|
||||
socks = [
|
||||
{ name = "socksio" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "httpx-sse"
|
||||
version = "0.4.3"
|
||||
@@ -563,15 +533,6 @@ wheels = [
|
||||
{ url = "https://mirrors.ustc.edu.cn/pypi/packages/d2/fd/6668e5aec43ab844de6fc74927e155a3b37bf40d7c3790e49fc0406b6578/httpx_sse-0.4.3-py3-none-any.whl", hash = "sha256:0ac1c9fe3c0afad2e0ebb25a934a59f4c7823b60792691f779fad2c5568830fc", size = 8960, upload-time = "2025-10-10T21:48:21.158Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyperframe"
|
||||
version = "6.1.0"
|
||||
source = { registry = "https://mirrors.ustc.edu.cn/pypi/simple" }
|
||||
sdist = { url = "https://mirrors.ustc.edu.cn/pypi/packages/02/e7/94f8232d4a74cc99514c13a9f995811485a6903d48e5d952771ef6322e30/hyperframe-6.1.0.tar.gz", hash = "sha256:f630908a00854a7adeabd6382b43923a4c4cd4b821fcb527e6ab9e15382a3b08", size = 26566, upload-time = "2025-01-22T21:41:49.302Z" }
|
||||
wheels = [
|
||||
{ url = "https://mirrors.ustc.edu.cn/pypi/packages/48/30/47d0bf6072f7252e6521f3447ccfa40b421b6824517f82854703d0f5a98b/hyperframe-6.1.0-py3-none-any.whl", hash = "sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5", size = 13007, upload-time = "2025-01-22T21:41:47.295Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "idna"
|
||||
version = "3.15"
|
||||
@@ -1480,15 +1441,6 @@ wheels = [
|
||||
{ url = "https://mirrors.ustc.edu.cn/pypi/packages/b7/46/f5af3402b579fd5e11573ce652019a67074317e18c1935cc0b4ba9b35552/secretstorage-3.5.0-py3-none-any.whl", hash = "sha256:0ce65888c0725fcb2c5bc0fdb8e5438eece02c523557ea40ce0703c266248137", size = 15554, upload-time = "2025-11-23T19:02:51.545Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "socksio"
|
||||
version = "1.0.0"
|
||||
source = { registry = "https://mirrors.ustc.edu.cn/pypi/simple" }
|
||||
sdist = { url = "https://mirrors.ustc.edu.cn/pypi/packages/f8/5c/48a7d9495be3d1c651198fd99dbb6ce190e2274d0f28b9051307bdec6b85/socksio-1.0.0.tar.gz", hash = "sha256:f88beb3da5b5c38b9890469de67d0cb0f9d494b78b106ca1845f96c10b91c4ac", size = 19055, upload-time = "2020-04-17T15:50:34.664Z" }
|
||||
wheels = [
|
||||
{ url = "https://mirrors.ustc.edu.cn/pypi/packages/37/c3/6eeb6034408dac0fa653d126c9204ade96b819c936e136c5e8a6897eee9c/socksio-1.0.0-py3-none-any.whl", hash = "sha256:95dc1f15f9b34e8d7b16f06d74b8ccf48f609af32ab33c608d08761c5dcbb1f3", size = 12763, upload-time = "2020-04-17T15:50:31.878Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sse-starlette"
|
||||
version = "3.3.2"
|
||||
@@ -1743,7 +1695,6 @@ dependencies = [
|
||||
{ name = "emoji" },
|
||||
{ name = "fastapi" },
|
||||
{ name = "fastmcp" },
|
||||
{ name = "httpx", extra = ["http2", "socks"] },
|
||||
{ name = "lxml" },
|
||||
{ name = "pyperclip" },
|
||||
{ name = "pywebview" },
|
||||
@@ -1762,7 +1713,6 @@ requires-dist = [
|
||||
{ name = "emoji", specifier = ">=2.15.0" },
|
||||
{ name = "fastapi", specifier = ">=0.139.0" },
|
||||
{ name = "fastmcp", specifier = ">=3.4.4" },
|
||||
{ name = "httpx", extras = ["http2", "socks"], specifier = ">=0.28.1" },
|
||||
{ name = "lxml", specifier = ">=6.1.1" },
|
||||
{ name = "pyperclip", specifier = ">=1.11.0" },
|
||||
{ name = "pywebview", specifier = ">=6.2.1" },
|
||||
|
||||
Reference in New Issue
Block a user