From d1bca5a896db4980854afc27ea08b45c2bb15a07 Mon Sep 17 00:00:00 2001 From: JoeanAmier Date: Sun, 4 Aug 2024 15:15:39 +0800 Subject: [PATCH] =?UTF-8?q?perf:=20=E4=BC=98=E5=8C=96=E9=87=8D=E5=AE=9A?= =?UTF-8?q?=E5=90=91=E9=93=BE=E6=8E=A5=E8=8E=B7=E5=8F=96=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. 更新 chunk 参数默认值 2. 更新配置文件参数说明 Closes #138 Closes #139 --- README.md | 8 +++++++- README_EN.md | 8 +++++++- source/application/request.py | 32 ++++++++++++++++++++++++-------- source/module/settings.py | 2 +- 4 files changed, 39 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 705f009..6e9e522 100644 --- a/README.md +++ b/README.md @@ -287,7 +287,7 @@ async def example(): chunk int 下载文件时，每次从服务器获取的数据块大小，单位：字节 -1048576(1 MB) +2097152(2 MB) max_retry @@ -332,6 +332,12 @@ async def example(): false +download_record +bool +是否记录下载成功的作品 ID，如果开启，程序将会自动跳过下载存在记录的作品 +true + + language str 设置程序语言，目前支持：zh_CN、en_GB diff --git a/README_EN.md b/README_EN.md index e69de45..1a58b8c 100644 --- a/README_EN.md +++ b/README_EN.md @@ -291,7 +291,7 @@ async def example(): chunk int Size of data chunk to fetch from the server each time when downloading files, in bytes -1048576(1 MB) +2097152(2 MB) max_retry @@ -336,6 +336,12 @@ async def example(): false +download_record +bool +Do record the ID of successfully downloaded works? If enabled, the program will automatically skip downloading works with records +true + + language str Set program language. Currently supported: zh_CN, en_GB diff --git a/source/application/request.py b/source/application/request.py index 4807e7c..6ac702c 100644 --- a/source/application/request.py +++ b/source/application/request.py @@ -24,14 +24,16 @@ class Html: log=None, **kwargs, ) -> str: + headers = self.select_headers(url, ) try: - response = await self.client.get( - url, - headers=self.select_headers(url, ), - **kwargs, - ) - response.raise_for_status() - return response.text if content else str(response.url) + match content: + case True: + response = await self.__request_url_get(url, headers, **kwargs, ) + response.raise_for_status() + return response.text + case False: + response = await self.__request_url_head(url, headers, **kwargs, ) + return str(response.url) except HTTPError as error: logging(log, str(error), ERROR) logging( @@ -43,4 +45,18 @@ class Html: return bytes(url, "utf-8").decode("unicode_escape") def select_headers(self, url: str) -> dict: - return self.blank_headers if "discovery/item" in url else self.headers + return self.headers if "explore" in url else self.blank_headers + + async def __request_url_head(self, url: str, headers: dict, **kwargs, ): + return await self.client.head( + url, + headers=headers, + **kwargs, + ) + + async def __request_url_get(self, url: str, headers: dict, **kwargs, ): + return await self.client.get( + url, + headers=headers, + **kwargs, + ) diff --git a/source/module/settings.py b/source/module/settings.py index 681115a..141b37d 100644 --- a/source/module/settings.py +++ b/source/module/settings.py @@ -22,7 +22,7 @@ class Settings: "cookie": "", "proxy": None, "timeout": 10, - "chunk": 1024 * 1024, + "chunk": 1024 * 1024 * 2, "max_retry": 5, "record_data": False, "image_format": "PNG",