From a21854df314189dd52dcbbe751c72cf91c7fa226 Mon Sep 17 00:00:00 2001 From: ihmily <961532186@qq.com> Date: Sun, 3 Dec 2023 21:35:27 +0800 Subject: [PATCH] Add xiaohongshu live record --- README.md | 15 +++++++-- main.py | 97 ++++++++++++++++++++++++------------------------------- spider.py | 53 +++++++++++++++++++++++++++--- utils.py | 46 ++++++++++++++++++++++++++ 4 files changed, 149 insertions(+), 62 deletions(-) create mode 100644 utils.py diff --git a/README.md b/README.md index ef0d864..532be5a 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ - [x] 斗鱼 - [x] YY - [x] B站 +- [x] 小红书 - [ ] 更多平台正在更新中 @@ -34,6 +35,7 @@ ├── /libs -> (dll file) ├── main.py -> (main file) ├── spider.py-> (get live url) + ├── utils.py -> (contains utility functions) ├── web_rid.py -> (get web_rid) ├── msg_push.py -> (send live status update message) ├── cookies.py -> (get douyin cookies) @@ -51,12 +53,12 @@ - 在 `config` 文件夹内的配置文件中对录制进行配置，并在 `URL_config.ini` 中添加录制直播间地址。 - 抖音录制需要使用到PC网页端直播间页面的Cookie，请先在config.ini配置文件中添加后再进行抖音录制（有默认的cookie，但最好还是自己添加自己的） - 录制Tiktok时需要科学上网，请先在配置文件中设置开启代理并添加proxy_addr链接 如：`http://127.0.0.1:7890` -- 可以在URL_config.ini中的链接开头加上#，此时将不会录制该条链接对应的直播 +- 可以在URL_config.ini中的链接开头加上#，此时将不会录制该条链接对应的直播 - 测试时有可能会出现在IDE如Pycharm中运行代码进行直播录制，录制出来的视频却无法正常播放的现象，如果遇到这个问题 在命令控制台DOS界面运行代码，录制出来的视频即可正常播放。 - 当同时在录制多个直播时，最好线程数设置大一些，否则可能出现其中一个直播录制出错。当然设置的过大也没用，要同时考虑自身电脑的配置，如CPU内核数、网络带宽等限制。 - 如果想直接使用打包好的录制软件，进入[Releases](https://github.com/ihmily/DouyinLiveRecorder/releases) 下载最新发布的 zip压缩包即可，有些电脑可能会报毒，直接忽略即可。 - 如果要长时间挂着软件循环监测直播，最好循环时间设置长一点，避免因请求频繁导致被官方封禁IP 。 -- 最后，欢迎大家提交PR，一起维护该仓库！ +- 最后，欢迎大家提交PR   @@ -85,6 +87,9 @@ https://www.yy.com/22490906/22490906 B站： https://live.bilibili.com/320 + +小红书： +https://www.xiaohongshu.com/hina/livestream/568980065082002402?appuid=5f3f478a00000000010005b3&apptime= ``` Tiktok目前只支持PC网页端地址（我没下载app），其他平台 app端直播间分享地址和网页端长地址都能正常进行录制（抖音尽量用长链接，避免因短链接转换失效导致不能正常录制）。 @@ -125,9 +130,13 @@ GET https://hmily.vip/api/jx/live/convert.php?url=https://v.douyin.com/iQLgKSj/ ## ⏳提交日志 +- 20231203 + - 新增小红书直播录制（全网首发），目前小红书官方没有切换清晰度功能，因此直播录制也只有默认画质 + - 小红书录制暂时无法循环监测，每次主播开启直播，都要重新获取一次链接 + - 获取链接的方式为 将直播间转发到微信，在微信中打开后，复制页面的链接。 - 20231030 - 本次更新只是进行修复，没时间新增功能。 - - 欢迎各位大佬提pr 帮忙更新维护 ，Come on ！ + - 欢迎各位大佬提pr 帮忙更新维护 - 20230930 - 新增抖音从接口获取直播流，增强稳定性 diff --git a/main.py b/main.py index dbf863d..7a5fb68 100644 --- a/main.py +++ b/main.py @@ -4,46 +4,49 @@ Author: Hmily GitHub: https://github.com/ihmily Date: 2023-07-17 23:52:05 -Update: 2023-10-31 01:56:37 +Update: 2023-12-03 20:46:00 Copyright (c) 2023 by Hmily, All Rights Reserved. Function: Record live stream video. """ -import functools import random import os import sys -import traceback import urllib.parse import configparser import subprocess import threading import logging import datetime +import time +import json +import re import shutil -from spider import * -from web_rid import * -from msg_push import * +from spider import ( + get_douyin_stream_data, + get_tiktok_stream_data, + get_kuaishou_stream_data2, + get_huya_stream_data, + get_douyu_info_data, + get_douyu_stream_data, + get_yy_stream_data, + get_bilibili_stream_data, + get_xhs_stream_url +) + +from web_rid import ( + get_live_room_id, + get_sec_user_id +) +from utils import ( + logger, check_md5, + trace_error_decorator +) +from msg_push import dingtalk, xizhi # 版本号 -version = "v2.0.2" -platforms = "抖音|Tiktok|快手|虎牙|斗鱼|YY|B站" - -# --------------------------log日志------------------------------------- -# 创建一个logger -logger = logging.getLogger('DouyinLiveRecorder直播录制%s版' % str(version)) -logger.setLevel(logging.INFO) -# 创建一个handler，用于写入日志文件 -if not os.path.exists("./log"): - os.makedirs("./log") -fh = logging.FileHandler("./log/错误日志文件.log", encoding="utf-8-sig", mode="a") -fh.setLevel(logging.WARNING) -# 定义handler的输出格式 -formatter = logging.Formatter('%(asctime)s - %(message)s') -fh.setFormatter(formatter) -# 给logger添加handler -logger.addHandler(fh) - +version = "v2.0.3" +platforms = "抖音|Tiktok|快手|虎牙|斗鱼|YY|B站|小红书" # --------------------------全局变量------------------------------------- recording = set() unrecording = set() @@ -73,20 +76,6 @@ default_path = os.getcwd() # --------------------------用到的函数------------------------------------- -def trace_error_decorator(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - try: - return func(*args, **kwargs) - except Exception as e: - error_line = traceback.extract_tb(e.__traceback__)[-1].lineno - error_info = f"错误信息: type: {type(e).__name__}, {str(e)} in function {func.__name__} at line: {error_line}" - print(error_info) - logger.warning(error_info) - return [] - - return wrapper - def display_info(): # TODO: 显示当前录制配置信息 @@ -314,7 +303,7 @@ def get_tiktok_stream_url(json_data): quality_list = list(stream_data.keys()) # ["origin","uhd","sd","ld"] while len(quality_list) < 4: quality_list.append(quality_list[-1]) - video_qualities = {"原画": 0,"蓝光": 0,"超清": 1,"高清": 2,"标清": 3} + video_qualities = {"原画": 0, "蓝光": 0, "超清": 1, "高清": 2, "标清": 3} quality_index = video_qualities.get(video_quality) quality_key = quality_list[quality_index] video_quality_urls = get_video_quality_url(stream_data, quality_key) @@ -598,7 +587,11 @@ def start_record(url_tuple, count_variable=-1): json_data = get_bilibili_stream_data(record_url, bili_cookie) port_info = get_bilibili_stream_url(json_data) - anchor_name = port_info.get("anchor_name", '') + elif record_url.find("https://www.xiaohongshu.com/") > -1: + with semaphore: + port_info = get_xhs_stream_url(record_url, xhs_cookie) + + anchor_name:str= port_info.get("anchor_name", '') if not anchor_name: print(f'序号{count_variable} 网址内容获取失败,进行重试中...获取失败的地址是:{url_tuple}') @@ -652,8 +645,10 @@ def start_record(url_tuple, count_variable=-1): logger.warning(f"错误信息: {e} 发生错误的行数: {e.__traceback__.tb_lineno}") if not os.path.exists(full_path): - print("保存路径不存在,不能生成录制.请避免把本程序放在c盘,桌面,下载文件夹,qq默认传输目录.请重新检查设置") - logger.warning("错误信息: 保存路径不存在,不能生成录制.请避免把本程序放在c盘,桌面,下载文件夹,qq默认传输目录.请重新检查设置") + print( + "保存路径不存在,不能生成录制.请避免把本程序放在c盘,桌面,下载文件夹,qq默认传输目录.请重新检查设置") + logger.warning( + "错误信息: 保存路径不存在,不能生成录制.请避免把本程序放在c盘,桌面,下载文件夹,qq默认传输目录.请重新检查设置") ffmpeg_command = [ ffmpeg_path, "-y", @@ -883,7 +878,8 @@ def start_record(url_tuple, count_variable=-1): except subprocess.CalledProcessError as e: logging.warning(str(e.output)) - logger.warning(f"错误信息: {e} 发生错误的行数: {e.__traceback__.tb_lineno}") + logger.warning( + f"错误信息: {e} 发生错误的行数: {e.__traceback__.tb_lineno}") break @@ -976,15 +972,6 @@ def start_record(url_tuple, count_variable=-1): time.sleep(2) -def check_md5(file_path): - """ - 计算文件的md5值 - """ - with open(file_path, 'rb') as fp: - file_md5 = hashlib.md5(fp.read()).hexdigest() - return file_md5 - - def backup_file(file_path, backup_dir): """ 备份配置文件到备份目录 @@ -1131,6 +1118,7 @@ while True: douyu_cookie = read_config_value(config, 'Cookie', '斗鱼cookie', '') yy_cookie = read_config_value(config, 'Cookie', 'YY_cookie', '') bili_cookie = read_config_value(config, 'Cookie', 'B站cookie', '') + xhs_cookie = read_config_value(config, 'Cookie', '小红书cookie', '') if len(video_save_type) > 0: if video_save_type.upper().lower() == "FLV".lower(): @@ -1202,7 +1190,8 @@ while True: 'www.huya.com', 'www.douyu.com', 'www.yy.com', - 'live.bilibili.com' + 'live.bilibili.com', + 'www.xiaohongshu.com' ] if url_host in host_list: new_line = (url, split_line[1]) @@ -1250,5 +1239,3 @@ while True: firstRunOtherLine = False time.sleep(3) - - diff --git a/spider.py b/spider.py index 0aa9d51..0da5b82 100644 --- a/spider.py +++ b/spider.py @@ -4,7 +4,7 @@ Author: Hmily GitHub:https://github.com/ihmily Date: 2023-07-15 23:15:00 -Update: 2023-10-31 01:55:19 +Update: 2023-12-03 20:48:35 Copyright (c) 2023 by Hmily, All Rights Reserved. Function: Get live stream data. """ @@ -18,11 +18,13 @@ import re import json import execjs import urllib.request +from utils import trace_error_decorator no_proxy_handler = urllib.request.ProxyHandler({}) opener = urllib.request.build_opener(no_proxy_handler) +@trace_error_decorator def get_douyin_stream_data(url: str, cookies: Union[str, None] = None) -> Dict[str, Any]: headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0', @@ -64,6 +66,7 @@ def get_douyin_stream_data(url: str, cookies: Union[str, None] = None) -> Dict[s return room_data +@trace_error_decorator def get_tiktok_stream_data(url: str, proxy_addr: Union[str, None] = None, cookies: Union[str, None] = None) -> Dict[ str, Any]: headers = { @@ -94,6 +97,7 @@ def get_tiktok_stream_data(url: str, proxy_addr: Union[str, None] = None, cookie return json_data +@trace_error_decorator def get_kuaishou_stream_data(url: str, cookies: Union[str, None] = None) -> Dict[str, Any]: headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0', @@ -130,6 +134,7 @@ def get_kuaishou_stream_data(url: str, cookies: Union[str, None] = None) -> Dict return result +@trace_error_decorator def get_kuaishou_stream_data2(url: str, cookies: Union[str, None] = None) -> Dict[str, Any]: headers = { 'User-Agent': 'Mozilla/5.0 (Linux; Android 11; SAMSUNG SM-G973U) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/14.2 Chrome/87.0.4280.141 Mobile Safari/537.36', @@ -176,6 +181,7 @@ def get_kuaishou_stream_data2(url: str, cookies: Union[str, None] = None) -> Dic return get_kuaishou_stream_data(url, cookies=cookies) +@trace_error_decorator def get_huya_stream_data(url: str, cookies: Union[str, None] = None) -> Dict[str, Any]: headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0', @@ -227,6 +233,7 @@ def get_token_js(rid: str, did: str) -> Union[list, Dict[str, Any]]: return params_list +@trace_error_decorator def get_douyu_info_data(url: str) -> Dict[str, Any]: match_rid = re.search('rid=(.*?)&', url) if match_rid: @@ -247,6 +254,7 @@ def get_douyu_info_data(url: str) -> Dict[str, Any]: return json_data +@trace_error_decorator def get_douyu_stream_data(rid: str, rate: str = '-1', cookies: Union[str, None] = None) -> Dict[str, Any]: did = '10000000000000000000000000003306' params_list = get_token_js(rid, did) @@ -278,6 +286,7 @@ def get_douyu_stream_data(rid: str, rate: str = '-1', cookies: Union[str, None] return json_data +@trace_error_decorator def get_yy_stream_data(url: str, cookies: Union[str, None] = None) -> Dict[str, Any]: cid = re.search('yy.com/(.*?)/', url).group(1) @@ -307,6 +316,7 @@ def get_yy_stream_data(url: str, cookies: Union[str, None] = None) -> Dict[str, return json_data +@trace_error_decorator def get_bilibili_stream_data(url: str, cookies: Union[str, None] = None) -> Dict[str, Any]: headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0', @@ -324,6 +334,40 @@ def get_bilibili_stream_data(url: str, cookies: Union[str, None] = None) -> Dict return json_data +@trace_error_decorator +def get_xhs_stream_url(url: str, cookies: Union[str, None] = None) -> Dict[str, Any]: + headers = { + 'User-Agent': 'Mozilla/5.0 (Linux; Android 11; SAMSUNG SM-G973U) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/14.2 Chrome/87.0.4280.141 Mobile Safari/537.36', + 'Accept': 'application/json, text/plain, */*', + 'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2', + 'Referer': 'https://www.xiaohongshu.com/hina/livestream/568979931846654360', + } + if cookies: + headers['Cookie'] = cookies + + room_id = url.split('?')[0].rsplit('/',maxsplit=1)[1] + appuid = re.search('appuid=(.*?)&', url).group(1) + url = f'https://www.xiaohongshu.com/api/sns/red/live/app/v1/ecology/outside/share_info?room_id={room_id}' + req = urllib.request.Request(url, headers=headers) + response = opener.open(req, timeout=15) + json_str = response.read().decode('utf-8') + json_data = json.loads(json_str) + anchor_name = json_data['data']['host_info']['nickname'] + live_status = json_data['data']['room']['status'] + result = { + "anchor_name": anchor_name, + "is_live": False, + } + + # 这个判断不准确，无论是否在直播都为0 + if live_status == 0: + flv_url = f'http://live-play.xhscdn.com/live/{room_id}.flv?uid={appuid}' + result['flv_url'] = flv_url + result['is_live'] = True + result['record_url'] = flv_url + return result + + if __name__ == '__main__': # 尽量用自己的cookie，以避免默认的不可用导致无法获取数据 url = "https://live.douyin.com/745964462470" # 抖音直播 @@ -334,6 +378,9 @@ if __name__ == '__main__': # url = 'https://www.douyu.com/3637778?dyshid' # url = 'https://www.yy.com/22490906/22490906' # YY直播 # url = 'https://live.bilibili.com/21593109' # b站直播 + # 小红书直播 + # url = 'https://www.xiaohongshu.com/hina/livestream/568980065082002402?appuid=5f3f478a00000000010005b3&apptime=' + print(get_douyin_stream_data(url)) # print(get_tiktok_stream_data(url,'http://127.0.0.1:7890')) @@ -343,7 +390,5 @@ if __name__ == '__main__': # print(get_douyu_stream_data("4921614",rate='-1')) # print(get_yy_stream_data(url)) # print(get_bilibili_stream_data(url)) - - - + # print(get_xhs_stream_url(url)) diff --git a/utils.py b/utils.py new file mode 100644 index 0000000..17ba087 --- /dev/null +++ b/utils.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- + +import functools +import hashlib +import logging +import os +import traceback + +# --------------------------log日志------------------------------------- +# 创建一个logger +logger = logging.getLogger('record_logger') +logger.setLevel(logging.INFO) +# 创建一个handler，用于写入日志文件 +if not os.path.exists("./log"): + os.makedirs("./log") +fh = logging.FileHandler("./log/错误日志文件.log", encoding="utf-8-sig", mode="a") +fh.setLevel(logging.WARNING) +# 定义handler的输出格式 +formatter = logging.Formatter('%(asctime)s - %(message)s') +fh.setFormatter(formatter) +# 给logger添加handler +logger.addHandler(fh) + + +def trace_error_decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except Exception as e: + error_line = traceback.extract_tb(e.__traceback__)[-1].lineno + error_info = f"错误信息: type: {type(e).__name__}, {str(e)} in function {func.__name__} at line: {error_line}" + print(error_info) + logger.warning(error_info) + return [] + + return wrapper + + +def check_md5(file_path): + """ + 计算文件的md5值 + """ + with open(file_path, 'rb') as fp: + file_md5 = hashlib.md5(fp.read()).hexdigest() + return file_md5