highperfocused/nostrdvm
1
0
Fork 0
mirror of https://github.com/believethehype/nostrdvm.git synced 2025-06-22 14:10:47 +02:00
Code Issues Packages Projects Releases Wiki Activity

run scrapper on browserless systems (where supported)

Browse Source
This commit is contained in:
Believethehype 2024-11-07 10:30:30 +01:00
parent 93c9571753
commit fbedb7a73f
2 changed files with 53 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
nostr_dvm/utils/scrapper/media_scrapper.py
View File

@ -11,6 +11,7 @@ import yt_dlp
from tqdm import tqdm from tqdm import tqdm
browser = "chrome" # "firefox" browser = "chrome" # "firefox"
usebrowser = False
def download_xvideo(url, target_location) -> None: def download_xvideo(url, target_location) -> None:
@ -124,9 +125,13 @@ def YTDownload(link, path, audio_only=True):
def get_media_duration(url): def get_media_duration(url):
try: try:
# ℹ️ See help(yt_dlp.YoutubeDL) for a list of available options and public functions # ℹ️ See help(yt_dlp.YoutubeDL) for a list of available options and public functions
if usebrowser:
ydl_opts = { ydl_opts = {
'cookiesfrombrowser': (browser, None, None, None), 'cookiesfrombrowser': (browser, None, None, None),
} }
else:
ydl_opts = {}
with yt_dlp.YoutubeDL(ydl_opts) as ydl: with yt_dlp.YoutubeDL(ydl_opts) as ydl:
info = ydl.extract_info(url, download=False) info = ydl.extract_info(url, download=False)
@ -139,9 +144,12 @@ def get_media_duration(url):
def get_media_info(url): def get_media_info(url):
try: try:
# ℹ️ See help(yt_dlp.YoutubeDL) for a list of available options and public functions # ℹ️ See help(yt_dlp.YoutubeDL) for a list of available options and public functions
if usebrowser:
ydl_opts = { ydl_opts = {
'cookiesfrombrowser': (browser, None, None, None), 'cookiesfrombrowser': (browser, None, None, None),
} }
else:
ydl_opts = {}
with yt_dlp.YoutubeDL(ydl_opts) as ydl: with yt_dlp.YoutubeDL(ydl_opts) as ydl:
info = ydl.extract_info(url, download=False) info = ydl.extract_info(url, download=False)
@ -153,6 +161,7 @@ def get_media_info(url):
def get_audio(URLS): def get_audio(URLS):
try: try:
if usebrowser:
ydl_opts = { ydl_opts = {
'cookiesfrombrowser': (browser, None, None, None), 'cookiesfrombrowser': (browser, None, None, None),
'format': 'm4a/bestaudio/best', 'format': 'm4a/bestaudio/best',
@ -164,6 +173,17 @@ def get_audio(URLS):
'preferredcodec': 'mp3', 'preferredcodec': 'mp3',
}] }]
} }
else:
ydl_opts = {
'format': 'm4a/bestaudio/best',
"outtmpl": 'outputs/audio',
'overwrites': 'True',
# ℹ️ See help(yt_dlp.postprocessor) for a list of available Postprocessors and their arguments
'postprocessors': [{ # Extract audio using ffmpeg
'key': 'FFmpegExtractAudio',
'preferredcodec': 'mp3',
}]
}
with yt_dlp.YoutubeDL(ydl_opts) as ydl: with yt_dlp.YoutubeDL(ydl_opts) as ydl:
error_code = ydl.download(URLS) error_code = ydl.download(URLS)
@ -174,13 +194,21 @@ def get_audio(URLS):
def get_video(URLS): def get_video(URLS):
try: try:
if usebrowser:
ydl_opts = { ydl_opts = {
'cookiesfrombrowser': (browser, None, None, None), 'cookiesfrombrowser': (browser, None, None, None),
'format': 'mp4', 'format': 'mp4',
'overwrites': 'True', 'overwrites': 'True',
# "outtmpl": '/%(uploader)s_%(title)s.%(ext)s',
"outtmpl": 'outputs/video.mp4', "outtmpl": 'outputs/video.mp4',
} }
else:
ydl_opts = {
'format': 'mp4',
'overwrites': 'True',
"outtmpl": 'outputs/video.mp4',
}
with yt_dlp.YoutubeDL(ydl_opts) as ydl: with yt_dlp.YoutubeDL(ydl_opts) as ydl:
ydl.download(URLS) ydl.download(URLS)

2
setup.py
View File

@ -17,7 +17,7 @@ setup(
install_requires=["nostr-sdk==0.36.0", install_requires=["nostr-sdk==0.36.0",
"bech32==1.2.0", "bech32==1.2.0",
"pycryptodome==3.20.0", "pycryptodome==3.20.0",
"yt-dlp==2024.5.27", "yt-dlp==2024.11.04",
"python-dotenv==1.0.0", "python-dotenv==1.0.0",
"emoji==2.12.1", "emoji==2.12.1",
"ffmpegio==0.9.1", "ffmpegio==0.9.1",