highperfocused/nostrdvm
1
0
Fork 0
mirror of https://github.com/believethehype/nostrdvm.git synced 2025-08-03 11:53:35 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #17 from believethehype/svd

Browse Source 
add local stable video diffusion
This commit is contained in:
believethehype
2024-01-15 11:04:43 +01:00
committed by GitHub
parent 9a18823e3a a135002f4a
commit 0fec71373a
11 changed files with 268 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
nostr_dvm/backends/nova_server/modules/image_interrogator/requirements.txt
View File

@@ -1,4 +1,4 @@
hcai-nova-utils>=1.5.5
hcai-nova-utils>=1.5.7
--extra-index-url https://download.pytorch.org/whl/cu118
torch==2.1.1
clip_interrogator

2
nostr_dvm/backends/nova_server/modules/image_upscale/requirements.txt
View File

@@ -1,5 +1,5 @@
realesrgan @git+https://github.com/xinntao/Real-ESRGAN.git
hcai-nova-utils>=1.5.5
hcai-nova-utils>=1.5.7
--extra-index-url https://download.pytorch.org/whl/cu118
torch==2.1.0
torchvision

2
nostr_dvm/backends/nova_server/modules/stablediffusionxl/requirements.txt
View File

@@ -1,4 +1,4 @@
hcai-nova-utils>=1.5.5
hcai-nova-utils>=1.5.7
--extra-index-url https://download.pytorch.org/whl/cu118
torch==2.1.0
compel~=2.0.2

7
nostr_dvm/backends/nova_server/modules/stablevideodiffusion/requirements.txt Normal file
View File

@@ -0,0 +1,7 @@
hcai-nova-utils>=1.5.7
--extra-index-url https://download.pytorch.org/whl/cu118
torch==2.1.0
git+https://github.com/huggingface/diffusers.git
transformers
accelerate
opencv-python

100
nostr_dvm/backends/nova_server/modules/stablevideodiffusion/stablevideodiffusion.py Normal file
View File

@@ -0,0 +1,100 @@
import gc
import sys
import os
sys.path.insert(0, os.path.dirname(__file__))
from ssl import Options
from nova_utils.interfaces.server_module import Processor
import torch
from diffusers import StableVideoDiffusionPipeline
from diffusers.utils import load_image, export_to_video
from nova_utils.utils.cache_utils import get_file
import numpy as np
from PIL import Image as PILImage
# Setting defaults
_default_options = {"model": "stabilityai/stable-video-diffusion-img2vid-xt", "fps":"7", "seed":""}
# TODO: add log infos,
class StableVideoDiffusion(Processor):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.options = _default_options | self.options
self.device = None
self.ds_iter = None
self.current_session = None
# IO shortcuts
self.input = [x for x in self.model_io if x.io_type == "input"]
self.output = [x for x in self.model_io if x.io_type == "output"]
self.input = self.input[0]
self.output = self.output[0]
def process_data(self, ds_iter) -> dict:
self.device = "cuda" if torch.cuda.is_available() else "cpu"
self.ds_iter = ds_iter
current_session_name = self.ds_iter.session_names[0]
self.current_session = self.ds_iter.sessions[current_session_name]['manager']
input_image = self.current_session.input_data['input_image'].data
try:
pipe = StableVideoDiffusionPipeline.from_pretrained(
self.options["model"], torch_dtype=torch.float16, variant="fp16"
)
pipe.enable_model_cpu_offload()
# Load the conditioning image
image = PILImage.fromarray(input_image)
image = image.resize((1024, 576))
if self.options["seed"] != "" and self.options["seed"] != " ":
generator = torch.manual_seed(int(self.options["seed"]))
frames = pipe(image, decode_chunk_size=8, generator=generator).frames[0]
else:
frames = pipe(image, decode_chunk_size=8).frames[0]
if torch.cuda.is_available():
del pipe
gc.collect()
torch.cuda.empty_cache()
torch.cuda.ipc_collect()
np_video = np.stack([np.asarray(x) for x in frames])
return np_video
except Exception as e:
print(e)
sys.stdout.flush()
return "Error"
def calculate_aspect(self, width: int, height: int):
def gcd(a, b):
"""The GCD (greatest common divisor) is the highest number that evenly divides both width and height."""
return a if b == 0 else gcd(b, a % b)
r = gcd(width, height)
x = int(width / r)
y = int(height / r)
return x, y
def to_output(self, data: list):
video = self.current_session.output_data_templates['output_video']
video.data = data
video.meta_data.sample_rate = int(self.options['fps'])
video.meta_data.media_type = 'video'
return self.current_session.output_data_templates

9
nostr_dvm/backends/nova_server/modules/stablevideodiffusion/stablevideodiffusion.trainer Normal file
View File

@@ -0,0 +1,9 @@
<?xml version="1.0" ?>
<trainer ssi-v="5">
<info trained="true" seed="1234"/>
<meta backend="nova-server" category="VideoGeneration" description="Generates Video from Image/prompt" is_iterable="False">
<io type="input" id="input_image" data="Image" default_value=""/>
<io type="output" id="output_video" data="stream:Video" default_value="sd_generated.mp4"/>
</meta>
<model create="StableVideoDiffusion" script="stablevideodiffusion.py" optstr="{model:LIST:stabilityai/stable-video-diffusion-img2vid-xt,stabilityai/stable-video-diffusion-img2vid};{fps:STRING:7};{seed:STRING: }"/>
</trainer>

12
nostr_dvm/backends/nova_server/modules/stablevideodiffusion/version.py Normal file
View File

@@ -0,0 +1,12 @@
""" Stable Video Diffusion
"""
# We follow Semantic Versioning (https://semver.org/)
_MAJOR_VERSION = '1'
_MINOR_VERSION = '0'
_PATCH_VERSION = '0'
__version__ = '.'.join([
_MAJOR_VERSION,
_MINOR_VERSION,
_PATCH_VERSION,
])

2
nostr_dvm/backends/nova_server/modules/whisperx/requirements.txt
View File

@@ -1,4 +1,4 @@
hcai-nova-utils>=1.5.5
hcai-nova-utils>=1.5.7
--extra-index-url https://download.pytorch.org/whl/cu118
torch==2.1.0+cu118
torchvision>= 0.15.1+cu118

12
nostr_dvm/backends/nova_server/utils.py
View File

@@ -6,6 +6,7 @@ import zipfile
import pandas as pd
import requests
import PIL.Image as Image
from moviepy.video.io.VideoFileClip import VideoFileClip
from nostr_dvm.utils.output_utils import upload_media_to_hoster
@@ -87,6 +88,17 @@ def check_server_status(jobID, address) -> str | pd.DataFrame:
result = upload_media_to_hoster("./outputs/image.jpg")
os.remove("./outputs/image.jpg")
return result
elif content_type == 'video/mp4':
with open('./outputs/video.mp4', 'wb') as f:
f.write(response.content)
f.close()
clip = VideoFileClip("./outputs/video.mp4")
clip.write_videofile("./outputs/video2.mp4")
result = upload_media_to_hoster("./outputs/video2.mp4")
clip.close()
os.remove("./outputs/video.mp4")
os.remove("./outputs/video2.mp4")
return result
elif content_type == 'text/plain; charset=utf-8':
return response.content.decode('utf-8')
elif content_type == "application/x-zip-compressed":

1
nostr_dvm/tasks/README.md
View File

@@ -25,6 +25,7 @@ Current List of Tasks:
| ImageUpscale | 5100 | Upscales an Image | nserver |
| MediaConverter | 5200 | Converts a link of a media file and uploads it | openAI |
| VideoGenerationReplicateSVD | 5202 (inoff) | Generates a Video from an Image | replicate |
| VideoGenerationSVD | 5202 (inoff) | Generates a Video from an Image | nserver |
| TextToSpeech | 5250 | Generate Audio from a prompt | local |
| TrendingNotesNostrBand | 5300 | Show trending notes on nostr.band | nostr.band api |
| DiscoverInactiveFollows | 5301 | Find inactive Nostr users | local |

123
nostr_dvm/tasks/videogeneration_svd.py Normal file
View File

@@ -0,0 +1,123 @@
import json
from multiprocessing.pool import ThreadPool
from nostr_dvm.backends.nova_server.utils import check_server_status, send_request_to_server
from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface, process_venv
from nostr_dvm.utils.admin_utils import AdminConfig
from nostr_dvm.utils.dvmconfig import DVMConfig, build_default_config
from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag
from nostr_dvm.utils.definitions import EventDefinitions
"""
This File contains a module to transform an Image to a short Video Clip on n-server and receive results back.
Accepted Inputs: An url to an Image
Outputs: An url to a video
,
"""
class VideoGenerationSVD(DVMTaskInterface):
KIND: int = EventDefinitions.KIND_NIP90_GENERATE_VIDEO
TASK: str = "image-to-video"
FIX_COST: float = 120
def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config,
admin_config: AdminConfig = None, options=None):
super().__init__(name, dvm_config, nip89config, admin_config, options)
def is_input_supported(self, tags, client=None, dvm_config=None):
for tag in tags:
if tag.as_vec()[0] == 'i':
input_value = tag.as_vec()[1]
input_type = tag.as_vec()[2]
if input_type != "url":
return False
return True
def create_request_from_nostr_event(self, event, client=None, dvm_config=None):
request_form = {"jobID": event.id().to_hex() + "_" + self.NAME.replace(" ", "")}
request_form["trainerFilePath"] = r'modules\stablevideodiffusion\stablevideodiffusion.trainer'
url = ""
frames = 7 # 25
model = "stabilityai/stable-video-diffusion-img2vid-xt" #,stabilityai/stable-video-diffusion-img2vid
for tag in event.tags():
if tag.as_vec()[0] == 'i':
input_type = tag.as_vec()[2]
if input_type == "url":
url = str(tag.as_vec()[1]).split('#')[0]
# TODO add params as defined above
io_input = {
"id": "input_image",
"type": "input",
"src": "url:Image",
"uri": url
}
io_output = {
"id": "output_video",
"type": "output",
"src": "request:stream:Video"
}
request_form['data'] = json.dumps([io_input, io_output])
options = {
"model": model,
"fps": frames
}
request_form['options'] = json.dumps(options)
return request_form
def process(self, request_form):
try:
# Call the process route of n-server with our request form.
response = send_request_to_server(request_form, self.options['server'])
if bool(json.loads(response)['success']):
print("Job " + request_form['jobID'] + " sent to server")
pool = ThreadPool(processes=1)
thread = pool.apply_async(check_server_status, (request_form['jobID'], self.options['server']))
print("Wait for results of server...")
result = thread.get()
return result
except Exception as e:
raise Exception(e)
# We build an example here that we can call by either calling this file directly from the main directory,
# or by adding it to our playground. You can call the example and adjust it to your needs or redefine it in the
# playground or elsewhere
def build_example(name, identifier, admin_config, server_address):
dvm_config = build_default_config(identifier)
dvm_config.USE_OWN_VENV = False
admin_config.LUD16 = dvm_config.LN_ADDRESS
# A module might have options it can be initialized with, here we set a default model, and the server
# address it should use. These parameters can be freely defined in the task component
options = {'server': server_address}
nip89info = {
"name": name,
"image": "https://image.nostr.build/c33ca6fc4cc038ca4adb46fdfdfda34951656f87ee364ef59095bae1495ce669.jpg",
"about": "I create a short video based on an image",
"encryptionSupported": True,
"cashuAccepted": True,
"nip90Params": {}
}
nip89config = NIP89Config()
nip89config.DTAG = check_and_set_d_tag(identifier, name, dvm_config.PRIVATE_KEY, nip89info["image"])
nip89config.CONTENT = json.dumps(nip89info)
return VideoGenerationSVD(name=name, dvm_config=dvm_config, nip89config=nip89config,
admin_config=admin_config, options=options)
if __name__ == '__main__':
process_venv(VideoGenerationSVD)