diff --git a/.idea/dvm.iml b/.idea/dvm.iml index 007729a..5de74eb 100644 --- a/.idea/dvm.iml +++ b/.idea/dvm.iml @@ -4,8 +4,10 @@ + + - + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml index 3ce312c..af05b9d 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -3,5 +3,5 @@ - + \ No newline at end of file diff --git a/main.py b/main.py index edc65c8..79355cf 100644 --- a/main.py +++ b/main.py @@ -1,26 +1,19 @@ import os from pathlib import Path import dotenv -from bot.bot import Bot -import tasks.convert_media as convert_media -import tasks.discovery_inactive_follows as discovery_inactive_follows -import tasks.imagegeneration_openai_dalle as imagegeneration_openai_dalle -import tasks.textextraction_pdf as textextraction_pdf -import tasks.textextraction_google as textextraction_google -import tasks.translation_google as translation_google -import tasks.translation_libretranslate as translation_libretranslate -from tasks import imagegeneration_replicate_sdxl, videogeneration_replicate_svd, trending_notes_nostrband, \ - textgeneration_llmlite - -from utils.admin_utils import AdminConfig -from utils.backend_utils import keep_alive -from utils.definitions import EventDefinitions -from utils.dvmconfig import DVMConfig -from utils.external_dvm_utils import build_external_dvm -from utils.nostr_utils import check_and_set_private_key -from utils.output_utils import PostProcessFunctionType -from utils.zap_utils import check_and_set_ln_bits_keys +from nostr_dvm.bot import Bot +from nostr_dvm.tasks import videogeneration_replicate_svd, imagegeneration_replicate_sdxl, textgeneration_llmlite, \ + trending_notes_nostrband, discovery_inactive_follows, translation_google, textextraction_pdf, \ + translation_libretranslate, textextraction_google, convert_media, imagegeneration_openai_dalle +from nostr_dvm.utils.admin_utils import AdminConfig +from nostr_dvm.utils.backend_utils import keep_alive +from nostr_dvm.utils.definitions import EventDefinitions +from nostr_dvm.utils.dvmconfig import DVMConfig +from nostr_dvm.utils.external_dvm_utils import build_external_dvm +from nostr_dvm.utils.nostr_utils import check_and_set_private_key +from nostr_dvm.utils.output_utils import PostProcessFunctionType +from nostr_dvm.utils.zap_utils import check_and_set_ln_bits_keys from nostr_sdk import Keys @@ -115,7 +108,7 @@ def playground(): # Spawn DVM 7 Find inactive followers googleextractor = textextraction_google.build_example("Extractor", "speech_recognition", - admin_config) + admin_config) bot_config.SUPPORTED_DVMS.append(googleextractor) googleextractor.run() diff --git a/nostr_dvm/__init__.py b/nostr_dvm/__init__.py new file mode 100644 index 0000000..e6ed13d --- /dev/null +++ b/nostr_dvm/__init__.py @@ -0,0 +1,10 @@ +# We follow Semantic Versioning (https://semver.org/) +_MAJOR_VERSION = '0' +_MINOR_VERSION = '0' +_PATCH_VERSION = '1' + +__version__ = '.'.join([ + _MAJOR_VERSION, + _MINOR_VERSION, + _PATCH_VERSION, +]) \ No newline at end of file diff --git a/nostr_dvm/backends/__init__.py b/nostr_dvm/backends/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nostr_dvm/backends/nova_server/__init__.py b/nostr_dvm/backends/nova_server/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nostr_dvm/backends/nova_server/utils.py b/nostr_dvm/backends/nova_server/utils.py new file mode 100644 index 0000000..613032e --- /dev/null +++ b/nostr_dvm/backends/nova_server/utils.py @@ -0,0 +1,112 @@ +import io +import json +import os +import time +import zipfile +import pandas as pd +import requests +import PIL.Image as Image + +from nostr_dvm.utils.output_utils import upload_media_to_hoster + +""" +This file contains basic calling functions for ML tasks that are outsourced to nova server. It is an Open-Source backend +that enables running models locally based on preefined modules, by accepting a request form. +Modules are deployed in in separate virtual environments so dependencies won't conflict. +""" + +""" +send_request_to_n_server(request_form, address) +Function to send a request_form to the server, containing all the information we parsed from the Nostr event and added +in the module that is calling the server + +""" + +def send_request_to_server(request_form, address): + print("Sending job to Server") + url = ('http://' + address + '/process') + headers = {'Content-type': 'application/x-www-form-urlencoded'} + response = requests.post(url, headers=headers, data=request_form) + return response.text + + +def send_file_to_server(filepath, address): + print("Sending file to Server") + url = ('http://' + address + '/upload') + try: + fp = open(filepath, 'rb') + response = requests.post(url, files={'file': fp}) + result = response.content.decode('utf-8') + except Exception as e: + print(e) + print(response.content.decode('utf-8')) + + return result + + # headers = {'Content-type': 'application/x-www-form-urlencoded'} + + +""" +check_n_server_status(request_form, address) +Function that requests the status of the current process with the jobID (we use the Nostr event as jobID). +When the Job is successfully finished we grab the result and depending on the type return the output +We throw an exception on error +""" + + +def check_server_status(jobID, address) -> str | pd.DataFrame: + headers = {'Content-type': 'application/x-www-form-urlencoded'} + url_status = 'http://' + address + '/job_status' + url_log = 'http://' + address + '/log' + + print("Sending Status Request to Server") + data = {"jobID": jobID} + + status = 0 + length = 0 + while status != 2 and status != 3: + response_status = requests.post(url_status, headers=headers, data=data) + response_log = requests.post(url_log, headers=headers, data=data) + status = int(json.loads(response_status.text)['status']) + log_content = str(json.loads(response_log.text)['message']).replace("ERROR", "").replace("INFO", "") + log = log_content[length:] + length = len(log_content) + if log != "": + print(log) + # WAITING = 0, RUNNING = 1, FINISHED = 2, ERROR = 3 + time.sleep(1.0) + + if status == 2: + try: + url_fetch = 'http://' + address + '/fetch_result' + print("Fetching Results from Server...") + data = {"jobID": jobID, "delete_after_download": True} + response = requests.post(url_fetch, headers=headers, data=data) + content_type = response.headers['content-type'] + print("Content-type: " + str(content_type)) + if content_type == "image/jpeg": + image = Image.open(io.BytesIO(response.content)) + image.save("./outputs/image.jpg") + result = upload_media_to_hoster("./outputs/image.jpg") + os.remove("./outputs/image.jpg") + return result + elif content_type == 'text/plain; charset=utf-8': + return response.content.decode('utf-8') + elif content_type == "application/x-zip-compressed": + zf = zipfile.ZipFile(io.BytesIO(response.content), "r") + + for fileinfo in zf.infolist(): + if fileinfo.filename.endswith(".annotation~"): + try: + anno_string = zf.read(fileinfo).decode('utf-8', errors='replace') + columns = ['from', 'to', 'name', 'conf'] + result = pd.DataFrame([row.split(';') for row in anno_string.split('\n')], + columns=columns) + return result + except Exception as e: + print(e) + except Exception as e: + print("Couldn't fetch result: " + str(e)) + + elif status == 3: + return "error" \ No newline at end of file diff --git a/bot/bot.py b/nostr_dvm/bot.py similarity index 98% rename from bot/bot.py rename to nostr_dvm/bot.py index f8f8e30..6879fed 100644 --- a/bot/bot.py +++ b/nostr_dvm/bot.py @@ -7,14 +7,14 @@ from datetime import timedelta from nostr_sdk import (Keys, Client, Timestamp, Filter, nip04_decrypt, HandleNotification, EventBuilder, PublicKey, Options, Tag, Event, nip04_encrypt) -from utils.admin_utils import admin_make_database_updates -from utils.database_utils import get_or_add_user, update_user_balance, create_sql_table, update_sql_table, User -from utils.definitions import EventDefinitions -from utils.nip89_utils import nip89_fetch_events_pubkey, NIP89Config -from utils.nostr_utils import send_event -from utils.output_utils import PostProcessFunctionType, post_process_list_to_users, post_process_list_to_events -from utils.zap_utils import parse_zap_event_tags, pay_bolt11_ln_bits, zap -from utils.cashu_utils import redeem_cashu +from nostr_dvm.utils.admin_utils import admin_make_database_updates +from nostr_dvm.utils.database_utils import get_or_add_user, update_user_balance, create_sql_table, update_sql_table +from nostr_dvm.utils.definitions import EventDefinitions +from nostr_dvm.utils.nip89_utils import nip89_fetch_events_pubkey, NIP89Config +from nostr_dvm.utils.nostr_utils import send_event +from nostr_dvm.utils.output_utils import PostProcessFunctionType, post_process_list_to_users, post_process_list_to_events +from nostr_dvm.utils.zap_utils import parse_zap_event_tags, pay_bolt11_ln_bits, zap +from nostr_dvm.utils.cashu_utils import redeem_cashu class Bot: diff --git a/core/dvm.py b/nostr_dvm/dvm.py similarity index 97% rename from core/dvm.py rename to nostr_dvm/dvm.py index b787dde..91ede4e 100644 --- a/core/dvm.py +++ b/nostr_dvm/dvm.py @@ -1,23 +1,22 @@ import json from datetime import timedelta -import pandas as pd from nostr_sdk import PublicKey, Keys, Client, Tag, Event, EventBuilder, Filter, HandleNotification, Timestamp, \ init_logger, LogLevel, Options, nip04_encrypt import time -from utils.definitions import EventDefinitions, RequiredJobToWatch, JobToWatch -from utils.dvmconfig import DVMConfig -from utils.admin_utils import admin_make_database_updates, AdminConfig -from utils.backend_utils import get_amount_per_task, check_task_is_supported, get_task -from utils.database_utils import create_sql_table, get_or_add_user, update_user_balance, update_sql_table -from utils.mediasource_utils import input_data_file_duration -from utils.nostr_utils import get_event_by_id, get_referenced_event_by_id, send_event, check_and_decrypt_tags -from utils.output_utils import build_status_reaction -from utils.zap_utils import check_bolt11_ln_bits_is_paid, create_bolt11_ln_bits, parse_zap_event_tags, \ +from nostr_dvm.utils.definitions import EventDefinitions, RequiredJobToWatch, JobToWatch +from nostr_dvm.utils.dvmconfig import DVMConfig +from nostr_dvm.utils.admin_utils import admin_make_database_updates, AdminConfig +from nostr_dvm.utils.backend_utils import get_amount_per_task, check_task_is_supported, get_task +from nostr_dvm.utils.database_utils import create_sql_table, get_or_add_user, update_user_balance, update_sql_table +from nostr_dvm.utils.mediasource_utils import input_data_file_duration +from nostr_dvm.utils.nostr_utils import get_event_by_id, get_referenced_event_by_id, send_event, check_and_decrypt_tags +from nostr_dvm.utils.output_utils import build_status_reaction +from nostr_dvm.utils.zap_utils import check_bolt11_ln_bits_is_paid, create_bolt11_ln_bits, parse_zap_event_tags, \ parse_amount_from_bolt11_invoice, zap, pay_bolt11_ln_bits -from utils.cashu_utils import redeem_cashu +from nostr_dvm.utils.cashu_utils import redeem_cashu use_logger = False if use_logger: diff --git a/nostr_dvm/interfaces/__init__.py b/nostr_dvm/interfaces/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/interfaces/dvmtaskinterface.py b/nostr_dvm/interfaces/dvmtaskinterface.py similarity index 82% rename from interfaces/dvmtaskinterface.py rename to nostr_dvm/interfaces/dvmtaskinterface.py index fb4dfa5..5aa50a5 100644 --- a/interfaces/dvmtaskinterface.py +++ b/nostr_dvm/interfaces/dvmtaskinterface.py @@ -1,13 +1,15 @@ import json +import subprocess +import sys from threading import Thread from nostr_sdk import Keys -from utils.admin_utils import AdminConfig -from utils.dvmconfig import DVMConfig -from utils.nip89_utils import NIP89Config -from core.dvm import DVM -from utils.output_utils import post_process_result +from nostr_dvm.dvm import DVM +from nostr_dvm.utils.admin_utils import AdminConfig +from nostr_dvm.utils.dvmconfig import DVMConfig +from nostr_dvm.utils.nip89_utils import NIP89Config +from nostr_dvm.utils.output_utils import post_process_result class DVMTaskInterface: @@ -23,11 +25,13 @@ class DVMTaskInterface: ACCEPTS_CASHU = True # DVMs build with this framework support encryption, but others might not. dvm_config: DVMConfig admin_config: AdminConfig + dependencies = [] def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config, admin_config: AdminConfig = None, options=None, task=None): self.init(name, dvm_config, admin_config, nip89config, task) self.options = options + self.install_dependencies(self.dependencies) def init(self, name, dvm_config, admin_config=None, nip89config=None, task=None): self.NAME = name @@ -80,6 +84,13 @@ class DVMTaskInterface: """Post-process the data and return the result Use default function, if not overwritten""" return post_process_result(result, event) + def install_dependencies(self, packages): + import pip + for package in packages: + try: + __import__(package.split("=")[0]) + except ImportError: + subprocess.check_call([sys.executable, "-m", "pip", "install", package]) @staticmethod def set_options(request_form): diff --git a/tasks/README.md b/nostr_dvm/tasks/README.md similarity index 100% rename from tasks/README.md rename to nostr_dvm/tasks/README.md diff --git a/nostr_dvm/tasks/__init__.py b/nostr_dvm/tasks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tasks/advanced_search.py b/nostr_dvm/tasks/advanced_search.py similarity index 96% rename from tasks/advanced_search.py rename to nostr_dvm/tasks/advanced_search.py index 33da089..ab72b78 100644 --- a/tasks/advanced_search.py +++ b/nostr_dvm/tasks/advanced_search.py @@ -1,22 +1,19 @@ import json import os -import re from datetime import timedelta from pathlib import Path -from threading import Thread import dotenv -from nostr_sdk import Client, Timestamp, PublicKey, Tag, Keys, Options, Alphabet, SecretKey +from nostr_sdk import Client, Timestamp, PublicKey, Tag, Keys, Options, SecretKey from interfaces.dvmtaskinterface import DVMTaskInterface -from tasks.convert_media import MediaConverter from utils.admin_utils import AdminConfig from utils.backend_utils import keep_alive from utils.definitions import EventDefinitions from utils.dvmconfig import DVMConfig from utils.nip89_utils import NIP89Config, check_and_set_d_tag -from utils.nostr_utils import get_event_by_id, check_and_set_private_key -from utils.output_utils import post_process_list_to_users, post_process_list_to_events +from utils.nostr_utils import check_and_set_private_key +from utils.output_utils import post_process_list_to_events from utils.zap_utils import check_and_set_ln_bits_keys """ diff --git a/tasks/convert_media.py b/nostr_dvm/tasks/convert_media.py similarity index 87% rename from tasks/convert_media.py rename to nostr_dvm/tasks/convert_media.py index 711660c..262db79 100644 --- a/tasks/convert_media.py +++ b/nostr_dvm/tasks/convert_media.py @@ -1,21 +1,19 @@ import json import os -import signal -import time from pathlib import Path import dotenv -from interfaces.dvmtaskinterface import DVMTaskInterface -from utils.admin_utils import AdminConfig -from utils.backend_utils import keep_alive -from utils.definitions import EventDefinitions -from utils.dvmconfig import DVMConfig -from utils.nip89_utils import NIP89Config, check_and_set_d_tag -from utils.mediasource_utils import organize_input_media_data -from utils.nostr_utils import check_and_set_private_key -from utils.output_utils import upload_media_to_hoster -from utils.zap_utils import check_and_set_ln_bits_keys +from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface +from nostr_dvm.utils.admin_utils import AdminConfig +from nostr_dvm.utils.backend_utils import keep_alive +from nostr_dvm.utils.definitions import EventDefinitions +from nostr_dvm.utils.dvmconfig import DVMConfig +from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag +from nostr_dvm.utils.mediasource_utils import organize_input_media_data +from nostr_dvm.utils.nostr_utils import check_and_set_private_key +from nostr_dvm.utils.output_utils import upload_media_to_hoster +from nostr_dvm.utils.zap_utils import check_and_set_ln_bits_keys from nostr_sdk import Keys """ diff --git a/tasks/discovery_inactive_follows.py b/nostr_dvm/tasks/discovery_inactive_follows.py similarity index 93% rename from tasks/discovery_inactive_follows.py rename to nostr_dvm/tasks/discovery_inactive_follows.py index e4f148b..b9a93a9 100644 --- a/tasks/discovery_inactive_follows.py +++ b/nostr_dvm/tasks/discovery_inactive_follows.py @@ -1,23 +1,21 @@ import json import os -import re from datetime import timedelta from pathlib import Path from threading import Thread import dotenv -from nostr_sdk import Client, Timestamp, PublicKey, Tag, Keys, Options, Alphabet +from nostr_sdk import Client, Timestamp, PublicKey, Tag, Keys, Options -from interfaces.dvmtaskinterface import DVMTaskInterface -from tasks.convert_media import MediaConverter -from utils.admin_utils import AdminConfig -from utils.backend_utils import keep_alive -from utils.definitions import EventDefinitions -from utils.dvmconfig import DVMConfig -from utils.nip89_utils import NIP89Config, check_and_set_d_tag -from utils.nostr_utils import get_event_by_id, check_and_set_private_key -from utils.output_utils import post_process_list_to_users -from utils.zap_utils import check_and_set_ln_bits_keys +from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface +from nostr_dvm.utils.admin_utils import AdminConfig +from nostr_dvm.utils.backend_utils import keep_alive +from nostr_dvm.utils.definitions import EventDefinitions +from nostr_dvm.utils.dvmconfig import DVMConfig +from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag +from nostr_dvm.utils.nostr_utils import check_and_set_private_key +from nostr_dvm.utils.output_utils import post_process_list_to_users +from nostr_dvm.utils.zap_utils import check_and_set_ln_bits_keys """ This File contains a Module to find inactive follows for a user on nostr diff --git a/tasks/imagegeneration_openai_dalle.py b/nostr_dvm/tasks/imagegeneration_openai_dalle.py similarity index 91% rename from tasks/imagegeneration_openai_dalle.py rename to nostr_dvm/tasks/imagegeneration_openai_dalle.py index 42f7e09..b536cb8 100644 --- a/tasks/imagegeneration_openai_dalle.py +++ b/nostr_dvm/tasks/imagegeneration_openai_dalle.py @@ -7,15 +7,15 @@ import dotenv import requests from PIL import Image -from interfaces.dvmtaskinterface import DVMTaskInterface -from utils.admin_utils import AdminConfig -from utils.backend_utils import keep_alive -from utils.definitions import EventDefinitions -from utils.dvmconfig import DVMConfig -from utils.nip89_utils import NIP89Config, check_and_set_d_tag -from utils.nostr_utils import check_and_set_private_key -from utils.output_utils import upload_media_to_hoster -from utils.zap_utils import get_price_per_sat, check_and_set_ln_bits_keys +from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface +from nostr_dvm.utils.admin_utils import AdminConfig +from nostr_dvm.utils.backend_utils import keep_alive +from nostr_dvm.utils.definitions import EventDefinitions +from nostr_dvm.utils.dvmconfig import DVMConfig +from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag +from nostr_dvm.utils.nostr_utils import check_and_set_private_key +from nostr_dvm.utils.output_utils import upload_media_to_hoster +from nostr_dvm.utils.zap_utils import get_price_per_sat, check_and_set_ln_bits_keys from nostr_sdk import Keys """ @@ -30,6 +30,7 @@ class ImageGenerationDALLE(DVMTaskInterface): KIND: int = EventDefinitions.KIND_NIP90_GENERATE_IMAGE TASK: str = "text-to-image" FIX_COST: float = 120 + dependencies = ["openai==1.3.5"] def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config, admin_config: AdminConfig = None, options=None): diff --git a/tasks/imagegeneration_replicate_sdxl.py b/nostr_dvm/tasks/imagegeneration_replicate_sdxl.py similarity index 91% rename from tasks/imagegeneration_replicate_sdxl.py rename to nostr_dvm/tasks/imagegeneration_replicate_sdxl.py index 8119e98..4d22f88 100644 --- a/tasks/imagegeneration_replicate_sdxl.py +++ b/nostr_dvm/tasks/imagegeneration_replicate_sdxl.py @@ -7,15 +7,15 @@ import dotenv import requests from PIL import Image -from interfaces.dvmtaskinterface import DVMTaskInterface -from utils.admin_utils import AdminConfig -from utils.backend_utils import keep_alive -from utils.definitions import EventDefinitions -from utils.dvmconfig import DVMConfig -from utils.nip89_utils import NIP89Config, check_and_set_d_tag -from utils.nostr_utils import check_and_set_private_key -from utils.output_utils import upload_media_to_hoster -from utils.zap_utils import get_price_per_sat, check_and_set_ln_bits_keys +from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface +from nostr_dvm.utils.admin_utils import AdminConfig +from nostr_dvm.utils.backend_utils import keep_alive +from nostr_dvm.utils.definitions import EventDefinitions +from nostr_dvm.utils.dvmconfig import DVMConfig +from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag +from nostr_dvm.utils.nostr_utils import check_and_set_private_key +from nostr_dvm.utils.output_utils import upload_media_to_hoster +from nostr_dvm.utils.zap_utils import get_price_per_sat, check_and_set_ln_bits_keys from nostr_sdk import Keys """ @@ -31,6 +31,7 @@ class ImageGenerationReplicateSDXL(DVMTaskInterface): KIND: int = EventDefinitions.KIND_NIP90_GENERATE_IMAGE TASK: str = "text-to-image" FIX_COST: float = 120 + dependencies = ["replicate==0.21.1"] def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config, admin_config: AdminConfig = None, options=None): diff --git a/tasks/textextraction_google.py b/nostr_dvm/tasks/textextraction_google.py similarity index 92% rename from tasks/textextraction_google.py rename to nostr_dvm/tasks/textextraction_google.py index 49a6559..edf57cf 100644 --- a/tasks/textextraction_google.py +++ b/nostr_dvm/tasks/textextraction_google.py @@ -1,20 +1,19 @@ import json import os import time -from multiprocessing.pool import ThreadPool from pathlib import Path import dotenv -from interfaces.dvmtaskinterface import DVMTaskInterface -from utils.admin_utils import AdminConfig -from utils.backend_utils import keep_alive -from utils.dvmconfig import DVMConfig -from utils.mediasource_utils import organize_input_media_data -from utils.nip89_utils import NIP89Config, check_and_set_d_tag -from utils.definitions import EventDefinitions -from utils.nostr_utils import check_and_set_private_key -from utils.zap_utils import check_and_set_ln_bits_keys +from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface +from nostr_dvm.utils.admin_utils import AdminConfig +from nostr_dvm.utils.backend_utils import keep_alive +from nostr_dvm.utils.dvmconfig import DVMConfig +from nostr_dvm.utils.mediasource_utils import organize_input_media_data +from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag +from nostr_dvm.utils.definitions import EventDefinitions +from nostr_dvm.utils.nostr_utils import check_and_set_private_key +from nostr_dvm.utils.zap_utils import check_and_set_ln_bits_keys from nostr_sdk import Keys """ diff --git a/tasks/textextraction_pdf.py b/nostr_dvm/tasks/textextraction_pdf.py similarity index 89% rename from tasks/textextraction_pdf.py rename to nostr_dvm/tasks/textextraction_pdf.py index a2d22d8..3e2b4da 100644 --- a/tasks/textextraction_pdf.py +++ b/nostr_dvm/tasks/textextraction_pdf.py @@ -5,14 +5,14 @@ from pathlib import Path import dotenv -from interfaces.dvmtaskinterface import DVMTaskInterface -from utils.admin_utils import AdminConfig -from utils.backend_utils import keep_alive -from utils.definitions import EventDefinitions -from utils.dvmconfig import DVMConfig -from utils.nip89_utils import NIP89Config, check_and_set_d_tag -from utils.nostr_utils import get_event_by_id, check_and_set_private_key -from utils.zap_utils import check_and_set_ln_bits_keys +from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface +from nostr_dvm.utils.admin_utils import AdminConfig +from nostr_dvm.utils.backend_utils import keep_alive +from nostr_dvm.utils.definitions import EventDefinitions +from nostr_dvm.utils.dvmconfig import DVMConfig +from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag +from nostr_dvm.utils.nostr_utils import get_event_by_id, check_and_set_private_key +from nostr_dvm.utils.zap_utils import check_and_set_ln_bits_keys from nostr_sdk import Keys """ @@ -28,11 +28,14 @@ class TextExtractionPDF(DVMTaskInterface): KIND: int = EventDefinitions.KIND_NIP90_EXTRACT_TEXT TASK: str = "pdf-to-text" FIX_COST: float = 0 + dependencies = ["pypdf==3.17.1"] def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config, admin_config: AdminConfig = None, options=None): super().__init__(name, dvm_config, nip89config, admin_config, options) + + def is_input_supported(self, tags): for tag in tags: if tag.as_vec()[0] == 'i': diff --git a/tasks/textgeneration_llmlite.py b/nostr_dvm/tasks/textgeneration_llmlite.py similarity index 90% rename from tasks/textgeneration_llmlite.py rename to nostr_dvm/tasks/textgeneration_llmlite.py index 750fd7e..c5f4866 100644 --- a/tasks/textgeneration_llmlite.py +++ b/nostr_dvm/tasks/textgeneration_llmlite.py @@ -3,16 +3,16 @@ import os from pathlib import Path import dotenv -from litellm import completion -from interfaces.dvmtaskinterface import DVMTaskInterface -from utils.admin_utils import AdminConfig -from utils.backend_utils import keep_alive -from utils.definitions import EventDefinitions -from utils.dvmconfig import DVMConfig -from utils.nip89_utils import NIP89Config, check_and_set_d_tag -from utils.nostr_utils import check_and_set_private_key -from utils.zap_utils import check_and_set_ln_bits_keys + +from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface +from nostr_dvm.utils.admin_utils import AdminConfig +from nostr_dvm.utils.backend_utils import keep_alive +from nostr_dvm.utils.definitions import EventDefinitions +from nostr_dvm.utils.dvmconfig import DVMConfig +from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag +from nostr_dvm.utils.nostr_utils import check_and_set_private_key +from nostr_dvm.utils.zap_utils import check_and_set_ln_bits_keys from nostr_sdk import Keys """ @@ -27,6 +27,7 @@ class TextGenerationOLLAMA(DVMTaskInterface): KIND: int = EventDefinitions.KIND_NIP90_GENERATE_TEXT TASK: str = "text-to-text" FIX_COST: float = 0 + dependencies = ["litellm==1.12.3"] def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config, admin_config: AdminConfig = None, options=None): @@ -71,6 +72,8 @@ class TextGenerationOLLAMA(DVMTaskInterface): return request_form def process(self, request_form): + from litellm import completion + options = DVMTaskInterface.set_options(request_form) try: diff --git a/tasks/translation_google.py b/nostr_dvm/tasks/translation_google.py similarity index 92% rename from tasks/translation_google.py rename to nostr_dvm/tasks/translation_google.py index 87ff5d5..518e186 100644 --- a/tasks/translation_google.py +++ b/nostr_dvm/tasks/translation_google.py @@ -4,14 +4,14 @@ from pathlib import Path import dotenv -from interfaces.dvmtaskinterface import DVMTaskInterface -from utils.admin_utils import AdminConfig -from utils.backend_utils import keep_alive -from utils.definitions import EventDefinitions -from utils.dvmconfig import DVMConfig -from utils.nip89_utils import NIP89Config, check_and_set_d_tag -from utils.nostr_utils import get_referenced_event_by_id, get_event_by_id, check_and_set_private_key -from utils.zap_utils import check_and_set_ln_bits_keys +from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface +from nostr_dvm.utils.admin_utils import AdminConfig +from nostr_dvm.utils.backend_utils import keep_alive +from nostr_dvm.utils.definitions import EventDefinitions +from nostr_dvm.utils.dvmconfig import DVMConfig +from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag +from nostr_dvm.utils.nostr_utils import get_referenced_event_by_id, get_event_by_id, check_and_set_private_key +from nostr_dvm.utils.zap_utils import check_and_set_ln_bits_keys from nostr_sdk import Keys """ @@ -27,6 +27,7 @@ class TranslationGoogle(DVMTaskInterface): KIND: int = EventDefinitions.KIND_NIP90_TRANSLATE_TEXT TASK: str = "translation" FIX_COST: float = 0 + dependencies = ["translatepy==2.3"] def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config, admin_config: AdminConfig = None, options=None): diff --git a/tasks/translation_libretranslate.py b/nostr_dvm/tasks/translation_libretranslate.py similarity index 93% rename from tasks/translation_libretranslate.py rename to nostr_dvm/tasks/translation_libretranslate.py index 2963951..8bc5d7d 100644 --- a/tasks/translation_libretranslate.py +++ b/nostr_dvm/tasks/translation_libretranslate.py @@ -5,14 +5,14 @@ from pathlib import Path import dotenv import requests -from interfaces.dvmtaskinterface import DVMTaskInterface -from utils.admin_utils import AdminConfig -from utils.backend_utils import keep_alive -from utils.definitions import EventDefinitions -from utils.dvmconfig import DVMConfig -from utils.nip89_utils import NIP89Config, check_and_set_d_tag -from utils.nostr_utils import get_referenced_event_by_id, get_event_by_id, check_and_set_private_key -from utils.zap_utils import check_and_set_ln_bits_keys +from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface +from nostr_dvm.utils.admin_utils import AdminConfig +from nostr_dvm.utils.backend_utils import keep_alive +from nostr_dvm.utils.definitions import EventDefinitions +from nostr_dvm.utils.dvmconfig import DVMConfig +from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag +from nostr_dvm.utils.nostr_utils import get_referenced_event_by_id, get_event_by_id, check_and_set_private_key +from nostr_dvm.utils.zap_utils import check_and_set_ln_bits_keys from nostr_sdk import Keys """ diff --git a/tasks/trending_notes_nostrband.py b/nostr_dvm/tasks/trending_notes_nostrband.py similarity index 86% rename from tasks/trending_notes_nostrband.py rename to nostr_dvm/tasks/trending_notes_nostrband.py index e770712..36c1a66 100644 --- a/tasks/trending_notes_nostrband.py +++ b/nostr_dvm/tasks/trending_notes_nostrband.py @@ -1,23 +1,19 @@ import json import os -import re -from datetime import timedelta from pathlib import Path -from threading import Thread import dotenv -from nostr_sdk import Client, Timestamp, PublicKey, Tag, Keys, Options, Alphabet, SecretKey, Event +from nostr_sdk import Tag, Keys -from interfaces.dvmtaskinterface import DVMTaskInterface - -from utils.admin_utils import AdminConfig -from utils.backend_utils import keep_alive -from utils.definitions import EventDefinitions -from utils.dvmconfig import DVMConfig -from utils.nip89_utils import NIP89Config, check_and_set_d_tag -from utils.nostr_utils import get_event_by_id, check_and_set_private_key -from utils.output_utils import post_process_list_to_users, post_process_list_to_events -from utils.zap_utils import check_and_set_ln_bits_keys +from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface +from nostr_dvm.utils.admin_utils import AdminConfig +from nostr_dvm.utils.backend_utils import keep_alive +from nostr_dvm.utils.definitions import EventDefinitions +from nostr_dvm.utils.dvmconfig import DVMConfig +from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag +from nostr_dvm.utils.nostr_utils import check_and_set_private_key +from nostr_dvm.utils.output_utils import post_process_list_to_events +from nostr_dvm.utils.zap_utils import check_and_set_ln_bits_keys """ This File contains a Module to search for notes @@ -63,7 +59,6 @@ class TrendingNotesNostrBand(DVMTaskInterface): return request_form def process(self, request_form): - from nostr_sdk import Filter options = DVMTaskInterface.set_options(request_form) import requests diff --git a/tasks/videogeneration_replicate_svd.py b/nostr_dvm/tasks/videogeneration_replicate_svd.py similarity index 91% rename from tasks/videogeneration_replicate_svd.py rename to nostr_dvm/tasks/videogeneration_replicate_svd.py index f4451a6..5b9ed02 100644 --- a/tasks/videogeneration_replicate_svd.py +++ b/nostr_dvm/tasks/videogeneration_replicate_svd.py @@ -8,15 +8,15 @@ import requests import urllib.request from PIL import Image -from interfaces.dvmtaskinterface import DVMTaskInterface -from utils.admin_utils import AdminConfig -from utils.backend_utils import keep_alive -from utils.definitions import EventDefinitions -from utils.dvmconfig import DVMConfig -from utils.nip89_utils import NIP89Config, check_and_set_d_tag -from utils.nostr_utils import check_and_set_private_key -from utils.output_utils import upload_media_to_hoster -from utils.zap_utils import get_price_per_sat, check_and_set_ln_bits_keys +from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface +from nostr_dvm.utils.admin_utils import AdminConfig +from nostr_dvm.utils.backend_utils import keep_alive +from nostr_dvm.utils.definitions import EventDefinitions +from nostr_dvm.utils.dvmconfig import DVMConfig +from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag +from nostr_dvm.utils.nostr_utils import check_and_set_private_key +from nostr_dvm.utils.output_utils import upload_media_to_hoster +from nostr_dvm.utils.zap_utils import get_price_per_sat, check_and_set_ln_bits_keys from nostr_sdk import Keys """ diff --git a/nostr_dvm/utils/__init__.py b/nostr_dvm/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/utils/admin_utils.py b/nostr_dvm/utils/admin_utils.py similarity index 86% rename from utils/admin_utils.py rename to nostr_dvm/utils/admin_utils.py index 48bd402..3b32a16 100644 --- a/utils/admin_utils.py +++ b/nostr_dvm/utils/admin_utils.py @@ -1,15 +1,12 @@ # ADMINISTRARIVE DB MANAGEMENT -import time -from datetime import timedelta -from nostr_sdk import Keys, EventBuilder, PublicKey, Client, Filter, EventId +from nostr_sdk import Keys, PublicKey, Client -from utils.database_utils import get_from_sql_table, list_db, delete_from_sql_table, update_sql_table, \ +from nostr_dvm.utils.database_utils import get_from_sql_table, list_db, delete_from_sql_table, update_sql_table, \ get_or_add_user, clean_db -from utils.dvmconfig import DVMConfig -from utils.nip89_utils import nip89_announce_tasks, NIP89Config, nip89_delete_announcement, \ - fetch_nip89_paramters_for_deletion -from utils.nostr_utils import send_event, update_profile +from nostr_dvm.utils.dvmconfig import DVMConfig +from nostr_dvm.utils.nip89_utils import nip89_announce_tasks, fetch_nip89_paramters_for_deletion +from nostr_dvm.utils.nostr_utils import update_profile class AdminConfig: diff --git a/utils/backend_utils.py b/nostr_dvm/utils/backend_utils.py similarity index 97% rename from utils/backend_utils.py rename to nostr_dvm/utils/backend_utils.py index 16ee43b..7bcfb28 100644 --- a/utils/backend_utils.py +++ b/nostr_dvm/utils/backend_utils.py @@ -3,11 +3,11 @@ import signal import time import requests -from nostr_sdk import Event, Tag +from nostr_sdk import Event -from utils.definitions import EventDefinitions -from utils.mediasource_utils import check_source_type, media_source -from utils.nostr_utils import get_event_by_id, get_referenced_event_by_id +from nostr_dvm.utils.definitions import EventDefinitions +from nostr_dvm.utils.mediasource_utils import check_source_type, media_source +from nostr_dvm.utils.nostr_utils import get_event_by_id, get_referenced_event_by_id def get_task(event, client, dvm_config): diff --git a/utils/cashu_utils.py b/nostr_dvm/utils/cashu_utils.py similarity index 97% rename from utils/cashu_utils.py rename to nostr_dvm/utils/cashu_utils.py index 52876ea..60823c9 100644 --- a/utils/cashu_utils.py +++ b/nostr_dvm/utils/cashu_utils.py @@ -1,9 +1,8 @@ -import asyncio import base64 import json import requests -from utils.database_utils import get_or_add_user -from utils.zap_utils import create_bolt11_ln_bits, create_bolt11_lud16 +from nostr_dvm.utils.database_utils import get_or_add_user +from nostr_dvm.utils.zap_utils import create_bolt11_ln_bits, create_bolt11_lud16 async def get_cashu_balance(url): diff --git a/utils/database_utils.py b/nostr_dvm/utils/database_utils.py similarity index 99% rename from utils/database_utils.py rename to nostr_dvm/utils/database_utils.py index f0da560..32e44a1 100644 --- a/utils/database_utils.py +++ b/nostr_dvm/utils/database_utils.py @@ -1,7 +1,6 @@ # DATABASE LOGIC import json import sqlite3 -import time from _sqlite3 import Error from dataclasses import dataclass @@ -9,7 +8,7 @@ from datetime import timedelta from logging import Filter from nostr_sdk import Timestamp, Keys, PublicKey, EventBuilder, Filter -from utils.nostr_utils import send_event +from nostr_dvm.utils.nostr_utils import send_event @dataclass diff --git a/utils/definitions.py b/nostr_dvm/utils/definitions.py similarity index 100% rename from utils/definitions.py rename to nostr_dvm/utils/definitions.py diff --git a/utils/dvmconfig.py b/nostr_dvm/utils/dvmconfig.py similarity index 90% rename from utils/dvmconfig.py rename to nostr_dvm/utils/dvmconfig.py index 9e37440..63474c4 100644 --- a/utils/dvmconfig.py +++ b/nostr_dvm/utils/dvmconfig.py @@ -1,5 +1,5 @@ -from utils.nip89_utils import NIP89Config -from utils.output_utils import PostProcessFunctionType +from nostr_dvm.utils.nip89_utils import NIP89Config +from nostr_dvm.utils.output_utils import PostProcessFunctionType class DVMConfig: diff --git a/utils/external_dvm_utils.py b/nostr_dvm/utils/external_dvm_utils.py similarity index 90% rename from utils/external_dvm_utils.py rename to nostr_dvm/utils/external_dvm_utils.py index c6e61ac..2845c8c 100644 --- a/utils/external_dvm_utils.py +++ b/nostr_dvm/utils/external_dvm_utils.py @@ -3,10 +3,10 @@ from datetime import timedelta from nostr_sdk import PublicKey, Options, Keys, Client -from interfaces.dvmtaskinterface import DVMTaskInterface -from utils.dvmconfig import DVMConfig -from utils.nip89_utils import NIP89Config, nip89_fetch_events_pubkey -from utils.output_utils import PostProcessFunctionType +from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface +from nostr_dvm.utils.dvmconfig import DVMConfig +from nostr_dvm.utils.nip89_utils import NIP89Config, nip89_fetch_events_pubkey +from nostr_dvm.utils.output_utils import PostProcessFunctionType def build_external_dvm(pubkey, task, kind, fix_cost, per_unit_cost, config, diff --git a/utils/mediasource_utils.py b/nostr_dvm/utils/mediasource_utils.py similarity index 96% rename from utils/mediasource_utils.py rename to nostr_dvm/utils/mediasource_utils.py index 4477d09..9b00c01 100644 --- a/utils/mediasource_utils.py +++ b/nostr_dvm/utils/mediasource_utils.py @@ -3,9 +3,9 @@ import urllib from datetime import time from urllib.parse import urlparse import ffmpegio -from decord import AudioReader, VideoReader, cpu +from decord import AudioReader, cpu import requests -from utils.nostr_utils import get_event_by_id +from nostr_dvm.utils.nostr_utils import get_event_by_id def input_data_file_duration(event, dvm_config, client, start=0, end=0): @@ -332,30 +332,30 @@ def get_media_link(url) -> (str, str): def downloadOvercast(source_url, target_location): - from utils.scrapper.media_scrapper import OvercastDownload + from scrapper.media_scrapper import OvercastDownload result = OvercastDownload(source_url, target_location) return result def downloadTwitter(videourl, path): - from utils.scrapper.media_scrapper import XitterDownload + from scrapper.media_scrapper import XitterDownload result = XitterDownload(videourl, path + "x.mp4") return result def downloadTikTok(videourl, path): - from utils.scrapper.media_scrapper import TiktokDownloadAll + from scrapper.media_scrapper import TiktokDownloadAll result = TiktokDownloadAll([videourl], path) return result def downloadInstagram(videourl, path): - from utils.scrapper.media_scrapper import InstagramDownload + from scrapper.media_scrapper import InstagramDownload result = InstagramDownload(videourl, "insta", path) return result def downloadYouTube(link, path, audioonly=True): - from utils.scrapper.media_scrapper import YouTubeDownload + from scrapper.media_scrapper import YouTubeDownload result = YouTubeDownload(link, path, audio_only=audioonly) return result diff --git a/utils/nip89_utils.py b/nostr_dvm/utils/nip89_utils.py similarity index 96% rename from utils/nip89_utils.py rename to nostr_dvm/utils/nip89_utils.py index e1b9e1a..5db4530 100644 --- a/utils/nip89_utils.py +++ b/nostr_dvm/utils/nip89_utils.py @@ -4,10 +4,10 @@ from hashlib import sha256 from pathlib import Path import dotenv -from nostr_sdk import Tag, Keys, EventBuilder, Filter, Alphabet, PublicKey, Event, Client, EventId +from nostr_sdk import Tag, Keys, EventBuilder, Filter, Alphabet, PublicKey, Client, EventId -from utils.definitions import EventDefinitions -from utils.nostr_utils import send_event +from nostr_dvm.utils.definitions import EventDefinitions +from nostr_dvm.utils.nostr_utils import send_event class NIP89Config: diff --git a/utils/nostr_utils.py b/nostr_dvm/utils/nostr_utils.py similarity index 100% rename from utils/nostr_utils.py rename to nostr_dvm/utils/nostr_utils.py diff --git a/utils/output_utils.py b/nostr_dvm/utils/output_utils.py similarity index 100% rename from utils/output_utils.py rename to nostr_dvm/utils/output_utils.py diff --git a/nostr_dvm/utils/scrapper/__init__.py b/nostr_dvm/utils/scrapper/__init__.py new file mode 100644 index 0000000..9a970eb --- /dev/null +++ b/nostr_dvm/utils/scrapper/__init__.py @@ -0,0 +1 @@ +from dvm import DVM \ No newline at end of file diff --git a/utils/scrapper/media_scrapper.py b/nostr_dvm/utils/scrapper/media_scrapper.py similarity index 100% rename from utils/scrapper/media_scrapper.py rename to nostr_dvm/utils/scrapper/media_scrapper.py diff --git a/utils/scrapper/request_details.json b/nostr_dvm/utils/scrapper/request_details.json similarity index 100% rename from utils/scrapper/request_details.json rename to nostr_dvm/utils/scrapper/request_details.json diff --git a/utils/zap_utils.py b/nostr_dvm/utils/zap_utils.py similarity index 99% rename from utils/zap_utils.py rename to nostr_dvm/utils/zap_utils.py index 8c7cf98..30e2865 100644 --- a/utils/zap_utils.py +++ b/nostr_dvm/utils/zap_utils.py @@ -9,8 +9,8 @@ from Crypto.Cipher import AES from Crypto.Util.Padding import pad from bech32 import bech32_decode, convertbits, bech32_encode from nostr_sdk import nostr_sdk, PublicKey, SecretKey, Event, EventBuilder, Tag, Keys -from utils.dvmconfig import DVMConfig -from utils.nostr_utils import get_event_by_id, check_and_decrypt_own_tags, update_profile +from nostr_dvm.utils.dvmconfig import DVMConfig +from nostr_dvm.utils.nostr_utils import get_event_by_id, check_and_decrypt_own_tags import lnurl from hashlib import sha256 import dotenv diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..4101063 --- /dev/null +++ b/setup.py @@ -0,0 +1,38 @@ +from setuptools import setup, find_packages + +VERSION = '0.0.1' +DESCRIPTION = 'A framework to build and run NIP90 Data Vending Machines' +LONG_DESCRIPTION = '-' + +# Setting up +setup( + name="nostr-dvm", + version=VERSION, + author="Believethehype", + author_email="-", + description=DESCRIPTION, + long_description=LONG_DESCRIPTION, + packages=find_packages(include=['nostr_dvm']), + install_requires=["nostr-sdk==0.0.5", + "bech32==1.2.0", + "pycryptodome==3.19.0", + "python-dotenv==1.0.0", + "emoji==2.8.0", + "eva-decord==0.6.1", + "ffmpegio==0.8.5", + "lnurl==0.4.1", + "pandas==2.1.3", + "Pillow==10.1.0", + "PyUpload==0.1.4", + "requests==2.31.0", + "instaloader==4.10.1", + "pytube==15.0.0", + "moviepy==2.0.0.dev2" + ], + keywords=['nostr', 'nip90', 'dvm', 'data vending machine'], + classifiers=[ + "Development Status :: 3 - Alpha", + "Intended Audience :: Education", + "Programming Language :: Python :: 3", + ] +) \ No newline at end of file diff --git a/tests/test_dvm_client.py b/tests/test_dvm_client.py index f03a5db..0737d5e 100644 --- a/tests/test_dvm_client.py +++ b/tests/test_dvm_client.py @@ -1,7 +1,5 @@ import json -import os import time -import datetime as datetime from pathlib import Path from threading import Thread