highperfocused/nostrdvm
1
0
Fork 0
mirror of https://github.com/believethehype/nostrdvm.git synced 2025-03-18 05:41:51 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #11 from believethehype/package

Browse Source 
Package
This commit is contained in:
believethehype 2023-12-13 22:29:46 +01:00 committed by GitHub
commit c189489536
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
43 changed files with 353 additions and 198 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.idea/dvm.iml generated
View File

@ -4,8 +4,10 @@
<content url="file://$MODULE_DIR$">
<excludeFolder url="file://$MODULE_DIR$/.idea/dataSources" />
<excludeFolder url="file://$MODULE_DIR$/venv" />
<excludeFolder url="file://$MODULE_DIR$/venv_package_test" />
<excludeFolder url="file://$MODULE_DIR$/venv_package_test2" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="jdk" jdkName="Python 3.10 (dvm) (3)" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>

2
.idea/misc.xml generated
View File

@ -3,5 +3,5 @@
<component name="Black">
<option name="sdkName" value="Python 3.10 (dvm)" />
</component>
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (dvm) (2)" project-jdk-type="Python SDK" />
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (dvm) (3)" project-jdk-type="Python SDK" />
</project>

33
main.py
View File

@ -1,26 +1,19 @@
import os
from pathlib import Path
import dotenv
from bot.bot import Bot
import tasks.convert_media as convert_media
import tasks.discovery_inactive_follows as discovery_inactive_follows
import tasks.imagegeneration_openai_dalle as imagegeneration_openai_dalle
import tasks.textextraction_pdf as textextraction_pdf
import tasks.textextraction_google as textextraction_google
import tasks.translation_google as translation_google
import tasks.translation_libretranslate as translation_libretranslate
from tasks import imagegeneration_replicate_sdxl, videogeneration_replicate_svd, trending_notes_nostrband, \
textgeneration_llmlite
from utils.admin_utils import AdminConfig
from utils.backend_utils import keep_alive
from utils.definitions import EventDefinitions
from utils.dvmconfig import DVMConfig
from utils.external_dvm_utils import build_external_dvm
from utils.nostr_utils import check_and_set_private_key
from utils.output_utils import PostProcessFunctionType
from utils.zap_utils import check_and_set_ln_bits_keys
from nostr_dvm.bot import Bot
from nostr_dvm.tasks import videogeneration_replicate_svd, imagegeneration_replicate_sdxl, textgeneration_llmlite, \
trending_notes_nostrband, discovery_inactive_follows, translation_google, textextraction_pdf, \
translation_libretranslate, textextraction_google, convert_media, imagegeneration_openai_dalle
from nostr_dvm.utils.admin_utils import AdminConfig
from nostr_dvm.utils.backend_utils import keep_alive
from nostr_dvm.utils.definitions import EventDefinitions
from nostr_dvm.utils.dvmconfig import DVMConfig
from nostr_dvm.utils.external_dvm_utils import build_external_dvm
from nostr_dvm.utils.nostr_utils import check_and_set_private_key
from nostr_dvm.utils.output_utils import PostProcessFunctionType
from nostr_dvm.utils.zap_utils import check_and_set_ln_bits_keys
from nostr_sdk import Keys
@ -115,7 +108,7 @@ def playground():
# Spawn DVM 7 Find inactive followers
googleextractor = textextraction_google.build_example("Extractor", "speech_recognition",
admin_config)
admin_config)
bot_config.SUPPORTED_DVMS.append(googleextractor)
googleextractor.run()

10
nostr_dvm/__init__.py Normal file
View File

@ -0,0 +1,10 @@
# We follow Semantic Versioning (https://semver.org/)
_MAJOR_VERSION = '0'
_MINOR_VERSION = '0'
_PATCH_VERSION = '1'
__version__ = '.'.join([
_MAJOR_VERSION,
_MINOR_VERSION,
_PATCH_VERSION,
])

0
nostr_dvm/backends/__init__.py Normal file
View File

0
nostr_dvm/backends/nova_server/__init__.py Normal file
View File

112
nostr_dvm/backends/nova_server/utils.py Normal file
View File

@ -0,0 +1,112 @@
import io
import json
import os
import time
import zipfile
import pandas as pd
import requests
import PIL.Image as Image
from nostr_dvm.utils.output_utils import upload_media_to_hoster
"""
This file contains basic calling functions for ML tasks that are outsourced to nova server. It is an Open-Source backend
that enables running models locally based on preefined modules, by accepting a request form.
Modules are deployed in in separate virtual environments so dependencies won't conflict.
"""
"""
send_request_to_n_server(request_form, address)
Function to send a request_form to the server, containing all the information we parsed from the Nostr event and added
in the module that is calling the server
"""
def send_request_to_server(request_form, address):
print("Sending job to Server")
url = ('http://' + address + '/process')
headers = {'Content-type': 'application/x-www-form-urlencoded'}
response = requests.post(url, headers=headers, data=request_form)
return response.text
def send_file_to_server(filepath, address):
print("Sending file to Server")
url = ('http://' + address + '/upload')
try:
fp = open(filepath, 'rb')
response = requests.post(url, files={'file': fp})
result = response.content.decode('utf-8')
except Exception as e:
print(e)
print(response.content.decode('utf-8'))
return result
# headers = {'Content-type': 'application/x-www-form-urlencoded'}
"""
check_n_server_status(request_form, address)
Function that requests the status of the current process with the jobID (we use the Nostr event as jobID).
When the Job is successfully finished we grab the result and depending on the type return the output
We throw an exception on error
"""
def check_server_status(jobID, address) -> str | pd.DataFrame:
headers = {'Content-type': 'application/x-www-form-urlencoded'}
url_status = 'http://' + address + '/job_status'
url_log = 'http://' + address + '/log'
print("Sending Status Request to Server")
data = {"jobID": jobID}
status = 0
length = 0
while status != 2 and status != 3:
response_status = requests.post(url_status, headers=headers, data=data)
response_log = requests.post(url_log, headers=headers, data=data)
status = int(json.loads(response_status.text)['status'])
log_content = str(json.loads(response_log.text)['message']).replace("ERROR", "").replace("INFO", "")
log = log_content[length:]
length = len(log_content)
if log != "":
print(log)
# WAITING = 0, RUNNING = 1, FINISHED = 2, ERROR = 3
time.sleep(1.0)
if status == 2:
try:
url_fetch = 'http://' + address + '/fetch_result'
print("Fetching Results from Server...")
data = {"jobID": jobID, "delete_after_download": True}
response = requests.post(url_fetch, headers=headers, data=data)
content_type = response.headers['content-type']
print("Content-type: " + str(content_type))
if content_type == "image/jpeg":
image = Image.open(io.BytesIO(response.content))
image.save("./outputs/image.jpg")
result = upload_media_to_hoster("./outputs/image.jpg")
os.remove("./outputs/image.jpg")
return result
elif content_type == 'text/plain; charset=utf-8':
return response.content.decode('utf-8')
elif content_type == "application/x-zip-compressed":
zf = zipfile.ZipFile(io.BytesIO(response.content), "r")
for fileinfo in zf.infolist():
if fileinfo.filename.endswith(".annotation~"):
try:
anno_string = zf.read(fileinfo).decode('utf-8', errors='replace')
columns = ['from', 'to', 'name', 'conf']
result = pd.DataFrame([row.split(';') for row in anno_string.split('\n')],
columns=columns)
return result
except Exception as e:
print(e)
except Exception as e:
print("Couldn't fetch result: " + str(e))
elif status == 3:
return "error"

16
bot/bot.py → nostr_dvm/bot.py
View File

@ -7,14 +7,14 @@ from datetime import timedelta
from nostr_sdk import (Keys, Client, Timestamp, Filter, nip04_decrypt, HandleNotification, EventBuilder, PublicKey,
Options, Tag, Event, nip04_encrypt)
from utils.admin_utils import admin_make_database_updates
from utils.database_utils import get_or_add_user, update_user_balance, create_sql_table, update_sql_table, User
from utils.definitions import EventDefinitions
from utils.nip89_utils import nip89_fetch_events_pubkey, NIP89Config
from utils.nostr_utils import send_event
from utils.output_utils import PostProcessFunctionType, post_process_list_to_users, post_process_list_to_events
from utils.zap_utils import parse_zap_event_tags, pay_bolt11_ln_bits, zap
from utils.cashu_utils import redeem_cashu
from nostr_dvm.utils.admin_utils import admin_make_database_updates
from nostr_dvm.utils.database_utils import get_or_add_user, update_user_balance, create_sql_table, update_sql_table
from nostr_dvm.utils.definitions import EventDefinitions
from nostr_dvm.utils.nip89_utils import nip89_fetch_events_pubkey, NIP89Config
from nostr_dvm.utils.nostr_utils import send_event
from nostr_dvm.utils.output_utils import PostProcessFunctionType, post_process_list_to_users, post_process_list_to_events
from nostr_dvm.utils.zap_utils import parse_zap_event_tags, pay_bolt11_ln_bits, zap
from nostr_dvm.utils.cashu_utils import redeem_cashu
class Bot:

21
core/dvm.py → nostr_dvm/dvm.py
View File

@ -1,23 +1,22 @@
import json
from datetime import timedelta
import pandas as pd
from nostr_sdk import PublicKey, Keys, Client, Tag, Event, EventBuilder, Filter, HandleNotification, Timestamp, \
init_logger, LogLevel, Options, nip04_encrypt
import time
from utils.definitions import EventDefinitions, RequiredJobToWatch, JobToWatch
from utils.dvmconfig import DVMConfig
from utils.admin_utils import admin_make_database_updates, AdminConfig
from utils.backend_utils import get_amount_per_task, check_task_is_supported, get_task
from utils.database_utils import create_sql_table, get_or_add_user, update_user_balance, update_sql_table
from utils.mediasource_utils import input_data_file_duration
from utils.nostr_utils import get_event_by_id, get_referenced_event_by_id, send_event, check_and_decrypt_tags
from utils.output_utils import build_status_reaction
from utils.zap_utils import check_bolt11_ln_bits_is_paid, create_bolt11_ln_bits, parse_zap_event_tags, \
from nostr_dvm.utils.definitions import EventDefinitions, RequiredJobToWatch, JobToWatch
from nostr_dvm.utils.dvmconfig import DVMConfig
from nostr_dvm.utils.admin_utils import admin_make_database_updates, AdminConfig
from nostr_dvm.utils.backend_utils import get_amount_per_task, check_task_is_supported, get_task
from nostr_dvm.utils.database_utils import create_sql_table, get_or_add_user, update_user_balance, update_sql_table
from nostr_dvm.utils.mediasource_utils import input_data_file_duration
from nostr_dvm.utils.nostr_utils import get_event_by_id, get_referenced_event_by_id, send_event, check_and_decrypt_tags
from nostr_dvm.utils.output_utils import build_status_reaction
from nostr_dvm.utils.zap_utils import check_bolt11_ln_bits_is_paid, create_bolt11_ln_bits, parse_zap_event_tags, \
parse_amount_from_bolt11_invoice, zap, pay_bolt11_ln_bits
from utils.cashu_utils import redeem_cashu
from nostr_dvm.utils.cashu_utils import redeem_cashu
use_logger = False
if use_logger:

0
nostr_dvm/interfaces/__init__.py Normal file
View File

21
interfaces/dvmtaskinterface.py → nostr_dvm/interfaces/dvmtaskinterface.py
View File

@ -1,13 +1,15 @@
import json
import subprocess
import sys
from threading import Thread
from nostr_sdk import Keys
from utils.admin_utils import AdminConfig
from utils.dvmconfig import DVMConfig
from utils.nip89_utils import NIP89Config
from core.dvm import DVM
from utils.output_utils import post_process_result
from nostr_dvm.dvm import DVM
from nostr_dvm.utils.admin_utils import AdminConfig
from nostr_dvm.utils.dvmconfig import DVMConfig
from nostr_dvm.utils.nip89_utils import NIP89Config
from nostr_dvm.utils.output_utils import post_process_result
class DVMTaskInterface:
@ -23,11 +25,13 @@ class DVMTaskInterface:
ACCEPTS_CASHU = True # DVMs build with this framework support encryption, but others might not.
dvm_config: DVMConfig
admin_config: AdminConfig
dependencies = []
def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config, admin_config: AdminConfig = None,
options=None, task=None):
self.init(name, dvm_config, admin_config, nip89config, task)
self.options = options
self.install_dependencies(self.dependencies)
def init(self, name, dvm_config, admin_config=None, nip89config=None, task=None):
self.NAME = name
@ -80,6 +84,13 @@ class DVMTaskInterface:
"""Post-process the data and return the result Use default function, if not overwritten"""
return post_process_result(result, event)
def install_dependencies(self, packages):
import pip
for package in packages:
try:
__import__(package.split("=")[0])
except ImportError:
subprocess.check_call([sys.executable, "-m", "pip", "install", package])
@staticmethod
def set_options(request_form):

0
tasks/README.md → nostr_dvm/tasks/README.md
View File

0
nostr_dvm/tasks/__init__.py Normal file
View File

9
tasks/advanced_search.py → nostr_dvm/tasks/advanced_search.py
View File

@ -1,22 +1,19 @@
import json
import os
import re
from datetime import timedelta
from pathlib import Path
from threading import Thread
import dotenv
from nostr_sdk import Client, Timestamp, PublicKey, Tag, Keys, Options, Alphabet, SecretKey
from nostr_sdk import Client, Timestamp, PublicKey, Tag, Keys, Options, SecretKey
from interfaces.dvmtaskinterface import DVMTaskInterface
from tasks.convert_media import MediaConverter
from utils.admin_utils import AdminConfig
from utils.backend_utils import keep_alive
from utils.definitions import EventDefinitions
from utils.dvmconfig import DVMConfig
from utils.nip89_utils import NIP89Config, check_and_set_d_tag
from utils.nostr_utils import get_event_by_id, check_and_set_private_key
from utils.output_utils import post_process_list_to_users, post_process_list_to_events
from utils.nostr_utils import check_and_set_private_key
from utils.output_utils import post_process_list_to_events
from utils.zap_utils import check_and_set_ln_bits_keys
"""

22
tasks/convert_media.py → nostr_dvm/tasks/convert_media.py
View File

@ -1,21 +1,19 @@
import json
import os
import signal
import time
from pathlib import Path
import dotenv
from interfaces.dvmtaskinterface import DVMTaskInterface
from utils.admin_utils import AdminConfig
from utils.backend_utils import keep_alive
from utils.definitions import EventDefinitions
from utils.dvmconfig import DVMConfig
from utils.nip89_utils import NIP89Config, check_and_set_d_tag
from utils.mediasource_utils import organize_input_media_data
from utils.nostr_utils import check_and_set_private_key
from utils.output_utils import upload_media_to_hoster
from utils.zap_utils import check_and_set_ln_bits_keys
from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface
from nostr_dvm.utils.admin_utils import AdminConfig
from nostr_dvm.utils.backend_utils import keep_alive
from nostr_dvm.utils.definitions import EventDefinitions
from nostr_dvm.utils.dvmconfig import DVMConfig
from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag
from nostr_dvm.utils.mediasource_utils import organize_input_media_data
from nostr_dvm.utils.nostr_utils import check_and_set_private_key
from nostr_dvm.utils.output_utils import upload_media_to_hoster
from nostr_dvm.utils.zap_utils import check_and_set_ln_bits_keys
from nostr_sdk import Keys
"""

22
tasks/discovery_inactive_follows.py → nostr_dvm/tasks/discovery_inactive_follows.py
View File

@ -1,23 +1,21 @@
import json
import os
import re
from datetime import timedelta
from pathlib import Path
from threading import Thread
import dotenv
from nostr_sdk import Client, Timestamp, PublicKey, Tag, Keys, Options, Alphabet
from nostr_sdk import Client, Timestamp, PublicKey, Tag, Keys, Options
from interfaces.dvmtaskinterface import DVMTaskInterface
from tasks.convert_media import MediaConverter
from utils.admin_utils import AdminConfig
from utils.backend_utils import keep_alive
from utils.definitions import EventDefinitions
from utils.dvmconfig import DVMConfig
from utils.nip89_utils import NIP89Config, check_and_set_d_tag
from utils.nostr_utils import get_event_by_id, check_and_set_private_key
from utils.output_utils import post_process_list_to_users
from utils.zap_utils import check_and_set_ln_bits_keys
from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface
from nostr_dvm.utils.admin_utils import AdminConfig
from nostr_dvm.utils.backend_utils import keep_alive
from nostr_dvm.utils.definitions import EventDefinitions
from nostr_dvm.utils.dvmconfig import DVMConfig
from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag
from nostr_dvm.utils.nostr_utils import check_and_set_private_key
from nostr_dvm.utils.output_utils import post_process_list_to_users
from nostr_dvm.utils.zap_utils import check_and_set_ln_bits_keys
"""
This File contains a Module to find inactive follows for a user on nostr

19
tasks/imagegeneration_openai_dalle.py → nostr_dvm/tasks/imagegeneration_openai_dalle.py
View File

@ -7,15 +7,15 @@ import dotenv
import requests
from PIL import Image
from interfaces.dvmtaskinterface import DVMTaskInterface
from utils.admin_utils import AdminConfig
from utils.backend_utils import keep_alive
from utils.definitions import EventDefinitions
from utils.dvmconfig import DVMConfig
from utils.nip89_utils import NIP89Config, check_and_set_d_tag
from utils.nostr_utils import check_and_set_private_key
from utils.output_utils import upload_media_to_hoster
from utils.zap_utils import get_price_per_sat, check_and_set_ln_bits_keys
from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface
from nostr_dvm.utils.admin_utils import AdminConfig
from nostr_dvm.utils.backend_utils import keep_alive
from nostr_dvm.utils.definitions import EventDefinitions
from nostr_dvm.utils.dvmconfig import DVMConfig
from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag
from nostr_dvm.utils.nostr_utils import check_and_set_private_key
from nostr_dvm.utils.output_utils import upload_media_to_hoster
from nostr_dvm.utils.zap_utils import get_price_per_sat, check_and_set_ln_bits_keys
from nostr_sdk import Keys
"""
@ -30,6 +30,7 @@ class ImageGenerationDALLE(DVMTaskInterface):
KIND: int = EventDefinitions.KIND_NIP90_GENERATE_IMAGE
TASK: str = "text-to-image"
FIX_COST: float = 120
dependencies = ["openai==1.3.5"]
def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config,
admin_config: AdminConfig = None, options=None):

19
tasks/imagegeneration_replicate_sdxl.py → nostr_dvm/tasks/imagegeneration_replicate_sdxl.py
View File

@ -7,15 +7,15 @@ import dotenv
import requests
from PIL import Image
from interfaces.dvmtaskinterface import DVMTaskInterface
from utils.admin_utils import AdminConfig
from utils.backend_utils import keep_alive
from utils.definitions import EventDefinitions
from utils.dvmconfig import DVMConfig
from utils.nip89_utils import NIP89Config, check_and_set_d_tag
from utils.nostr_utils import check_and_set_private_key
from utils.output_utils import upload_media_to_hoster
from utils.zap_utils import get_price_per_sat, check_and_set_ln_bits_keys
from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface
from nostr_dvm.utils.admin_utils import AdminConfig
from nostr_dvm.utils.backend_utils import keep_alive
from nostr_dvm.utils.definitions import EventDefinitions
from nostr_dvm.utils.dvmconfig import DVMConfig
from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag
from nostr_dvm.utils.nostr_utils import check_and_set_private_key
from nostr_dvm.utils.output_utils import upload_media_to_hoster
from nostr_dvm.utils.zap_utils import get_price_per_sat, check_and_set_ln_bits_keys
from nostr_sdk import Keys
"""
@ -31,6 +31,7 @@ class ImageGenerationReplicateSDXL(DVMTaskInterface):
KIND: int = EventDefinitions.KIND_NIP90_GENERATE_IMAGE
TASK: str = "text-to-image"
FIX_COST: float = 120
dependencies = ["replicate==0.21.1"]
def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config,
admin_config: AdminConfig = None, options=None):

19
tasks/textextraction_google.py → nostr_dvm/tasks/textextraction_google.py
View File

@ -1,20 +1,19 @@
import json
import os
import time
from multiprocessing.pool import ThreadPool
from pathlib import Path
import dotenv
from interfaces.dvmtaskinterface import DVMTaskInterface
from utils.admin_utils import AdminConfig
from utils.backend_utils import keep_alive
from utils.dvmconfig import DVMConfig
from utils.mediasource_utils import organize_input_media_data
from utils.nip89_utils import NIP89Config, check_and_set_d_tag
from utils.definitions import EventDefinitions
from utils.nostr_utils import check_and_set_private_key
from utils.zap_utils import check_and_set_ln_bits_keys
from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface
from nostr_dvm.utils.admin_utils import AdminConfig
from nostr_dvm.utils.backend_utils import keep_alive
from nostr_dvm.utils.dvmconfig import DVMConfig
from nostr_dvm.utils.mediasource_utils import organize_input_media_data
from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag
from nostr_dvm.utils.definitions import EventDefinitions
from nostr_dvm.utils.nostr_utils import check_and_set_private_key
from nostr_dvm.utils.zap_utils import check_and_set_ln_bits_keys
from nostr_sdk import Keys
"""

19
tasks/textextraction_pdf.py → nostr_dvm/tasks/textextraction_pdf.py
View File

@ -5,14 +5,14 @@ from pathlib import Path
import dotenv
from interfaces.dvmtaskinterface import DVMTaskInterface
from utils.admin_utils import AdminConfig
from utils.backend_utils import keep_alive
from utils.definitions import EventDefinitions
from utils.dvmconfig import DVMConfig
from utils.nip89_utils import NIP89Config, check_and_set_d_tag
from utils.nostr_utils import get_event_by_id, check_and_set_private_key
from utils.zap_utils import check_and_set_ln_bits_keys
from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface
from nostr_dvm.utils.admin_utils import AdminConfig
from nostr_dvm.utils.backend_utils import keep_alive
from nostr_dvm.utils.definitions import EventDefinitions
from nostr_dvm.utils.dvmconfig import DVMConfig
from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag
from nostr_dvm.utils.nostr_utils import get_event_by_id, check_and_set_private_key
from nostr_dvm.utils.zap_utils import check_and_set_ln_bits_keys
from nostr_sdk import Keys
"""
@ -28,11 +28,14 @@ class TextExtractionPDF(DVMTaskInterface):
KIND: int = EventDefinitions.KIND_NIP90_EXTRACT_TEXT
TASK: str = "pdf-to-text"
FIX_COST: float = 0
dependencies = ["pypdf==3.17.1"]
def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config,
admin_config: AdminConfig = None, options=None):
super().__init__(name, dvm_config, nip89config, admin_config, options)
def is_input_supported(self, tags):
for tag in tags:
if tag.as_vec()[0] == 'i':

21
tasks/textgeneration_llmlite.py → nostr_dvm/tasks/textgeneration_llmlite.py
View File

@ -3,16 +3,16 @@ import os
from pathlib import Path
import dotenv
from litellm import completion
from interfaces.dvmtaskinterface import DVMTaskInterface
from utils.admin_utils import AdminConfig
from utils.backend_utils import keep_alive
from utils.definitions import EventDefinitions
from utils.dvmconfig import DVMConfig
from utils.nip89_utils import NIP89Config, check_and_set_d_tag
from utils.nostr_utils import check_and_set_private_key
from utils.zap_utils import check_and_set_ln_bits_keys
from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface
from nostr_dvm.utils.admin_utils import AdminConfig
from nostr_dvm.utils.backend_utils import keep_alive
from nostr_dvm.utils.definitions import EventDefinitions
from nostr_dvm.utils.dvmconfig import DVMConfig
from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag
from nostr_dvm.utils.nostr_utils import check_and_set_private_key
from nostr_dvm.utils.zap_utils import check_and_set_ln_bits_keys
from nostr_sdk import Keys
"""
@ -27,6 +27,7 @@ class TextGenerationOLLAMA(DVMTaskInterface):
KIND: int = EventDefinitions.KIND_NIP90_GENERATE_TEXT
TASK: str = "text-to-text"
FIX_COST: float = 0
dependencies = ["litellm==1.12.3"]
def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config,
admin_config: AdminConfig = None, options=None):
@ -71,6 +72,8 @@ class TextGenerationOLLAMA(DVMTaskInterface):
return request_form
def process(self, request_form):
from litellm import completion
options = DVMTaskInterface.set_options(request_form)
try:

17
tasks/translation_google.py → nostr_dvm/tasks/translation_google.py
View File

@ -4,14 +4,14 @@ from pathlib import Path
import dotenv
from interfaces.dvmtaskinterface import DVMTaskInterface
from utils.admin_utils import AdminConfig
from utils.backend_utils import keep_alive
from utils.definitions import EventDefinitions
from utils.dvmconfig import DVMConfig
from utils.nip89_utils import NIP89Config, check_and_set_d_tag
from utils.nostr_utils import get_referenced_event_by_id, get_event_by_id, check_and_set_private_key
from utils.zap_utils import check_and_set_ln_bits_keys
from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface
from nostr_dvm.utils.admin_utils import AdminConfig
from nostr_dvm.utils.backend_utils import keep_alive
from nostr_dvm.utils.definitions import EventDefinitions
from nostr_dvm.utils.dvmconfig import DVMConfig
from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag
from nostr_dvm.utils.nostr_utils import get_referenced_event_by_id, get_event_by_id, check_and_set_private_key
from nostr_dvm.utils.zap_utils import check_and_set_ln_bits_keys
from nostr_sdk import Keys
"""
@ -27,6 +27,7 @@ class TranslationGoogle(DVMTaskInterface):
KIND: int = EventDefinitions.KIND_NIP90_TRANSLATE_TEXT
TASK: str = "translation"
FIX_COST: float = 0
dependencies = ["translatepy==2.3"]
def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config,
admin_config: AdminConfig = None, options=None):

16
tasks/translation_libretranslate.py → nostr_dvm/tasks/translation_libretranslate.py
View File

@ -5,14 +5,14 @@ from pathlib import Path
import dotenv
import requests
from interfaces.dvmtaskinterface import DVMTaskInterface
from utils.admin_utils import AdminConfig
from utils.backend_utils import keep_alive
from utils.definitions import EventDefinitions
from utils.dvmconfig import DVMConfig
from utils.nip89_utils import NIP89Config, check_and_set_d_tag
from utils.nostr_utils import get_referenced_event_by_id, get_event_by_id, check_and_set_private_key
from utils.zap_utils import check_and_set_ln_bits_keys
from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface
from nostr_dvm.utils.admin_utils import AdminConfig
from nostr_dvm.utils.backend_utils import keep_alive
from nostr_dvm.utils.definitions import EventDefinitions
from nostr_dvm.utils.dvmconfig import DVMConfig
from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag
from nostr_dvm.utils.nostr_utils import get_referenced_event_by_id, get_event_by_id, check_and_set_private_key
from nostr_dvm.utils.zap_utils import check_and_set_ln_bits_keys
from nostr_sdk import Keys
"""

25
tasks/trending_notes_nostrband.py → nostr_dvm/tasks/trending_notes_nostrband.py
View File

@ -1,23 +1,19 @@
import json
import os
import re
from datetime import timedelta
from pathlib import Path
from threading import Thread
import dotenv
from nostr_sdk import Client, Timestamp, PublicKey, Tag, Keys, Options, Alphabet, SecretKey, Event
from nostr_sdk import Tag, Keys
from interfaces.dvmtaskinterface import DVMTaskInterface
from utils.admin_utils import AdminConfig
from utils.backend_utils import keep_alive
from utils.definitions import EventDefinitions
from utils.dvmconfig import DVMConfig
from utils.nip89_utils import NIP89Config, check_and_set_d_tag
from utils.nostr_utils import get_event_by_id, check_and_set_private_key
from utils.output_utils import post_process_list_to_users, post_process_list_to_events
from utils.zap_utils import check_and_set_ln_bits_keys
from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface
from nostr_dvm.utils.admin_utils import AdminConfig
from nostr_dvm.utils.backend_utils import keep_alive
from nostr_dvm.utils.definitions import EventDefinitions
from nostr_dvm.utils.dvmconfig import DVMConfig
from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag
from nostr_dvm.utils.nostr_utils import check_and_set_private_key
from nostr_dvm.utils.output_utils import post_process_list_to_events
from nostr_dvm.utils.zap_utils import check_and_set_ln_bits_keys
"""
This File contains a Module to search for notes
@ -63,7 +59,6 @@ class TrendingNotesNostrBand(DVMTaskInterface):
return request_form
def process(self, request_form):
from nostr_sdk import Filter
options = DVMTaskInterface.set_options(request_form)
import requests

18
tasks/videogeneration_replicate_svd.py → nostr_dvm/tasks/videogeneration_replicate_svd.py
View File

@ -8,15 +8,15 @@ import requests
import urllib.request
from PIL import Image
from interfaces.dvmtaskinterface import DVMTaskInterface
from utils.admin_utils import AdminConfig
from utils.backend_utils import keep_alive
from utils.definitions import EventDefinitions
from utils.dvmconfig import DVMConfig
from utils.nip89_utils import NIP89Config, check_and_set_d_tag
from utils.nostr_utils import check_and_set_private_key
from utils.output_utils import upload_media_to_hoster
from utils.zap_utils import get_price_per_sat, check_and_set_ln_bits_keys
from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface
from nostr_dvm.utils.admin_utils import AdminConfig
from nostr_dvm.utils.backend_utils import keep_alive
from nostr_dvm.utils.definitions import EventDefinitions
from nostr_dvm.utils.dvmconfig import DVMConfig
from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag
from nostr_dvm.utils.nostr_utils import check_and_set_private_key
from nostr_dvm.utils.output_utils import upload_media_to_hoster
from nostr_dvm.utils.zap_utils import get_price_per_sat, check_and_set_ln_bits_keys
from nostr_sdk import Keys
"""

0
nostr_dvm/utils/__init__.py Normal file
View File

13
utils/admin_utils.py → nostr_dvm/utils/admin_utils.py
View File

@ -1,15 +1,12 @@
# ADMINISTRARIVE DB MANAGEMENT
import time
from datetime import timedelta
from nostr_sdk import Keys, EventBuilder, PublicKey, Client, Filter, EventId
from nostr_sdk import Keys, PublicKey, Client
from utils.database_utils import get_from_sql_table, list_db, delete_from_sql_table, update_sql_table, \
from nostr_dvm.utils.database_utils import get_from_sql_table, list_db, delete_from_sql_table, update_sql_table, \
get_or_add_user, clean_db
from utils.dvmconfig import DVMConfig
from utils.nip89_utils import nip89_announce_tasks, NIP89Config, nip89_delete_announcement, \
fetch_nip89_paramters_for_deletion
from utils.nostr_utils import send_event, update_profile
from nostr_dvm.utils.dvmconfig import DVMConfig
from nostr_dvm.utils.nip89_utils import nip89_announce_tasks, fetch_nip89_paramters_for_deletion
from nostr_dvm.utils.nostr_utils import update_profile
class AdminConfig:

8
utils/backend_utils.py → nostr_dvm/utils/backend_utils.py
View File

@ -3,11 +3,11 @@ import signal
import time
import requests
from nostr_sdk import Event, Tag
from nostr_sdk import Event
from utils.definitions import EventDefinitions
from utils.mediasource_utils import check_source_type, media_source
from utils.nostr_utils import get_event_by_id, get_referenced_event_by_id
from nostr_dvm.utils.definitions import EventDefinitions
from nostr_dvm.utils.mediasource_utils import check_source_type, media_source
from nostr_dvm.utils.nostr_utils import get_event_by_id, get_referenced_event_by_id
def get_task(event, client, dvm_config):

5
utils/cashu_utils.py → nostr_dvm/utils/cashu_utils.py
View File

@ -1,9 +1,8 @@
import asyncio
import base64
import json
import requests
from utils.database_utils import get_or_add_user
from utils.zap_utils import create_bolt11_ln_bits, create_bolt11_lud16
from nostr_dvm.utils.database_utils import get_or_add_user
from nostr_dvm.utils.zap_utils import create_bolt11_ln_bits, create_bolt11_lud16
async def get_cashu_balance(url):

3
utils/database_utils.py → nostr_dvm/utils/database_utils.py
View File

@ -1,7 +1,6 @@
# DATABASE LOGIC
import json
import sqlite3
import time
from _sqlite3 import Error
from dataclasses import dataclass
@ -9,7 +8,7 @@ from datetime import timedelta
from logging import Filter
from nostr_sdk import Timestamp, Keys, PublicKey, EventBuilder, Filter
from utils.nostr_utils import send_event
from nostr_dvm.utils.nostr_utils import send_event
@dataclass

0
utils/definitions.py → nostr_dvm/utils/definitions.py
View File

4
utils/dvmconfig.py → nostr_dvm/utils/dvmconfig.py
View File

@ -1,5 +1,5 @@
from utils.nip89_utils import NIP89Config
from utils.output_utils import PostProcessFunctionType
from nostr_dvm.utils.nip89_utils import NIP89Config
from nostr_dvm.utils.output_utils import PostProcessFunctionType
class DVMConfig:

8
utils/external_dvm_utils.py → nostr_dvm/utils/external_dvm_utils.py
View File

@ -3,10 +3,10 @@ from datetime import timedelta
from nostr_sdk import PublicKey, Options, Keys, Client
from interfaces.dvmtaskinterface import DVMTaskInterface
from utils.dvmconfig import DVMConfig
from utils.nip89_utils import NIP89Config, nip89_fetch_events_pubkey
from utils.output_utils import PostProcessFunctionType
from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface
from nostr_dvm.utils.dvmconfig import DVMConfig
from nostr_dvm.utils.nip89_utils import NIP89Config, nip89_fetch_events_pubkey
from nostr_dvm.utils.output_utils import PostProcessFunctionType
def build_external_dvm(pubkey, task, kind, fix_cost, per_unit_cost, config,

14
utils/mediasource_utils.py → nostr_dvm/utils/mediasource_utils.py
View File

@ -3,9 +3,9 @@ import urllib
from datetime import time
from urllib.parse import urlparse
import ffmpegio
from decord import AudioReader, VideoReader, cpu
from decord import AudioReader, cpu
import requests
from utils.nostr_utils import get_event_by_id
from nostr_dvm.utils.nostr_utils import get_event_by_id
def input_data_file_duration(event, dvm_config, client, start=0, end=0):
@ -332,30 +332,30 @@ def get_media_link(url) -> (str, str):
def downloadOvercast(source_url, target_location):
from utils.scrapper.media_scrapper import OvercastDownload
from scrapper.media_scrapper import OvercastDownload
result = OvercastDownload(source_url, target_location)
return result
def downloadTwitter(videourl, path):
from utils.scrapper.media_scrapper import XitterDownload
from scrapper.media_scrapper import XitterDownload
result = XitterDownload(videourl, path + "x.mp4")
return result
def downloadTikTok(videourl, path):
from utils.scrapper.media_scrapper import TiktokDownloadAll
from scrapper.media_scrapper import TiktokDownloadAll
result = TiktokDownloadAll([videourl], path)
return result
def downloadInstagram(videourl, path):
from utils.scrapper.media_scrapper import InstagramDownload
from scrapper.media_scrapper import InstagramDownload
result = InstagramDownload(videourl, "insta", path)
return result
def downloadYouTube(link, path, audioonly=True):
from utils.scrapper.media_scrapper import YouTubeDownload
from scrapper.media_scrapper import YouTubeDownload
result = YouTubeDownload(link, path, audio_only=audioonly)
return result

6
utils/nip89_utils.py → nostr_dvm/utils/nip89_utils.py
View File

@ -4,10 +4,10 @@ from hashlib import sha256
from pathlib import Path
import dotenv
from nostr_sdk import Tag, Keys, EventBuilder, Filter, Alphabet, PublicKey, Event, Client, EventId
from nostr_sdk import Tag, Keys, EventBuilder, Filter, Alphabet, PublicKey, Client, EventId
from utils.definitions import EventDefinitions
from utils.nostr_utils import send_event
from nostr_dvm.utils.definitions import EventDefinitions
from nostr_dvm.utils.nostr_utils import send_event
class NIP89Config:

0
utils/nostr_utils.py → nostr_dvm/utils/nostr_utils.py
View File

0
utils/output_utils.py → nostr_dvm/utils/output_utils.py
View File

1
nostr_dvm/utils/scrapper/__init__.py Normal file
View File

@ -0,0 +1 @@
from dvm import DVM

0
utils/scrapper/media_scrapper.py → nostr_dvm/utils/scrapper/media_scrapper.py
View File

0
utils/scrapper/request_details.json → nostr_dvm/utils/scrapper/request_details.json
View File

4
utils/zap_utils.py → nostr_dvm/utils/zap_utils.py
View File

@ -9,8 +9,8 @@ from Crypto.Cipher import AES
from Crypto.Util.Padding import pad
from bech32 import bech32_decode, convertbits, bech32_encode
from nostr_sdk import nostr_sdk, PublicKey, SecretKey, Event, EventBuilder, Tag, Keys
from utils.dvmconfig import DVMConfig
from utils.nostr_utils import get_event_by_id, check_and_decrypt_own_tags, update_profile
from nostr_dvm.utils.dvmconfig import DVMConfig
from nostr_dvm.utils.nostr_utils import get_event_by_id, check_and_decrypt_own_tags
import lnurl
from hashlib import sha256
import dotenv

38
setup.py Normal file
View File

@ -0,0 +1,38 @@
from setuptools import setup, find_packages
VERSION = '0.0.1'
DESCRIPTION = 'A framework to build and run NIP90 Data Vending Machines'
LONG_DESCRIPTION = '-'
# Setting up
setup(
name="nostr-dvm",
version=VERSION,
author="Believethehype",
author_email="-",
description=DESCRIPTION,
long_description=LONG_DESCRIPTION,
packages=find_packages(include=['nostr_dvm']),
install_requires=["nostr-sdk==0.0.5",
"bech32==1.2.0",
"pycryptodome==3.19.0",
"python-dotenv==1.0.0",
"emoji==2.8.0",
"eva-decord==0.6.1",
"ffmpegio==0.8.5",
"lnurl==0.4.1",
"pandas==2.1.3",
"Pillow==10.1.0",
"PyUpload==0.1.4",
"requests==2.31.0",
"instaloader==4.10.1",
"pytube==15.0.0",
"moviepy==2.0.0.dev2"
],
keywords=['nostr', 'nip90', 'dvm', 'data vending machine'],
classifiers=[
"Development Status :: 3 - Alpha",
"Intended Audience :: Education",
"Programming Language :: Python :: 3",
]
)

2
tests/test_dvm_client.py
View File

@ -1,7 +1,5 @@
import json
import os
import time
import datetime as datetime
from pathlib import Path
from threading import Thread