refcactor

This commit is contained in:
Believethehype 2023-11-19 20:35:40 +01:00
parent 659067551b
commit 1a5b4bf024
7 changed files with 147 additions and 131 deletions

92
dvm.py
View File

@ -30,7 +30,7 @@ def dvm(config):
pk = keys.public_key()
print(f"Nostr DVM public key: {pk.to_bech32()}, Hex: {pk.to_hex()} ")
print(f"Supported DVM tasks: {dvm_config.SUPPORTED_TASKS}")
print('Supported DVM tasks: ' + ', '.join(p.TASK for p in dvm_config.SUPPORTED_TASKS))
client = Client(keys)
for relay in dvm_config.RELAY_LIST:
@ -38,10 +38,12 @@ def dvm(config):
client.connect()
dm_zap_filter = Filter().pubkey(pk).kinds([EventDefinitions.KIND_ZAP]).since(Timestamp.now())
dvm_filter = (Filter().kinds([EventDefinitions.KIND_NIP90_GENERIC,
EventDefinitions.KIND_NIP90_EXTRACT_TEXT,
EventDefinitions.KIND_NIP90_TRANSLATE_TEXT,
]).since(Timestamp.now()))
kinds = [EventDefinitions.KIND_NIP90_GENERIC]
for dvm in dvm_config.SUPPORTED_TASKS:
if dvm.KIND not in kinds:
kinds.append(dvm.KIND)
dvm_filter = (Filter().kinds(kinds).since(Timestamp.now()))
client.subscribe([dm_zap_filter, dvm_filter])
create_sql_table()
@ -53,7 +55,7 @@ def dvm(config):
print(f"[Nostr] Received new NIP90 Job Request from {relay_url}: {nostr_event.as_json()}")
handle_nip90_job_event(nostr_event, dvm_config)
elif nostr_event.kind() == EventDefinitions.KIND_ZAP:
handle_zap(nostr_event)
handle_zap(nostr_event, dvm_config)
def handle_msg(self, relay_url, msg):
return
@ -75,12 +77,17 @@ def dvm(config):
elif task_supported:
print("Received new Task: " + task)
print(duration)
amount = get_amount_per_task(task, duration, config=dvm_config)
amount = get_amount_per_task(task, dvm_config, duration)
if amount is None:
return
if is_whitelisted:
print("[Nostr] Whitelisted for task " + task + ". Starting processing..")
task_is_free = False
for dvm in dvm_config.SUPPORTED_TASKS:
if dvm.TASK == task and dvm.COST == 0:
task_is_free = True
if is_whitelisted or task_is_free:
print("[Nostr] Free or Whitelisted for task " + task + ". Starting processing..")
send_job_status_reaction(event, "processing", True, 0, client=client, config=dvm_config)
do_work(event, is_from_bot=False)
# otherwise send payment request
@ -94,9 +101,9 @@ def dvm(config):
if bid > 0:
bid_offer = int(bid / 1000)
if bid_offer >= amount:
send_job_status_reaction(event, "payment-required", False,
amount, # bid_offer
client=client, config=dvm_config)
send_job_status_reaction(event, "payment-required", False,
amount, # bid_offer
client=client, config=dvm_config)
else: # If there is no bid, just request server rate from user
print("[Nostr] Requesting payment for Event: " + event.id().to_hex())
@ -105,7 +112,7 @@ def dvm(config):
else:
print("Task not supported on this DVM, skipping..")
def handle_zap(event):
def handle_zap(event, dvm_config):
zapped_event = None
invoice_amount = 0
anon = False
@ -205,38 +212,17 @@ def dvm(config):
or job_event.kind() == EventDefinitions.KIND_DM):
task = get_task(job_event, client=client, dvmconfig=dvm_config)
result = ""
try:
if task == Translation.TASK:
request_form = Translation.create_requestform_from_nostr_event(job_event,client,dvm_config)
options = setOptions(request_form)
result = Translation.process(options)
for dvm in dvm_config.SUPPORTED_TASKS:
try:
if task == dvm.TASK:
request_form = dvm.create_request_form_from_nostr_event(job_event, client, dvm_config)
result = dvm.process(request_form)
check_and_return_event(result, str(job_event.as_json()), dvm_key=dvm_config.PRIVATE_KEY)
elif task == TextExtractionPDF.TASK:
request_form = TextExtractionPDF.create_requestform_from_nostr_event(job_event, client, dvm_config)
options = setOptions(request_form)
result = TextExtractionPDF.process(options)
except Exception as e:
respond_to_error(e, job_event.as_json(), is_from_bot, dvm_config.PRIVATE_KEY)
#TODO Add more tasks here
check_and_return_event(result, str(job_event.as_json()), dvm_key=dvm_config.PRIVATE_KEY)
except Exception as e:
respond_to_error(e, job_event.as_json(), is_from_bot, dvm_config.PRIVATE_KEY)
def setOptions(request_form):
print("Setting options...")
opts = []
if request_form.get("optStr"):
for k, v in [option.split("=") for option in request_form["optStr"].split(";")]:
t = (k, v)
opts.append(t)
print(k + "=" + v)
print("...done.")
return dict(opts)
def check_event_has_not_unifinished_job_input(nevent, append, client, dvmconfig):
def check_event_has_not_unfinished_job_input(nevent, append, client, dvmconfig):
task_supported, task, duration = check_task_is_supported(nevent, client, False, config=dvmconfig)
if not task_supported:
return False
@ -400,6 +386,7 @@ def dvm(config):
send_event(event, key=key)
print("[Nostr] " + str(response_kind) + " Job Response event sent: " + event.as_json())
return event.as_json()
client.handle_notifications(NotificationHandler())
def respond_to_error(content, originaleventstr, is_from_bot=False, dvm_key=None):
@ -425,7 +412,7 @@ def dvm(config):
user = get_from_sql_table(sender)
is_whitelisted = user[2]
if not is_whitelisted:
amount = int(user[1]) + get_amount_per_task(task)
amount = int(user[1]) + get_amount_per_task(task, dvm_config)
update_sql_table(sender, amount, user[2], user[3], user[4], user[5], user[6],
Timestamp.now().as_secs())
message = "There was the following error : " + content + ". Credits have been reimbursed"
@ -454,7 +441,7 @@ def dvm(config):
job_list.remove(job)
for job in jobs_on_hold_list:
if check_event_has_not_unifinished_job_input(job.event, False, client=client, dvmconfig=dvm_config):
if check_event_has_not_unfinished_job_input(job.event, False, client=client, dvmconfig=dvm_config):
handle_nip90_job_event(job.event)
jobs_on_hold_list.remove(job)
@ -462,18 +449,3 @@ def dvm(config):
jobs_on_hold_list.remove(job)
time.sleep(1.0)

View File

@ -0,0 +1,32 @@
class DVMTaskInterface:
TASK: str
KIND: int
COST: int
def NIP89_announcement(self):
"""Define the NIP89 Announcement"""
pass
def is_input_supported(self, input_type, input_content) -> bool:
"""Check if input is supported for current Task."""
pass
def create_request_form_from_nostr_event(self, event, client=None, dvm_config=None) -> dict:
"""Parse input into a request form that will be given to the process method"""
pass
def process(self, request_form):
"Process the data and return the result"
pass
@staticmethod
def setOptions(request_form):
print("Setting options...")
opts = []
if request_form.get("optStr"):
for k, v in [option.split("=") for option in request_form["optStr"].split(";")]:
t = (k, v)
opts.append(t)
print(k + "=" + v)
print("...done.")
return dict(opts)

View File

@ -12,16 +12,19 @@ from utils.definitions import EventDefinitions
def run_nostr_dvm_with_local_config():
from dvm import dvm, DVMConfig
PDFextactor = TextExtractionPDF("PDF Extractor", env.NOSTR_PRIVATE_KEY)
Translator = Translation("Translator", env.NOSTR_PRIVATE_KEY)
dvmconfig = DVMConfig()
dvmconfig.PRIVATE_KEY = os.getenv(env.NOSTR_PRIVATE_KEY)
dvmconfig.SUPPORTED_TASKS = [Translation.TASK, TextExtractionPDF.TASK]
dvmconfig.SUPPORTED_TASKS = [PDFextactor, Translator]
dvmconfig.LNBITS_INVOICE_KEY = os.getenv(env.LNBITS_INVOICE_KEY)
dvmconfig.LNBITS_URL = os.getenv(env.LNBITS_HOST)
# In admin_utils, set rebroadcast_nip89 to true to (re)broadcast your DVM. You can create a valid dtag and the content on vendata.io
# Add the dtag in your .env file so you can update your dvm later and change the content in the module file as needed.
dvmconfig.NIP89s.append(TextExtractionPDF.NIP89_announcement())
dvmconfig.NIP89s.append(Translation.NIP89_announcement())
dvmconfig.NIP89s.append(PDFextactor.NIP89_announcement())
dvmconfig.NIP89s.append(Translator.NIP89_announcement())
nostr_dvm_thread = Thread(target=dvm, args=[dvmconfig])
nostr_dvm_thread.start()

View File

@ -1,34 +1,38 @@
import os
from typing import re
import re
from interfaces.dvmtaskinterface import DVMTaskInterface
from utils import env
from utils.definitions import EventDefinitions
from utils.nip89_utils import NIP89Announcement
from utils.nostr_utils import get_event_by_id, get_referenced_event_by_id
class TextExtractionPDF:
class TextExtractionPDF(DVMTaskInterface):
TASK: str = "pdf-to-text"
KIND: int = EventDefinitions.KIND_NIP90_EXTRACT_TEXT
COST: int = 20
@staticmethod
def NIP89_announcement():
def __init__(self, name, pk):
self.NAME = name
self.PK = pk
def NIP89_announcement(self):
nip89 = NIP89Announcement()
nip89.kind = EventDefinitions.KIND_NIP90_TRANSLATE_TEXT
nip89.dtag = os.getenv(env.TASK_TRANSLATION_NIP89_DTAG)
nip89.pk = os.getenv(env.NOSTR_PRIVATE_KEY)
nip89.content = "{\"name\":\"Translator\",\"image\":\"https://image.nostr.build/c33ca6fc4cc038ca4adb46fdfdfda34951656f87ee364ef59095bae1495ce669.jpg\",\"about\":\"I translate text from given text/event/job, currently using Google Translation Services into language defined in param. \",\"nip90Params\":{\"language\":{\"required\":true,\"values\":[\"af\",\"am\",\"ar\",\"az\",\"be\",\"bg\",\"bn\",\"bs\",\"ca\",\"ceb\",\"co\",\"cs\",\"cy\",\"da\",\"de\",\"el\",\"eo\",\"es\",\"et\",\"eu\",\"fa\",\"fi\",\"fr\",\"fy\",\"ga\",\"gd\",\"gl\",\"gu\",\"ha\",\"haw\",\"hi\",\"hmn\",\"hr\",\"ht\",\"hu\",\"hy\",\"id\",\"ig\",\"is\",\"it\",\"he\",\"ja\",\"jv\",\"ka\",\"kk\",\"km\",\"kn\",\"ko\",\"ku\",\"ky\",\"la\",\"lb\",\"lo\",\"lt\",\"lv\",\"mg\",\"mi\",\"mk\",\"ml\",\"mn\",\"mr\",\"ms\",\"mt\",\"my\",\"ne\",\"nl\",\"no\",\"ny\",\"or\",\"pa\",\"pl\",\"ps\",\"pt\",\"ro\",\"ru\",\"sd\",\"si\",\"sk\",\"sl\",\"sm\",\"sn\",\"so\",\"sq\",\"sr\",\"st\",\"su\",\"sv\",\"sw\",\"ta\",\"te\",\"tg\",\"th\",\"tl\",\"tr\",\"ug\",\"uk\",\"ur\",\"uz\",\"vi\",\"xh\",\"yi\",\"yo\",\"zh\",\"zu\"]}}}"
nip89.kind = self.KIND
nip89.pk = self.PK
nip89.dtag = os.getenv(env.TASK_TEXTEXTRACTION_NIP89_DTAG)
nip89.content = "{\"name\":\"" + self.NAME + "\",\"image\":\"https://image.nostr.build/c33ca6fc4cc038ca4adb46fdfdfda34951656f87ee364ef59095bae1495ce669.jpg\",\"about\":\"I extract Text from pdf documents\",\"nip90Params\":{}}"
return nip89
@staticmethod
def is_input_supported(input_type, input_content):
def is_input_supported(self, input_type, input_content):
if input_type != "url":
return False
return True
@staticmethod
def create_requestform_from_nostr_event(event, client=None, dvmconfig=None):
def create_request_form_from_nostr_event(self, event, client=None, dvm_config=None):
request_form = {"jobID": event.id().to_hex()}
# default values
@ -44,31 +48,35 @@ class TextExtractionPDF:
if input_type == "url":
url = input_content
elif input_type == "event":
evt = get_event_by_id(input_content, config=dvmconfig)
evt = get_event_by_id(input_content, config=dvm_config)
url = re.search("(?P<url>https?://[^\s]+)", evt.content()).group("url")
elif input_type == "job":
evt = get_referenced_event_by_id(input_content, [EventDefinitions.KIND_NIP90_RESULT_GENERATE_IMAGE],
client, config=dvmconfig)
client, config=dvm_config)
url = re.search("(?P<url>https?://[^\s]+)", evt.content()).group("url")
request_form["optStr"] = 'url=' + url
return request_form
@staticmethod
def process(options):
def process(self, request_form):
options = DVMTaskInterface.setOptions(request_form)
from pypdf import PdfReader
from pathlib import Path
import requests
file_path = Path('temp.pdf')
response = requests.get(options["url"])
file_path.write_bytes(response.content)
reader = PdfReader(file_path)
number_of_pages = len(reader.pages)
text = ""
for page_num in range(number_of_pages):
page = reader.pages[page_num]
text = text + page.extract_text()
try:
file_path = Path('temp.pdf')
response = requests.get(options["url"])
file_path.write_bytes(response.content)
reader = PdfReader(file_path)
number_of_pages = len(reader.pages)
text = ""
for page_num in range(number_of_pages):
page = reader.pages[page_num]
text = text + page.extract_text()
os.remove('temp.pdf')
return text
os.remove('temp.pdf')
return text
except Exception as e:
raise Exception(e)

View File

@ -1,35 +1,37 @@
import os
from interfaces.dvmtaskinterface import DVMTaskInterface
from utils import env
from utils.definitions import EventDefinitions
from utils.nip89_utils import NIP89Announcement
from utils.nostr_utils import get_referenced_event_by_id, get_event_by_id
class Translation:
class Translation(DVMTaskInterface):
TASK: str = "translation"
COST: int = 20
KIND: int = EventDefinitions.KIND_NIP90_TRANSLATE_TEXT
COST: int = 0
@staticmethod
def NIP89_announcement():
def __init__(self, name, pk):
self.NAME = name
self.PK = pk
def NIP89_announcement(self):
nip89 = NIP89Announcement()
nip89.kind = EventDefinitions.KIND_NIP90_TRANSLATE_TEXT
nip89.kind = self.KIND
nip89.pk = self.PK
nip89.dtag = os.getenv(env.TASK_TRANSLATION_NIP89_DTAG)
nip89.pk = os.getenv(env.NOSTR_PRIVATE_KEY)
nip89.content = "{\"name\":\"Translator\",\"image\":\"https://image.nostr.build/c33ca6fc4cc038ca4adb46fdfdfda34951656f87ee364ef59095bae1495ce669.jpg\",\"about\":\"I translate text from given text/event/job, currently using Google Translation Services into language defined in param. \",\"nip90Params\":{\"language\":{\"required\":true,\"values\":[\"af\",\"am\",\"ar\",\"az\",\"be\",\"bg\",\"bn\",\"bs\",\"ca\",\"ceb\",\"co\",\"cs\",\"cy\",\"da\",\"de\",\"el\",\"eo\",\"es\",\"et\",\"eu\",\"fa\",\"fi\",\"fr\",\"fy\",\"ga\",\"gd\",\"gl\",\"gu\",\"ha\",\"haw\",\"hi\",\"hmn\",\"hr\",\"ht\",\"hu\",\"hy\",\"id\",\"ig\",\"is\",\"it\",\"he\",\"ja\",\"jv\",\"ka\",\"kk\",\"km\",\"kn\",\"ko\",\"ku\",\"ky\",\"la\",\"lb\",\"lo\",\"lt\",\"lv\",\"mg\",\"mi\",\"mk\",\"ml\",\"mn\",\"mr\",\"ms\",\"mt\",\"my\",\"ne\",\"nl\",\"no\",\"ny\",\"or\",\"pa\",\"pl\",\"ps\",\"pt\",\"ro\",\"ru\",\"sd\",\"si\",\"sk\",\"sl\",\"sm\",\"sn\",\"so\",\"sq\",\"sr\",\"st\",\"su\",\"sv\",\"sw\",\"ta\",\"te\",\"tg\",\"th\",\"tl\",\"tr\",\"ug\",\"uk\",\"ur\",\"uz\",\"vi\",\"xh\",\"yi\",\"yo\",\"zh\",\"zu\"]}}}"
nip89.content = "{\"name\":\"" + self.NAME + "\",\"image\":\"https://image.nostr.build/c33ca6fc4cc038ca4adb46fdfdfda34951656f87ee364ef59095bae1495ce669.jpg\",\"about\":\"I translate text from given text/event/job, currently using Google Translation Services into language defined in param. \",\"nip90Params\":{\"language\":{\"required\":true,\"values\":[\"af\",\"am\",\"ar\",\"az\",\"be\",\"bg\",\"bn\",\"bs\",\"ca\",\"ceb\",\"co\",\"cs\",\"cy\",\"da\",\"de\",\"el\",\"eo\",\"es\",\"et\",\"eu\",\"fa\",\"fi\",\"fr\",\"fy\",\"ga\",\"gd\",\"gl\",\"gu\",\"ha\",\"haw\",\"hi\",\"hmn\",\"hr\",\"ht\",\"hu\",\"hy\",\"id\",\"ig\",\"is\",\"it\",\"he\",\"ja\",\"jv\",\"ka\",\"kk\",\"km\",\"kn\",\"ko\",\"ku\",\"ky\",\"la\",\"lb\",\"lo\",\"lt\",\"lv\",\"mg\",\"mi\",\"mk\",\"ml\",\"mn\",\"mr\",\"ms\",\"mt\",\"my\",\"ne\",\"nl\",\"no\",\"ny\",\"or\",\"pa\",\"pl\",\"ps\",\"pt\",\"ro\",\"ru\",\"sd\",\"si\",\"sk\",\"sl\",\"sm\",\"sn\",\"so\",\"sq\",\"sr\",\"st\",\"su\",\"sv\",\"sw\",\"ta\",\"te\",\"tg\",\"th\",\"tl\",\"tr\",\"ug\",\"uk\",\"ur\",\"uz\",\"vi\",\"xh\",\"yi\",\"yo\",\"zh\",\"zu\"]}}}"
return nip89
@staticmethod
def is_input_supported(input_type, input_content):
def is_input_supported(self, input_type, input_content):
if input_type != "event" and input_type != "job" and input_type != "text":
return False
if input_type != "text" and len(input_content) > 4999:
return False
return True
@staticmethod
def create_requestform_from_nostr_event(event, client=None, dvmconfig=None):
def create_request_form_from_nostr_event(self, event, client=None, dvm_config=None):
request_form = {"jobID": event.id().to_hex()}
#default values
@ -50,7 +52,7 @@ class Translation:
if input_type == "event":
for tag in event.tags():
if tag.as_vec()[0] == 'i':
evt = get_event_by_id(tag.as_vec()[1], config=dvmconfig)
evt = get_event_by_id(tag.as_vec()[1], config=dvm_config)
text = evt.content()
break
@ -67,7 +69,7 @@ class Translation:
[EventDefinitions.KIND_NIP90_RESULT_EXTRACT_TEXT,
EventDefinitions.KIND_NIP90_RESULT_SUMMARIZE_TEXT],
client,
config=dvmconfig)
config=dvm_config)
text = evt.content()
break
@ -76,8 +78,8 @@ class Translation:
replace(";", ","))
return request_form
@staticmethod
def process(options):
def process(self, request_form):
options = DVMTaskInterface.setOptions(request_form)
from translatepy.translators.google import GoogleTranslate
gtranslate = GoogleTranslate()
length = len(options["text"])
@ -101,8 +103,8 @@ class Translation:
try:
translated_text_part = str(gtranslate.translate(textpart, options["translation_lang"]))
print("Translated Text part:\n\n " + translated_text_part)
except:
translated_text_part = "An error occured"
except Exception as e:
raise Exception(e)
translated_text = translated_text + translated_text_part

View File

@ -82,18 +82,17 @@ def check_task_is_supported(event, client, get_duration=False, config=None):
task = get_task(event, client=client, dvmconfig=dvm_config)
if task not in dvm_config.SUPPORTED_TASKS: # The Tasks this DVM supports (can be extended)
return False, task, duration
if input_type == 'url' and check_url_is_readable(input_value) is None:
print("url not readable")
return False, task, duration
if task == Translation.TASK:
return Translation.is_input_supported(input_type, event.content()), task, duration
if task not in (x.TASK for x in dvm_config.SUPPORTED_TASKS):
return False, task, duration
elif task == TextExtractionPDF.TASK:
return TextExtractionPDF.is_input_supported(input_type, event.content()), task, duration
for dvm in dvm_config.SUPPORTED_TASKS:
if dvm.TASK == task:
if not dvm.is_input_supported(input_type, event.content()):
return False, task, duration
return True, task, duration
@ -101,7 +100,6 @@ def check_task_is_supported(event, client, get_duration=False, config=None):
def check_url_is_readable(url):
if not str(url).startswith("http"):
return None
# If link is comaptible with one of these file formats, move on.
req = requests.get(url)
content_type = req.headers['content-type']
@ -122,13 +120,12 @@ def check_url_is_readable(url):
return None
def get_amount_per_task(task, duration=0, config=None):
if task == Translation.TASK:
amount = Translation.COST
elif task == TextExtractionPDF.TASK:
amount = TextExtractionPDF.COST
def get_amount_per_task(task, dvm_config, duration=1):
print(dvm_config.SUPPORTED_TASKS)
for dvm in dvm_config.SUPPORTED_TASKS:
if dvm.TASK == task:
amount = dvm.COST * duration
return amount
else:
print("[Nostr] Task " + task + " is currently not supported by this instance, skipping")
return None
return amount

View File

@ -1,10 +1,14 @@
from dataclasses import dataclass
from nostr_sdk import Event
NEW_USER_BALANCE = 250
NEW_USER_BALANCE: int = 250 # Free credits for new users
RELAY_LIST = ["wss://relay.damus.io", "wss://nostr-pub.wellorder.net", "wss://nos.lol", "wss://nostr.wine",
"wss://relay.nostfiles.dev", "wss://nostr.mom", "wss://nostr.oxtr.dev", "wss://relay.nostr.bg", "wss://relay.f7z.io"]
"wss://relay.nostfiles.dev", "wss://nostr.mom", "wss://nostr.oxtr.dev", "wss://relay.nostr.bg",
"wss://relay.f7z.io"]
class EventDefinitions:
KIND_DM: int = 4
KIND_ZAP: int = 9735
@ -41,22 +45,19 @@ class DVMConfig:
PRIVATE_KEY: str
RELAY_LIST = ["wss://relay.damus.io", "wss://nostr-pub.wellorder.net", "wss://nos.lol", "wss://nostr.wine",
"wss://relay.nostfiles.dev", "wss://nostr.mom", "wss://nostr.oxtr.dev", "wss://relay.nostr.bg", "wss://relay.f7z.io"]
"wss://relay.nostfiles.dev", "wss://nostr.mom", "wss://nostr.oxtr.dev", "wss://relay.nostr.bg",
"wss://relay.f7z.io"]
RELAY_TIMEOUT = 5
LNBITS_INVOICE_KEY = ''
LNBITS_URL = 'https://lnbits.com'
REQUIRES_NIP05: bool = False
SHOWRESULTBEFOREPAYMENT: bool = True # if this is true show results even when not paid right after autoprocess
NEW_USER_BALANCE: int = 250 # Free credits for new users
NIP89s: list = []
@dataclass
class JobToWatch:
event_id: str
@ -71,7 +72,8 @@ class JobToWatch:
expires: int
from_bot: bool
@dataclass
class RequiredJobToWatch:
event: Event
timestamp: int
timestamp: int