highperfocused/nostrdvm
1
0
Fork 0
mirror of https://github.com/believethehype/nostrdvm.git synced 2025-11-18 21:21:32 +01:00
Code Issues Packages Projects Releases Wiki Activity

initial commit with translation use case

Browse Source
This commit is contained in:
Believethehype
2023-11-18 20:20:18 +01:00
parent ce319b238d
commit c450c8544f
27 changed files with 1604 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
.env_example Normal file
View File

@@ -0,0 +1,8 @@
NOSTR_PRIVATE_KEY = nostrSecretkeyinhex
NOSTR_TEST_CLIENT_PRIVATE_KEY = nostrSecretkeyinhex_forthetestclient
USER_DB_PATH = nostrzaps.db
LNBITS_INVOICE_KEY = lnbitswalletinvoicekey
LNBITS_HOST = https://lnbits.com
TASK_TRANSLATION_NIP89_DTAG = abcded

2
.gitignore vendored
View File

@@ -158,3 +158,5 @@ cython_debug/
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/
nostrzaps.db
.DS_Store

8
.idea/.gitignore generated vendored Normal file
View File

@@ -0,0 +1,8 @@
# Default ignored files
/shelf/
/workspace.xml
# Editor-based HTTP Client requests
/httpRequests/
# Datasource local storage ignored files
/dataSources/
/dataSources.local.xml

12
.idea/dataSources.xml generated Normal file
View File

@@ -0,0 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="DataSourceManagerImpl" format="xml" multifile-model="true">
<data-source source="LOCAL" name="nostrzaps" uuid="1b903aed-7909-4348-8b47-a953702395ff">
<driver-ref>sqlite.xerial</driver-ref>
<synchronize>true</synchronize>
<jdbc-driver>org.sqlite.JDBC</jdbc-driver>
<jdbc-url>jdbc:sqlite:$PROJECT_DIR$/nostrzaps.db</jdbc-url>
<working-dir>$ProjectFileDir$</working-dir>
</data-source>
</component>
</project>

10
.idea/dvm.iml generated Normal file
View File

@@ -0,0 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$">
<excludeFolder url="file://$MODULE_DIR$/venv" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>

6
.idea/inspectionProfiles/profiles_settings.xml generated Normal file
View File

@@ -0,0 +1,6 @@
<component name="InspectionProjectProfileManager">
<settings>
<option name="USE_PROJECT_PROFILE" value="false" />
<version value="1.0" />
</settings>
</component>

7
.idea/misc.xml generated Normal file
View File

@@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Black">
<option name="sdkName" value="Python 3.10 (dvm)" />
</component>
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (dvm)" project-jdk-type="Python SDK" />
</project>

8
.idea/modules.xml generated Normal file
View File

@@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/dvm.iml" filepath="$PROJECT_DIR$/.idea/dvm.iml" />
</modules>
</component>
</project>

6
.idea/sqldialects.xml generated Normal file
View File

@@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="SqlDialectMappings">
<file url="PROJECT" dialect="SQLite" />
</component>
</project>

6
.idea/vcs.xml generated Normal file
View File

@@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="" vcs="Git" />
</component>
</project>

13
README.md
View File

@@ -1,3 +1,12 @@
# Nostr Data Vending Machine
# Nostr Data Vending Machine Python Implementation
A basic Implementation of a DVM written in Python
This example DVM implementation in Python currently supports simple translations using Google translate.
At a later stage, additional example tasks will be added, as well as the integration into a larger Machine Learning backend
Place .env file (based on .env_example) in main folder, install requirements.txt (python 3.10) run main.py. Optionally supports LNbits to create invoices instead of lnaddresses.
Use vendata.io to create a nip89 announcement of your dvm and save the dtag in your .env config.
A tutorial on how to add additional tasks, as well as the larger server backend will be added soon.

470
dvm.py Normal file
View File

@@ -0,0 +1,470 @@
from nostr_sdk import PublicKey, Keys, Client, Tag, Event, EventBuilder, Filter, HandleNotification, Timestamp, \
init_logger, LogLevel
import time
import emoji
from utils.definitions import EventDefinitions, DVMConfig, RequiredJobToWatch, JobToWatch, LOCAL_TASKS
from utils.admin_utils import admin_make_database_updates
from utils.ai_utils import GoogleTranslate
from utils.backend_utils import get_amount_per_task, check_task_is_supported, get_task
from utils.database_utils import update_sql_table, get_from_sql_table, \
create_sql_table, get_or_add_user, update_user_balance
from utils.nostr_utils import get_event_by_id, get_referenced_event_by_id, send_event
from utils.output_utils import post_process_result
from utils.requestform_utils import create_requestform_from_nostr_event
from utils.zap_utils import check_bolt11_ln_bits_is_paid, parse_bolt11_invoice, \
check_for_zapplepay, decrypt_private_zap_message, create_bolt11_ln_bits
use_logger = False
if use_logger:
init_logger(LogLevel.DEBUG)
job_list = []
jobs_on_hold_list = []
dvm_config = DVMConfig()
def dvm(config):
dvm_config = config
keys = Keys.from_sk_str(dvm_config.PRIVATE_KEY)
pk = keys.public_key()
print(f"Nostr DVM public key: {pk.to_bech32()}, Hex: {pk.to_hex()} ")
print(f"Supported DVM tasks: {dvm_config.SUPPORTED_TASKS}")
client = Client(keys)
for relay in dvm_config.RELAY_LIST:
client.add_relay(relay)
client.connect()
dm_zap_filter = Filter().pubkey(pk).kinds([EventDefinitions.KIND_ZAP]).since(Timestamp.now())
dvm_filter = (Filter().kinds([EventDefinitions.KIND_NIP90_GENERIC,
EventDefinitions.KIND_NIP90_TRANSLATE_TEXT,
]).since(Timestamp.now()))
client.subscribe([dm_zap_filter, dvm_filter])
create_sql_table()
admin_make_database_updates(config=dvm_config, client=client)
class NotificationHandler(HandleNotification):
def handle(self, relay_url, nostr_event):
if EventDefinitions.KIND_NIP90_EXTRACT_TEXT <= nostr_event.kind() <= EventDefinitions.KIND_NIP90_GENERIC:
print(f"[Nostr] Received new NIP90 Job Request from {relay_url}: {nostr_event.as_json()}")
handle_nip90_job_event(nostr_event)
elif nostr_event.kind() == EventDefinitions.KIND_ZAP:
handle_zap(nostr_event)
def handle_msg(self, relay_url, msg):
return
def handle_nip90_job_event(event):
user = get_or_add_user(event.pubkey().to_hex())
is_whitelisted = user[2]
is_blacklisted = user[3]
if is_whitelisted:
task_supported, task, duration = check_task_is_supported(event, client=client, get_duration=False,
config=dvm_config)
print(task)
else:
task_supported, task, duration = check_task_is_supported(event, client=client, get_duration=True,
config=dvm_config)
if is_blacklisted:
send_job_status_reaction(event, "error", client=client, config=dvm_config)
print("[Nostr] Request by blacklisted user, skipped")
elif task_supported:
print("Received new Task: " + task)
print(duration)
amount = get_amount_per_task(task, duration, config=dvm_config)
if amount is None:
return
if is_whitelisted:
print("[Nostr] Whitelisted for task " + task + ". Starting processing..")
send_job_status_reaction(event, "processing", True, 0, client=client, config=dvm_config)
do_work(event, is_from_bot=False)
# otherwise send payment request
else:
bid = 0
for tag in event.tags():
if tag.as_vec()[0] == 'bid':
bid = int(tag.as_vec()[1])
print("[Nostr][Payment required] New Nostr " + task + " Job event: " + event.as_json())
if bid > 0:
bid_offer = int(bid / 1000)
if bid_offer >= amount:
send_job_status_reaction(event, "payment-required", False,
amount, # bid_offer
client=client, config=dvm_config)
else: # If there is no bid, just request server rate from user
print("[Nostr] Requesting payment for Event: " + event.id().to_hex())
send_job_status_reaction(event, "payment-required",
False, amount, client=client, config=dvm_config)
else:
print("Task not supported on this DVM, skipping..")
def handle_zap(event):
zapped_event = None
invoice_amount = 0
anon = False
sender = event.pubkey()
try:
for tag in event.tags():
if tag.as_vec()[0] == 'bolt11':
invoice_amount = parse_bolt11_invoice(tag.as_vec()[1])
elif tag.as_vec()[0] == 'e':
zapped_event = get_event_by_id(tag.as_vec()[1], config=dvm_config)
elif tag.as_vec()[0] == 'description':
zap_request_event = Event.from_json(tag.as_vec()[1])
sender = check_for_zapplepay(zap_request_event.pubkey().to_hex(),
zap_request_event.content())
for ztag in zap_request_event.tags():
if ztag.as_vec()[0] == 'anon':
if len(ztag.as_vec()) > 1:
print("Private Zap received.")
decrypted_content = decrypt_private_zap_message(ztag.as_vec()[1],
keys.secret_key(),
zap_request_event.pubkey())
decrypted_private_event = Event.from_json(decrypted_content)
if decrypted_private_event.kind() == 9733:
sender = decrypted_private_event.pubkey().to_hex()
message = decrypted_private_event.content()
if message != "":
print("Zap Message: " + message)
else:
anon = True
print("Anonymous Zap received. Unlucky, I don't know from whom, and never will")
user = get_or_add_user(sender)
print(str(user))
if zapped_event is not None:
if zapped_event.kind() == EventDefinitions.KIND_FEEDBACK: # if a reaction by us got zapped
if not dvm_config.IS_BOT:
print("Zap received for NIP90 task: " + str(invoice_amount) + " Sats from " + str(
user[6]))
amount = 0
job_event = None
for tag in zapped_event.tags():
if tag.as_vec()[0] == 'amount':
amount = int(float(tag.as_vec()[1]) / 1000)
elif tag.as_vec()[0] == 'e':
job_event = get_event_by_id(tag.as_vec()[1], config=dvm_config)
task_supported, task, duration = check_task_is_supported(job_event, client=client,
get_duration=False,
config=dvm_config)
if job_event is not None and task_supported:
if amount <= invoice_amount:
print("[Nostr] Payment-request fulfilled...")
send_job_status_reaction(job_event, "processing", client=client,
config=dvm_config)
indices = [i for i, x in enumerate(job_list) if
x.event_id == job_event.id().to_hex()]
index = -1
if len(indices) > 0:
index = indices[0]
if index > -1:
if job_list[index].is_processed: # If payment-required appears a processing
job_list[index].is_paid = True
check_and_return_event(job_list[index].result, str(job_event.as_json()),
dvm_key=dvm_config.PRIVATE_KEY)
elif not (job_list[index]).is_processed:
# If payment-required appears before processing
job_list.pop(index)
print("Starting work...")
do_work(job_event, is_from_bot=False)
else:
print("Job not in List, but starting work...")
do_work(job_event, is_from_bot=False)
else:
send_job_status_reaction(job_event, "payment-rejected",
False, invoice_amount, client=client, config=dvm_config)
print("[Nostr] Invoice was not paid sufficiently")
elif zapped_event.kind() in EventDefinitions.ANY_RESULT:
print("Someone zapped the result of an exisiting Task. Nice")
elif not anon and not dvm_config.PASSIVE_MODE:
print("Note Zap received for Bot balance: " + str(invoice_amount) + " Sats from " + str(
user[6]))
update_user_balance(sender, invoice_amount, config=dvm_config)
# a regular note
elif not anon and not dvm_config.PASSIVE_MODE:
print("Profile Zap received for Bot balance: " + str(invoice_amount) + " Sats from " + str(
user[6]))
update_user_balance(sender, invoice_amount, config=dvm_config)
except Exception as e:
print(f"Error during content decryption: {e}")
def do_work(job_event, is_from_bot=False):
if ((
EventDefinitions.KIND_NIP90_EXTRACT_TEXT <= job_event.kind() <= EventDefinitions.KIND_NIP90_GENERIC)
or job_event.kind() == EventDefinitions.KIND_DM):
#We're building a request form here because we want to send larger tasks to a processing server later
request_form = create_requestform_from_nostr_event(job_event, is_from_bot, client, dvm_config)
task = get_task(job_event, client=client, dvmconfig=dvm_config)
# TASKS that take time or need GPU are moved to a backend server in the future
if task not in LOCAL_TASKS:
if task.startswith("unknown"):
print("Task not (yet) supported")
return
else:
print("[Nostr] Scheduling " + task + " Job event: " + job_event.as_json())
print("We will employ a backend server here in the future")
else:
print("[Nostr] Scheduling local " + task + " Job event: " + job_event.as_json())
result = ""
print("Setting options...")
opts = []
if request_form.get("optStr"):
for k, v in [option.split("=") for option in request_form["optStr"].split(";")]:
t = (k, v)
opts.append(t)
print(k + "=" + v)
print("...done.")
options = dict(opts)
if task == "translation":
result = GoogleTranslate(options["text"], options["translation_lang"])
# TODO ADD FURTHER LOCAL TASKS HERE
check_and_return_event(result, str(job_event.as_json()),
dvm_key=dvm_config.PRIVATE_KEY)
client.handle_notifications(NotificationHandler())
while True:
for job in job_list:
if job.bolt11 != "" and job.payment_hash != "" and not job.is_paid:
if str(check_bolt11_ln_bits_is_paid(job.payment_hash, dvm_config)) == "True":
job.is_paid = True
event = get_event_by_id(job.event_id, config=dvm_config)
if event != None:
send_job_status_reaction(event, "processing", True, 0, client=client, config=dvm_config)
print("do work from joblist")
do_work(event, is_from_bot=False)
elif check_bolt11_ln_bits_is_paid(job.payment_hash, dvm_config) is None: # invoice expired
job_list.remove(job)
if Timestamp.now().as_secs() > job.expires:
job_list.remove(job)
for job in jobs_on_hold_list:
if check_event_has_not_unifinished_job_input(job.event, False, client=client, dvmconfig=dvm_config):
handle_nip90_job_event(job.event)
jobs_on_hold_list.remove(job)
if Timestamp.now().as_secs() > job.timestamp + 60 * 20: # remove jobs to look for after 20 minutes..
jobs_on_hold_list.remove(job)
time.sleep(1.0)
def check_event_has_not_unifinished_job_input(nevent, append, client, dvmconfig):
task_supported, task, duration = check_task_is_supported(nevent, client, False, config=dvmconfig)
if not task_supported:
return False
for tag in nevent.tags():
if tag.as_vec()[0] == 'i':
if len(tag.as_vec()) < 3:
print("Job Event missing/malformed i tag, skipping..")
return False
else:
input = tag.as_vec()[1]
input_type = tag.as_vec()[2]
if input_type == "job":
evt = get_referenced_event_by_id(input, EventDefinitions.ANY_RESULT, client, config=dvmconfig)
if evt is None:
if append:
job = RequiredJobToWatch(event=nevent, timestamp=Timestamp.now().as_secs())
jobs_on_hold_list.append(job)
send_job_status_reaction(nevent, "chain-scheduled", True, 0, client=client,
config=dvmconfig)
return False
else:
return True
def send_job_status_reaction(original_event, status, is_paid=True, amount=0, client=None, content=None, config=None,
key=None):
dvmconfig = config
altdesc = "This is a reaction to a NIP90 DVM AI task. "
task = get_task(original_event, client=client, dvmconfig=dvmconfig)
if status == "processing":
altdesc = "NIP90 DVM AI task " + task + " started processing. "
reaction = altdesc + emoji.emojize(":thumbs_up:")
elif status == "success":
altdesc = "NIP90 DVM AI task " + task + " finished successfully. "
reaction = altdesc + emoji.emojize(":call_me_hand:")
elif status == "chain-scheduled":
altdesc = "NIP90 DVM AI task " + task + " Chain Task scheduled"
reaction = altdesc + emoji.emojize(":thumbs_up:")
elif status == "error":
altdesc = "NIP90 DVM AI task " + task + " had an error. "
if content is None:
reaction = altdesc + emoji.emojize(":thumbs_down:")
else:
reaction = altdesc + emoji.emojize(":thumbs_down:") + content
elif status == "payment-required":
altdesc = "NIP90 DVM AI task " + task + " requires payment of min " + str(amount) + " Sats. "
reaction = altdesc + emoji.emojize(":orange_heart:")
elif status == "payment-rejected":
altdesc = "NIP90 DVM AI task " + task + " payment is below required amount of " + str(amount) + " Sats. "
reaction = altdesc + emoji.emojize(":thumbs_down:")
elif status == "user-blocked-from-service":
altdesc = "NIP90 DVM AI task " + task + " can't be performed. User has been blocked from Service. "
reaction = altdesc + emoji.emojize(":thumbs_down:")
else:
reaction = emoji.emojize(":thumbs_down:")
etag = Tag.parse(["e", original_event.id().to_hex()])
ptag = Tag.parse(["p", original_event.pubkey().to_hex()])
alttag = Tag.parse(["alt", altdesc])
statustag = Tag.parse(["status", status])
tags = [etag, ptag, alttag, statustag]
if status == "success" or status == "error": #
for x in job_list:
if x.event_id == original_event.id():
is_paid = x.is_paid
amount = x.amount
break
bolt11 = ""
payment_hash = ""
expires = original_event.created_at().as_secs() + (60 * 60 * 24)
if status == "payment-required" or (status == "processing" and not is_paid):
if dvmconfig.LNBITS_INVOICE_KEY != "":
try:
bolt11, payment_hash = create_bolt11_ln_bits(amount, dvmconfig)
except Exception as e:
print(e)
if not any(x.event_id == original_event.id().to_hex() for x in job_list):
job_list.append(
JobToWatch(event_id=original_event.id().to_hex(), timestamp=original_event.created_at().as_secs(),
amount=amount,
is_paid=is_paid,
status=status, result="", is_processed=False, bolt11=bolt11, payment_hash=payment_hash,
expires=expires, from_bot=False))
print(str(job_list))
if status == "payment-required" or status == "payment-rejected" or (status == "processing" and not is_paid) or (
status == "success" and not is_paid):
if dvmconfig.LNBITS_INVOICE_KEY != "":
amount_tag = Tag.parse(["amount", str(amount * 1000), bolt11])
else:
amount_tag = Tag.parse(["amount", str(amount * 1000)]) # to millisats
tags.append(amount_tag)
if key is not None:
keys = Keys.from_sk_str(key)
else:
keys = Keys.from_sk_str(dvmconfig.PRIVATE_KEY)
event = EventBuilder(EventDefinitions.KIND_FEEDBACK, reaction, tags).to_event(keys)
send_event(event, key=keys)
print("[Nostr] Sent Kind " + str(EventDefinitions.KIND_FEEDBACK) + " Reaction: " + status + " " + event.as_json())
return event.as_json()
def send_nostr_reply_event(content, original_event_as_str, key=None):
originalevent = Event.from_json(original_event_as_str)
requesttag = Tag.parse(["request", original_event_as_str.replace("\\", "")])
etag = Tag.parse(["e", originalevent.id().to_hex()])
ptag = Tag.parse(["p", originalevent.pubkey().to_hex()])
alttag = Tag.parse(["alt", "This is the result of a NIP90 DVM AI task with kind " + str(
originalevent.kind()) + ". The task was: " + originalevent.content()])
statustag = Tag.parse(["status", "success"])
replytags = [requesttag, etag, ptag, alttag, statustag]
for tag in originalevent.tags():
if tag.as_vec()[0] == "i":
icontent = tag.as_vec()[1]
ikind = tag.as_vec()[2]
itag = Tag.parse(["i", icontent, ikind])
replytags.append(itag)
if key is None:
key = Keys.from_sk_str(dvm_config.PRIVATE_KEY)
response_kind = originalevent.kind() + 1000
event = EventBuilder(response_kind, str(content), replytags).to_event(key)
send_event(event, key=key)
print("[Nostr] " + str(response_kind) + " Job Response event sent: " + event.as_json())
return event.as_json()
def respond_to_error(content, originaleventstr, is_from_bot=False, dvm_key=None):
print("ERROR")
if dvm_key is None:
keys = Keys.from_sk_str(dvm_config.PRIVATE_KEY)
else:
keys = Keys.from_sk_str(dvm_key)
original_event = Event.from_json(originaleventstr)
sender = ""
task = ""
if not is_from_bot:
send_job_status_reaction(original_event, "error", content=content, key=dvm_key)
# TODO Send Zap back
else:
for tag in original_event.tags():
if tag.as_vec()[0] == "p":
sender = tag.as_vec()[1]
elif tag.as_vec()[0] == "i":
task = tag.as_vec()[1]
user = get_from_sql_table(sender)
is_whitelisted = user[2]
if not is_whitelisted:
amount = int(user[1]) + get_amount_per_task(task)
update_sql_table(sender, amount, user[2], user[3], user[4], user[5], user[6],
Timestamp.now().as_secs())
message = "There was the following error : " + content + ". Credits have been reimbursed"
else:
# User didn't pay, so no reimbursement
message = "There was the following error : " + content
evt = EventBuilder.new_encrypted_direct_msg(keys, PublicKey.from_hex(sender), message, None).to_event(keys)
send_event(evt, key=keys)
def check_and_return_event(data, original_event_str: str, dvm_key=""):
original_event = Event.from_json(original_event_str)
keys = Keys.from_sk_str(dvm_key)
for x in job_list:
if x.event_id == original_event.id().to_hex():
is_paid = x.is_paid
amount = x.amount
x.result = data
x.is_processed = True
if dvm_config.SHOWRESULTBEFOREPAYMENT and not is_paid:
send_nostr_reply_event(data, original_event_str, key=keys)
send_job_status_reaction(original_event, "success", amount,
config=dvm_config) # or payment-required, or both?
elif not dvm_config.SHOWRESULTBEFOREPAYMENT and not is_paid:
send_job_status_reaction(original_event, "success", amount,
config=dvm_config) # or payment-required, or both?
if dvm_config.SHOWRESULTBEFOREPAYMENT and is_paid:
job_list.remove(x)
elif not dvm_config.SHOWRESULTBEFOREPAYMENT and is_paid:
job_list.remove(x)
send_nostr_reply_event(data, original_event_str, key=keys)
break
post_processed_content = post_process_result(data, original_event)
send_nostr_reply_event(post_processed_content, original_event_str, key=keys)

48
main.py Normal file
View File

@@ -0,0 +1,48 @@
import os
from pathlib import Path
from threading import Thread
import dotenv
import utils.env as env
from utils.definitions import EventDefinitions
def run_nostr_dvm_with_local_config():
from dvm import dvm, DVMConfig
from utils.nip89_utils import NIP89Announcement
dvmconfig = DVMConfig()
dvmconfig.PRIVATE_KEY = os.getenv(env.NOSTR_PRIVATE_KEY)
dvmconfig.SUPPORTED_TASKS = ["translation"]
dvmconfig.LNBITS_INVOICE_KEY = os.getenv(env.LNBITS_INVOICE_KEY)
dvmconfig.LNBITS_URL = os.getenv(env.LNBITS_HOST)
nip89translation = NIP89Announcement()
nip89translation.kind = EventDefinitions.KIND_NIP90_TRANSLATE_TEXT
nip89translation.dtag = os.getenv(env.TASK_TRANSLATION_NIP89_DTAG)
nip89translation.pk = os.getenv(env.NOSTR_PRIVATE_KEY)
nip89translation.content = "{\"name\":\"NostrAI DVM Translator\",\"image\":\"https://cdn.nostr.build/i/feb98d8700abe7d6c67d9106a72a20354bf50805af79869638f5a32d24a5ac2a.jpg\",\"about\":\"Translates Text from given text/event/job, currently using Google Translation Services into language defined in param. \",\"nip90Params\":{\"language\":{\"required\":true,\"values\":[\"af\",\"am\",\"ar\",\"az\",\"be\",\"bg\",\"bn\",\"bs\",\"ca\",\"ceb\",\"co\",\"cs\",\"cy\",\"da\",\"de\",\"el\",\"eo\",\"es\",\"et\",\"eu\",\"fa\",\"fi\",\"fr\",\"fy\",\"ga\",\"gd\",\"gl\",\"gu\",\"ha\",\"haw\",\"hi\",\"hmn\",\"hr\",\"ht\",\"hu\",\"hy\",\"id\",\"ig\",\"is\",\"it\",\"he\",\"ja\",\"jv\",\"ka\",\"kk\",\"km\",\"kn\",\"ko\",\"ku\",\"ky\",\"la\",\"lb\",\"lo\",\"lt\",\"lv\",\"mg\",\"mi\",\"mk\",\"ml\",\"mn\",\"mr\",\"ms\",\"mt\",\"my\",\"ne\",\"nl\",\"no\",\"ny\",\"or\",\"pa\",\"pl\",\"ps\",\"pt\",\"ro\",\"ru\",\"sd\",\"si\",\"sk\",\"sl\",\"sm\",\"sn\",\"so\",\"sq\",\"sr\",\"st\",\"su\",\"sv\",\"sw\",\"ta\",\"te\",\"tg\",\"th\",\"tl\",\"tr\",\"ug\",\"uk\",\"ur\",\"uz\",\"vi\",\"xh\",\"yi\",\"yo\",\"zh\",\"zu\"]}}}"
dvmconfig.NIP89s.append(nip89translation)
nostr_dvm_thread = Thread(target=dvm, args=[dvmconfig])
nostr_dvm_thread.start()
if __name__ == '__main__':
env_path = Path('.env')
if env_path.is_file():
print(f'loading environment from {env_path.resolve()}')
dotenv.load_dotenv(env_path, verbose=True, override=True)
else:
raise FileNotFoundError(f'.env file not found at {env_path} ')
run_nostr_dvm_with_local_config()

0
outputs/.gitkeep Normal file
View File

32
requirements.txt Normal file
View File

@@ -0,0 +1,32 @@
beautifulsoup4==4.12.2
bech32==1.2.0
blessed==1.20.0
certifi==2023.7.22
charset-normalizer==3.3.2
emoji==2.8.0
ffmpegio==0.8.5
ffmpegio-core==0.8.5
idna==3.4
inquirer==3.1.3
install==1.3.5
nostr-sdk==0.0.4
numpy==1.26.2
packaging==23.2
pandas==2.1.3
Pillow==10.1.0
pluggy==1.3.0
pycryptodome==3.19.0
python-dateutil==2.8.2
python-dotenv==1.0.0
python-editor==1.0.4
pytz==2023.3.post1
pyuseragents==1.0.5
readchar==4.0.5
requests==2.31.0
safeIO==1.2
six==1.16.0
soupsieve==2.5
translatepy==2.3
tzdata==2023.3
urllib3==2.1.0
wcwidth==0.2.10

100
test_client.py Normal file
View File

@@ -0,0 +1,100 @@
import os
import time
import datetime as datetime
from pathlib import Path
from threading import Thread
import dotenv
from nostr_sdk import Keys, Client, Tag, EventBuilder, Filter, HandleNotification, Timestamp, nip04_decrypt
from utils.nostr_utils import send_event
from utils.definitions import EventDefinitions, RELAY_LIST
import utils.env as env
#TODO HINT: Only use this path with a preiously whitelisted privkey, as zapping events is not implemented in the lib/code
def nostr_client_test_translation(input, kind, lang, sats, satsmax):
keys = Keys.from_sk_str(os.getenv(env.NOSTR_TEST_CLIENT_PRIVATE_KEY))
if kind == "text":
iTag = Tag.parse(["i", input, "text"])
elif kind == "event":
iTag = Tag.parse(["i", input, "event"])
paramTag1 = Tag.parse(["param", "language", lang])
bidTag = Tag.parse(['bid', str(sats * 1000), str(satsmax * 1000)])
relaysTag = Tag.parse(['relays', "wss://relay.damus.io", "wss://blastr.f7z.xyz", "wss://relayable.org", "wss://nostr-pub.wellorder.net"])
alttag = Tag.parse(["alt", "This is a NIP90 DVM AI task to translate a given Input"])
event = EventBuilder(EventDefinitions.KIND_NIP90_TRANSLATE_TEXT, str("Translate the given input."), [iTag, paramTag1, bidTag, relaysTag, alttag]).to_event(keys)
relay_list = ["wss://relay.damus.io", "wss://blastr.f7z.xyz", "wss://relayable.org",
"wss://nostr-pub.wellorder.net"]
client = Client(keys)
for relay in relay_list:
client.add_relay(relay)
client.connect()
send_event(event, client, keys)
return event.as_json()
def nostr_client():
keys = Keys.from_sk_str(os.getenv(env.NOSTR_TEST_CLIENT_PRIVATE_KEY))
sk = keys.secret_key()
pk = keys.public_key()
print(f"Nostr Client public key: {pk.to_bech32()}, Hex: {pk.to_hex()} ")
client = Client(keys)
for relay in RELAY_LIST:
client.add_relay(relay)
client.connect()
dm_zap_filter = Filter().pubkey(pk).kinds([EventDefinitions.KIND_DM,
EventDefinitions.KIND_ZAP]).since(Timestamp.now()) # events to us specific
dvm_filter = (Filter().kinds([EventDefinitions.KIND_NIP90_RESULT_TRANSLATE_TEXT,
EventDefinitions.KIND_FEEDBACK]).since(Timestamp.now())) # public events
client.subscribe([dm_zap_filter, dvm_filter])
#nostr_client_test_translation("This is the result of the DVM in spanish", "text", "es", 20, 20)
nostr_client_test_translation("44a0a8b395ade39d46b9d20038b3f0c8a11168e67c442e3ece95e4a1703e2beb", "event", "fr", 20, 20)
#nostr_client_test_image(sats=50, satsmax=10)
class NotificationHandler(HandleNotification):
def handle(self, relay_url, event):
print(f"Received new event from {relay_url}: {event.as_json()}")
if event.kind() == 7000:
print("[Nostr Client]: " + event.as_json())
elif event.kind() > 6000 and event.kind() < 6999:
print("[Nostr Client]: " + event.as_json())
print("[Nostr Client]: " + event.content())
elif event.kind() == 4:
dec_text = nip04_decrypt(sk, event.pubkey(), event.content())
print("[Nostr Client]: " + f"Received new msg: {dec_text}")
elif event.kind() == 9735:
print("[Nostr Client]: " + f"Received new zap:")
print(event.as_json())
def handle_msg(self, relay_url, msg):
None
client.handle_notifications(NotificationHandler())
while True:
time.sleep(5.0)
if __name__ == '__main__':
env_path = Path('.env')
if env_path.is_file():
print(f'loading environment from {env_path.resolve()}')
dotenv.load_dotenv(env_path, verbose=True, override=True)
else:
raise FileNotFoundError(f'.env file not found at {env_path} ')
nostr_dvm_thread = Thread(target=nostr_client())
nostr_dvm_thread.start()

67
utils/admin_utils.py Normal file
View File

@@ -0,0 +1,67 @@
# ADMINISTRARIVE DB MANAGEMENT
import time
from nostr_sdk import Keys, EventBuilder, PublicKey
from utils.database_utils import get_from_sql_table, list_db, clear_db, delete_from_sql_table, update_sql_table, \
get_or_add_user, update_user_metadata
from utils.nip89_utils import nip89_announce_tasks
from utils.nostr_utils import send_event
def admin_make_database_updates(config=None, client=None):
# This is called on start of Server, Admin function to manually whitelist/blacklist/add balance/delete users
dvmconfig = config
rebroadcast_nip89 = False
cleardb = False
listdatabase = False
deleteuser = False
whitelistuser = False
unwhitelistuser = False
blacklistuser = False
addbalance = False
additional_balance = 50
# publickey = PublicKey.from_bech32("npub1...").to_hex()
# use this if you have the npub
publickey = "asd123"
#use this if you have hex
if whitelistuser:
user = get_or_add_user(publickey)
update_sql_table(user[0], user[1], True, False, user[4], user[5], user[6], user[7])
user = get_from_sql_table(publickey)
print(str(user[6]) + " is whitelisted: " + str(user[2]))
if unwhitelistuser:
user = get_from_sql_table(publickey)
update_sql_table(user[0], user[1], False, False, user[4], user[5], user[6], user[7])
if blacklistuser:
user = get_from_sql_table(publickey)
update_sql_table(user[0], user[1], False, True, user[4], user[5], user[6], user[7])
if addbalance:
user = get_from_sql_table(publickey)
update_sql_table(user[0], (int(user[1]) + additional_balance), user[2], user[3], user[4], user[5], user[6],
user[7])
time.sleep(1.0)
message = str(additional_balance) + " Sats have been added to your balance. Your new balance is " + str(
(int(user[1]) + additional_balance)) + " Sats."
keys = Keys.from_sk_str(config.PRIVATE_KEY)
evt = EventBuilder.new_encrypted_direct_msg(keys, PublicKey.from_hex(publickey), message,
None).to_event(keys)
send_event(evt, key=keys)
if deleteuser:
delete_from_sql_table(publickey)
if cleardb:
clear_db()
if listdatabase:
list_db()
if rebroadcast_nip89:
nip89_announce_tasks(dvmconfig)

38
utils/ai_utils.py Normal file
View File

@@ -0,0 +1,38 @@
#We can add multiple Tasks here and call them in the do_work function.
#Also make sure to define your task in get supported tasks, and in get amount per task and listen to
#the according event type in the beginning of dvm.py and
def GoogleTranslate(text, translation_lang):
from translatepy.translators.google import GoogleTranslate
gtranslate = GoogleTranslate()
length = len(text)
step = 0
translated_text = ""
if length > 4999:
while step+5000 < length:
textpart = text[step:step+5000]
step = step + 5000
try:
translated_text_part = str(gtranslate.translate(textpart, translation_lang))
print("Translated Text part:\n\n " + translated_text_part)
except:
translated_text_part = "An error occured"
translated_text = translated_text + translated_text_part
if step < length:
textpart = text[step:length]
try:
translated_text_part = str(gtranslate.translate(textpart, translation_lang))
print("Translated Text part:\n\n " + translated_text_part)
except:
translated_text_part = "An error occured"
translated_text = translated_text + translated_text_part
return translated_text

109
utils/backend_utils.py Normal file
View File

@@ -0,0 +1,109 @@
import requests
from utils.definitions import EventDefinitions
from utils.nostr_utils import get_event_by_id
def get_task(event, client, dvmconfig):
if event.kind() == EventDefinitions.KIND_NIP90_GENERIC: # use this for events that have no id yet
for tag in event.tags():
if tag.as_vec()[0] == 'j':
return tag.as_vec()[1]
else:
return "unknown job: " + event.as_json()
elif event.kind() == EventDefinitions.KIND_DM: # dm
for tag in event.tags():
if tag.as_vec()[0] == 'j':
return tag.as_vec()[1]
else:
return "unknown job: " + event.as_json()
elif event.kind() == EventDefinitions.KIND_NIP90_TRANSLATE_TEXT:
return "translation"
else:
return "unknown type"
def check_task_is_supported(event, client, get_duration = False, config=None):
dvmconfig = config
input_value = ""
input_type = ""
duration = 1
output_is_set = True
for tag in event.tags():
if tag.as_vec()[0] == 'i':
if len(tag.as_vec()) < 3:
print("Job Event missing/malformed i tag, skipping..")
return False, "", 0
else:
input_value = tag.as_vec()[1]
input_type = tag.as_vec()[2]
if input_type == "event":
evt = get_event_by_id(input_value, config=dvmconfig)
if evt == None:
print("Event not found")
return False, "", 0
elif tag.as_vec()[0] == 'output':
output = tag.as_vec()[1]
output_is_set = True
if not (output == "text/plain" or output == "text/json" or output == "json" or output == "image/png" or "image/jpg" or output == ""):
print("Output format not supported, skipping..")
return False, "", 0
task = get_task(event, client=client, dvmconfig=dvmconfig)
if not output_is_set:
print("No output set")
if task not in dvmconfig.SUPPORTED_TASKS: # The Tasks this DVM supports (can be extended)
return False, task, duration
elif task == "translation" and (
input_type != "event" and input_type != "job" and input_type != "text"): # The input types per task
return False, task, duration
if task == "translation" and input_type != "text" and len(event.content()) > 4999: # Google Services have a limit of 5000 signs
return False, task, duration
if input_type == 'url' and check_url_is_readable(input_value) is None:
print("url not readable")
return False, task, duration
return True, task, duration
def check_url_is_readable(url):
if not str(url).startswith("http"):
return None
# If it's a YouTube oder Overcast link, we suppose we support it
if (str(url).replace("http://", "").replace("https://", "").replace("www.", "").replace("youtu.be/",
"youtube.com?v=")[
0:11] == "youtube.com" and str(url).find("live") == -1) or str(url).startswith('https://x.com') or str(url).startswith('https://twitter.com') :
return "video"
elif str(url).startswith("https://overcast.fm/"):
return "audio"
# If link is comaptible with one of these file formats, it's fine.
req = requests.get(url)
content_type = req.headers['content-type']
if content_type == 'audio/x-wav' or str(url).endswith(".wav") or content_type == 'audio/mpeg' or str(url).endswith(
".mp3") or content_type == 'audio/ogg' or str(url).endswith(".ogg"):
return "audio"
elif content_type == 'image/png' or str(url).endswith(".png") or content_type == 'image/jpg' or str(url).endswith(
".jpg") or content_type == 'image/jpeg' or str(url).endswith(".jpeg") or str(url).endswith(".pdf") or content_type == 'image/png' or str(
url).endswith(".png"):
return "image"
elif content_type == 'video/mp4' or str(url).endswith(".mp4") or content_type == 'video/avi' or str(url).endswith(
".avi") or content_type == 'video/mov' or str(url).endswith(".mov"):
return "video"
# Otherwise we will not offer to do the job.
return None
def get_amount_per_task(task, duration = 0, config=None):
dvmconfig = config
if task == "translation":
amount = dvmconfig.COSTPERUNIT_TRANSLATION
else:
print("[Nostr] Task " + task + " is currently not supported by this instance, skipping")
return None
return amount

202
utils/database_utils.py Normal file
View File

@@ -0,0 +1,202 @@
# DATABASE LOGIC
import os
import sqlite3
import time
from _sqlite3 import Error
from datetime import timedelta
from logging import Filter
from nostr_sdk import Timestamp, Keys, PublicKey, EventBuilder, Metadata, Filter
from utils import env
from utils.definitions import NEW_USER_BALANCE
from utils.nostr_utils import send_event
def create_sql_table():
try:
con = sqlite3.connect(os.getenv(env.USER_DB_PATH))
cur = con.cursor()
cur.execute(""" CREATE TABLE IF NOT EXISTS users (
npub text PRIMARY KEY,
sats integer NOT NULL,
iswhitelisted boolean,
isblacklisted boolean,
nip05 text,
lud16 text,
name text,
lastactive integer
); """)
cur.execute("SELECT name FROM sqlite_master")
con.close()
except Error as e:
print(e)
def add_sql_table_column():
try:
con = sqlite3.connect(os.getenv(env.USER_DB_PATH))
cur = con.cursor()
cur.execute(""" ALTER TABLE users ADD COLUMN lastactive 'integer' """)
con.close()
except Error as e:
print(e)
def add_to_sql_table(npub, sats, iswhitelisted, isblacklisted, nip05, lud16, name, lastactive):
try:
con = sqlite3.connect(os.getenv(env.USER_DB_PATH))
cur = con.cursor()
data = (npub, sats, iswhitelisted, isblacklisted, nip05, lud16, name, lastactive)
cur.execute("INSERT or IGNORE INTO users VALUES(?, ?, ?, ?, ?, ?, ?, ?)", data)
con.commit()
con.close()
except Error as e:
print(e)
def update_sql_table(npub, sats, iswhitelisted, isblacklisted, nip05, lud16, name, lastactive):
try:
con = sqlite3.connect(os.getenv(env.USER_DB_PATH))
cur = con.cursor()
data = (sats, iswhitelisted, isblacklisted, nip05, lud16, name, lastactive, npub)
cur.execute(""" UPDATE users
SET sats = ? ,
iswhitelisted = ? ,
isblacklisted = ? ,
nip05 = ? ,
lud16 = ? ,
name = ? ,
lastactive = ?
WHERE npub = ?""", data)
con.commit()
con.close()
except Error as e:
print(e)
def get_from_sql_table(npub):
try:
con = sqlite3.connect(os.getenv(env.USER_DB_PATH))
cur = con.cursor()
cur.execute("SELECT * FROM users WHERE npub=?", (npub,))
row = cur.fetchone()
con.close()
return row
except Error as e:
print(e)
def delete_from_sql_table(npub):
try:
con = sqlite3.connect(os.getenv(env.USER_DB_PATH))
cur = con.cursor()
cur.execute("DELETE FROM users WHERE npub=?", (npub,))
con.commit()
con.close()
except Error as e:
print(e)
def clear_db():
try:
con = sqlite3.connect(os.getenv(env.USER_DB_PATH))
cur = con.cursor()
cur.execute("SELECT * FROM users WHERE npub IS NULL OR npub = '' ")
rows = cur.fetchall()
for row in rows:
print(row)
delete_from_sql_table(row[0])
con.close()
return rows
except Error as e:
print(e)
def list_db():
try:
con = sqlite3.connect(os.getenv(env.USER_DB_PATH))
cur = con.cursor()
cur.execute("SELECT * FROM users ORDER BY sats DESC")
rows = cur.fetchall()
for row in rows:
print(row)
con.close()
except Error as e:
print(e)
def update_user_balance(sender, sats, config=None):
user = get_from_sql_table(sender)
if user is None:
add_to_sql_table(sender, (int(sats) + NEW_USER_BALANCE), False, False,
"", "", "", Timestamp.now().as_secs())
print("NEW USER: " + sender + " Zap amount: " + str(sats) + " Sats.")
else:
user = get_from_sql_table(sender)
print(str(sats))
nip05 =user[4]
lud16 = user[5]
name = user[6]
if nip05 is None:
nip05 = ""
if lud16 is None:
lud16 = ""
if name is None:
name = ""
new_balance = int(user[1]) + int(sats)
update_sql_table(sender, new_balance, user[2], user[3], nip05, lud16, name,
Timestamp.now().as_secs())
print("UPDATE USER BALANCE: " + str(name) + " Zap amount: " + str(sats) + " Sats.")
if config is not None:
keys = Keys.from_sk_str(config.PRIVATE_KEY)
time.sleep(1.0)
message = ("Added "+ str(sats) + " Sats to balance. New balance is " + str(new_balance) + " Sats. " )
evt = EventBuilder.new_encrypted_direct_msg(keys, PublicKey.from_hex(sender), message,
None).to_event(keys)
send_event(evt, key=keys)
def get_or_add_user(sender):
user = get_from_sql_table(sender)
if user is None:
add_to_sql_table(sender, NEW_USER_BALANCE, False, False, None,
None, None, Timestamp.now().as_secs())
user = get_from_sql_table(sender)
print(user)
return user
def update_user_metadata(sender, client):
user = get_from_sql_table(sender)
name = user[6]
lud16 = user[5]
nip05 = user[4]
try:
profile_filter = Filter().kind(0).author(sender).limit(1)
events = client.get_events_of([profile_filter], timedelta(seconds=3))
if len(events) > 0:
ev = events[0]
metadata = Metadata.from_json(ev.content())
name = metadata.get_display_name()
if str(name) == "" or name is None:
name = metadata.get_name()
nip05 = metadata.get_nip05()
lud16 = metadata.get_lud16()
except:
print("Couldn't get meta information")
update_sql_table(user[0], user[1], user[2], user[3], nip05, lud16,
name, Timestamp.now().as_secs())
user = get_from_sql_table(user[0])
return user

81
utils/definitions.py Normal file
View File

@@ -0,0 +1,81 @@
from dataclasses import dataclass
from nostr_sdk import Event
NEW_USER_BALANCE = 250
LOCAL_TASKS = ["conversion", "summarization","note-recommendation", "inactive-following", "image-upscale", "translation"]
# Tasks performed by the DVM and not send to nova-server (can change later)
RELAY_LIST = ["wss://relay.damus.io", "wss://nostr-pub.wellorder.net", "wss://nos.lol", "wss://nostr.wine",
"wss://relay.nostfiles.dev", "wss://nostr.mom", "wss://nostr.oxtr.dev", "wss://relay.nostr.bg", "wss://relay.f7z.io"]
class EventDefinitions:
KIND_DM: int = 4
KIND_ZAP: int = 9735
KIND_NIP94_METADATA: int = 1063
KIND_FEEDBACK: int = 7000
KIND_NIP90_EXTRACT_TEXT = 5000
KIND_NIP90_RESULT_EXTRACT_TEXT = 6000
KIND_NIP90_SUMMARIZE_TEXT = 5001
KIND_NIP90_RESULT_SUMMARIZE_TEXT = 6001
KIND_NIP90_TRANSLATE_TEXT = 5002
KIND_NIP90_RESULT_TRANSLATE_TEXT = 6002
KIND_NIP90_GENERATE_IMAGE = 5100
KIND_NIP90_RESULT_GENERATE_IMAGE = 6100
KIND_NIP90_RECOMMEND_NOTES = 65006
KIND_NIP90_RESULT_RECOMMEND_NOTES = 65001
KIND_NIP90_RECOMMEND_USERS = 65007
KIND_NIP90_RESULT_RECOMMEND_USERS = 65001
KIND_NIP90_CONVERT_VIDEO = 5200
KIND_NIP90_RESULT_CONVERT_VIDEO = 6200
KIND_NIP90_GENERIC = 5999
KIND_NIP90_RESULT_GENERIC = 6999
ANY_RESULT = [KIND_NIP90_RESULT_EXTRACT_TEXT,
KIND_NIP90_RESULT_SUMMARIZE_TEXT,
KIND_NIP90_RESULT_TRANSLATE_TEXT,
KIND_NIP90_RESULT_GENERATE_IMAGE,
KIND_NIP90_RESULT_RECOMMEND_NOTES,
KIND_NIP90_RESULT_RECOMMEND_USERS,
KIND_NIP90_RESULT_CONVERT_VIDEO,
KIND_NIP90_RESULT_GENERIC]
class DVMConfig:
SUPPORTED_TASKS = []
PRIVATE_KEY: str
RELAY_LIST = ["wss://relay.damus.io", "wss://nostr-pub.wellorder.net", "wss://nos.lol", "wss://nostr.wine",
"wss://relay.nostfiles.dev", "wss://nostr.mom", "wss://nostr.oxtr.dev", "wss://relay.nostr.bg", "wss://relay.f7z.io"]
RELAY_TIMEOUT = 5
LNBITS_INVOICE_KEY = ''
LNBITS_URL = 'https://lnbits.com'
REQUIRES_NIP05: bool = False
SHOWRESULTBEFOREPAYMENT: bool = True # if this is true show results even when not paid right after autoprocess
NEW_USER_BALANCE: int = 250 # Free credits for new users
COSTPERUNIT_TRANSLATION: int = 20 # Still need to multiply this by duration
NIP89s: list = []
@dataclass
class JobToWatch:
event_id: str
timestamp: int
is_paid: bool
amount: int
status: str
result: str
is_processed: bool
bolt11: str
payment_hash: str
expires: int
from_bot: bool
@dataclass
class RequiredJobToWatch:
event: Event
timestamp: int

11
utils/env.py Normal file
View File

@@ -0,0 +1,11 @@
NOSTR_PRIVATE_KEY = "NOSTR_PRIVATE_KEY"
NOSTR_TEST_CLIENT_PRIVATE_KEY = "NOSTR_TEST_CLIENT_PRIVATE_KEY"
USER_DB_PATH = "USER_DB_PATH"
LNBITS_INVOICE_KEY = "LNBITS_INVOICE_KEY"
LNBITS_HOST = "LNBITS_HOST"
TASK_TRANSLATION_NIP89_DTAG = "TASK_TRANSLATION_NIP89_DTAG"

19
utils/nip89_utils.py Normal file
View File

@@ -0,0 +1,19 @@
from nostr_sdk import Tag, Keys, EventBuilder
from utils.nostr_utils import send_event
class NIP89Announcement:
kind: int
dtag: str
pk: str
content: str
def nip89_announce_tasks(dvmconfig):
for nip89 in dvmconfig.NIP89s:
k_tag = Tag.parse(["k", str(nip89.kind)])
d_tag = Tag.parse(["d", nip89.dtag])
keys = Keys.from_sk_str(nip89.pk)
content = nip89.content
event = EventBuilder(31990, content, [k_tag, d_tag]).to_event(keys)
send_event(event, key=keys)
print("Announced NIP 89")

92
utils/nostr_utils.py Normal file
View File

@@ -0,0 +1,92 @@
from datetime import timedelta
from nostr_sdk import Keys, Filter, Client, Alphabet, EventId, Options
from utils.definitions import RELAY_LIST
def get_event_by_id(event_id, client=None, config=None):
is_new_client = False
if client is None:
keys = Keys.from_sk_str(config.PRIVATE_KEY)
client = Client(keys)
for relay in config.RELAY_LIST:
client.add_relay(relay)
client.connect()
is_new_client = True
split = event_id.split(":")
if len(split) == 3:
id_filter = Filter().author(split[1]).custom_tag(Alphabet.D, [split[2]])
events = client.get_events_of([id_filter], timedelta(seconds=config.RELAY_TIMEOUT))
else:
id_filter = Filter().id(event_id).limit(1)
events = client.get_events_of([id_filter], timedelta(seconds=config.RELAY_TIMEOUT))
if is_new_client:
client.disconnect()
if len(events) > 0:
return events[0]
else:
return None
def get_referenced_event_by_id(event_id, kinds=None, client=None, config=None):
if kinds is None:
kinds = []
is_new_client = False
if client is None:
keys = Keys.from_sk_str(config.PRIVATE_KEY)
client = Client(keys)
for relay in config.RELAY_LIST:
client.add_relay(relay)
client.connect()
is_new_client = True
if kinds is None:
kinds = []
if len(kinds) > 0:
job_id_filter = Filter().kinds(kinds).event(EventId.from_hex(event_id)).limit(1)
else:
job_id_filter = Filter().event(EventId.from_hex(event_id)).limit(1)
events = client.get_events_of([job_id_filter], timedelta(seconds=config.RELAY_TIMEOUT))
if is_new_client:
client.disconnect()
if len(events) > 0:
return events[0]
else:
return None
def send_event(event, client=None, key=None, config=None):
relays = []
is_new_client = False
for tag in event.tags():
if tag.as_vec()[0] == 'relays':
relays = tag.as_vec()[1].split(',')
if client is None:
print(key.secret_key().to_hex())
opts = Options().wait_for_send(False).send_timeout(timedelta(seconds=5)).skip_disconnected_relays(True)
client = Client.with_opts(key, opts)
for relay in RELAY_LIST:
client.add_relay(relay)
client.connect()
is_new_client = True
for relay in relays:
if relay not in RELAY_LIST:
client.add_relay(relay)
client.connect()
event_id = client.send_event(event)
for relay in relays:
if relay not in RELAY_LIST:
client.remove_relay(relay)
if is_new_client:
client.disconnect()
return event_id

90
utils/output_utils.py Normal file
View File

@@ -0,0 +1,90 @@
import json
import datetime as datetime
from types import NoneType
import pandas
def post_process_result(anno, original_event):
print("post-processing...")
if isinstance(anno, pandas.DataFrame): # if input is an anno we parse it to required output format
for tag in original_event.tags():
print(tag.as_vec()[0])
if tag.as_vec()[0] == "output":
print("HAS OUTPUT TAG")
output_format = tag.as_vec()[1]
print("requested output is " + str(tag.as_vec()[1]) + "...")
try:
if output_format == "text/plain":
result = ""
for each_row in anno['name']:
if each_row is not None:
for i in str(each_row).split('\n'):
result = result + i + "\n"
result = replace_broken_words(
str(result).replace("\"", "").replace('[', "").replace(']', "").lstrip(None))
return result
elif output_format == "text/vtt":
print(str(anno))
result = "WEBVTT\n\n"
for element in anno:
name = element["name"] # name
start = float(element["from"])
convertstart = str(datetime.timedelta(seconds=start))
end = float(element["to"])
convertend = str(datetime.timedelta(seconds=end))
print(str(convertstart) + " --> " + str(convertend))
cleared_name = str(name).lstrip("\'").rstrip("\'")
result = result + str(convertstart) + " --> " + str(
convertend) + "\n" + cleared_name + "\n\n"
result = replace_broken_words(
str(result).replace("\"", "").replace('[', "").replace(']', "").lstrip(None))
return result
elif output_format == "text/json" or output_format == "json":
# result = json.dumps(json.loads(anno.data.to_json(orient="records")))
result = replace_broken_words(json.dumps(anno.data.tolist()))
return result
# TODO add more
else:
result = ""
for element in anno.data:
element["name"] = str(element["name"]).lstrip()
element["from"] = (format(float(element["from"]), '.2f')).lstrip() # name
element["to"] = (format(float(element["to"]), '.2f')).lstrip() # name
result = result + "(" + str(element["from"]) + "," + str(element["to"]) + ")" + " " + str(
element["name"]) + "\n"
print(result)
result = replace_broken_words(result)
return result
except Exception as e:
print(e)
result = replace_broken_words(str(anno.data))
return result
else:
result = ""
for element in anno.data:
element["name"] = str(element["name"]).lstrip()
element["from"] = (format(float(element["from"]), '.2f')).lstrip() # name
element["to"] = (format(float(element["to"]), '.2f')).lstrip() # name
result = result + "(" + str(element["from"]) + "," + str(element["to"]) + ")" + " " + str(
element["name"]) + "\n"
print(result)
result = replace_broken_words(result)
return result
elif isinstance(anno, NoneType):
return "An error occurred"
else:
result = replace_broken_words(anno) #TODO
return result
def replace_broken_words(text):
result = (text.replace("Noster", "Nostr").replace("Nostra", "Nostr").replace("no stir", "Nostr").
replace("Nostro", "Nostr").replace("Impub", "npub").replace("sets", "Sats"))
return result

65
utils/requestform_utils.py Normal file
View File

@@ -0,0 +1,65 @@
import os
import re
from configparser import ConfigParser
import time
from nostr_sdk import PublicKey
from utils.definitions import EventDefinitions
from utils.backend_utils import get_task
from utils.nostr_utils import get_referenced_event_by_id, get_event_by_id
from utils.definitions import LOCAL_TASKS
import utils.env as env
def create_requestform_from_nostr_event(event, is_bot=False, client=None, dvmconfig=None):
task = get_task(event, client=client, dvmconfig=dvmconfig)
request_form = {"jobID": event.id().to_hex(), "frameSize": 0, "stride": 0,
"leftContext": 0, "rightContext": 0,
"startTime": "0", "endTime": "0"}
if task == "translation":
input_type = "event"
text = ""
translation_lang = "en"
for tag in event.tags():
if tag.as_vec()[0] == 'i':
input_type = tag.as_vec()[2]
elif tag.as_vec()[0] == 'param':
param = tag.as_vec()[1]
if param == "language": # check for paramtype
translation_lang = str(tag.as_vec()[2]).split('-')[0]
elif param == "lang": # check for paramtype
translation_lang = str(tag.as_vec()[2]).split('-')[0]
if input_type == "event":
for tag in event.tags():
if tag.as_vec()[0] == 'i':
evt = get_event_by_id(tag.as_vec()[1], config=dvmconfig)
text = evt.content()
break
elif input_type == "text":
for tag in event.tags():
if tag.as_vec()[0] == 'i':
text = tag.as_vec()[1]
break
elif input_type == "job":
for tag in event.tags():
if tag.as_vec()[0] == 'i':
evt = get_referenced_event_by_id(tag.as_vec()[1],
[EventDefinitions.KIND_NIP90_RESULT_EXTRACT_TEXT,
EventDefinitions.KIND_NIP90_RESULT_SUMMARIZE_TEXT],
client,
config=dvmconfig)
text = evt.content()
break
request_form["optStr"] = ('translation_lang=' + translation_lang + ';text=' +
text.replace('\U0001f919', "").replace("=", "equals").
replace(";", ","))
return request_form

96
utils/zap_utils.py Normal file
View File

@@ -0,0 +1,96 @@
# LIGHTNING FUNCTIONS
import json
import requests
from Crypto.Cipher import AES
from bech32 import bech32_decode, convertbits
from nostr_sdk import PublicKey, nostr_sdk
def parse_bolt11_invoice(invoice):
def get_index_of_first_letter(ip):
index = 0
for c in ip:
if c.isalpha():
return index
else:
index = index + 1
return len(ip)
remaining_invoice = invoice[4:]
index = get_index_of_first_letter(remaining_invoice)
identifier = remaining_invoice[index]
number_string = remaining_invoice[:index]
number = float(number_string)
if identifier == 'm':
number = number * 100000000 * 0.001
elif identifier == 'u':
number = number * 100000000 * 0.000001
elif identifier == 'n':
number = number * 100000000 * 0.000000001
elif identifier == 'p':
number = number * 100000000 * 0.000000000001
return int(number)
def create_bolt11_ln_bits(sats, config):
url = config.LNBITS_URL + "/api/v1/payments"
data = {'out': False, 'amount': sats, 'memo': "Nostr-DVM"}
headers = {'X-API-Key': config.LNBITS_INVOICE_KEY, 'Content-Type': 'application/json', 'charset': 'UTF-8'}
try:
res = requests.post(url, json=data, headers=headers)
obj = json.loads(res.text)
return obj["payment_request"], obj["payment_hash"]
except Exception as e:
print(e)
return None
def check_bolt11_ln_bits_is_paid(payment_hash, config):
url = config.LNBITS_URL + "/api/v1/payments/" + payment_hash
headers = {'X-API-Key': config.LNBITS_INVOICE_KEY, 'Content-Type': 'application/json', 'charset': 'UTF-8'}
try:
res = requests.get(url, headers=headers)
obj = json.loads(res.text)
return obj["paid"]
except Exception as e:
#print("Exception checking invoice is paid:" + e)
return None
# DECRYPT ZAPS
def check_for_zapplepay(sender, content):
try:
# Special case Zapplepay
if sender == PublicKey.from_bech32("npub1wxl6njlcgygduct7jkgzrvyvd9fylj4pqvll6p32h59wyetm5fxqjchcan").to_hex():
real_sender_bech32 = content.replace("From: nostr:", "")
sender = PublicKey.from_bech32(real_sender_bech32).to_hex()
return sender
except Exception as e:
print(e)
return sender
def decrypt_private_zap_message(msg, privkey, pubkey):
shared_secret = nostr_sdk.generate_shared_key(privkey, pubkey)
if len(shared_secret) != 16 and len(shared_secret) != 32:
return "invalid shared secret size"
parts = msg.split("_")
if len(parts) != 2:
return "invalid message format"
try:
_, encrypted_msg = bech32_decode(parts[0])
encrypted_bytes = convertbits(encrypted_msg, 5, 8, False)
_, iv = bech32_decode(parts[1])
iv_bytes = convertbits(iv, 5, 8, False)
except Exception as e:
return e
try:
cipher = AES.new(bytearray(shared_secret), AES.MODE_CBC, bytearray(iv_bytes))
decrypted_bytes = cipher.decrypt(bytearray(encrypted_bytes))
plaintext = decrypted_bytes.decode("utf-8")
decoded = plaintext.rsplit("}", 1)[0] + "}" # weird symbols at the end
return decoded
except Exception as ex:
return str(ex)