diff --git a/.env_example b/.env_example new file mode 100644 index 0000000..8f5e51f --- /dev/null +++ b/.env_example @@ -0,0 +1,8 @@ +NOSTR_PRIVATE_KEY = nostrSecretkeyinhex +NOSTR_TEST_CLIENT_PRIVATE_KEY = nostrSecretkeyinhex_forthetestclient +USER_DB_PATH = nostrzaps.db + +LNBITS_INVOICE_KEY = lnbitswalletinvoicekey +LNBITS_HOST = https://lnbits.com + +TASK_TRANSLATION_NIP89_DTAG = abcded diff --git a/.gitignore b/.gitignore index 68bc17f..8bbe727 100644 --- a/.gitignore +++ b/.gitignore @@ -158,3 +158,5 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ +nostrzaps.db +.DS_Store diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..13566b8 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/.idea/dataSources.xml b/.idea/dataSources.xml new file mode 100644 index 0000000..211f211 --- /dev/null +++ b/.idea/dataSources.xml @@ -0,0 +1,12 @@ + + + + + sqlite.xerial + true + org.sqlite.JDBC + jdbc:sqlite:$PROJECT_DIR$/nostrzaps.db + $ProjectFileDir$ + + + \ No newline at end of file diff --git a/.idea/dvm.iml b/.idea/dvm.iml new file mode 100644 index 0000000..74d515a --- /dev/null +++ b/.idea/dvm.iml @@ -0,0 +1,10 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..8b60158 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,7 @@ + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..610d591 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/sqldialects.xml b/.idea/sqldialects.xml new file mode 100644 index 0000000..c0e01ca --- /dev/null +++ b/.idea/sqldialects.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..35eb1dd --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/README.md b/README.md index 8c60aed..2367040 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,12 @@ -# Nostr Data Vending Machine +# Nostr Data Vending Machine Python Implementation -A basic Implementation of a DVM written in Python +This example DVM implementation in Python currently supports simple translations using Google translate. + +At a later stage, additional example tasks will be added, as well as the integration into a larger Machine Learning backend + + +Place .env file (based on .env_example) in main folder, install requirements.txt (python 3.10) run main.py. Optionally supports LNbits to create invoices instead of lnaddresses. + +Use vendata.io to create a nip89 announcement of your dvm and save the dtag in your .env config. + +A tutorial on how to add additional tasks, as well as the larger server backend will be added soon. diff --git a/dvm.py b/dvm.py new file mode 100644 index 0000000..c6f668b --- /dev/null +++ b/dvm.py @@ -0,0 +1,470 @@ +from nostr_sdk import PublicKey, Keys, Client, Tag, Event, EventBuilder, Filter, HandleNotification, Timestamp, \ + init_logger, LogLevel +import time +import emoji +from utils.definitions import EventDefinitions, DVMConfig, RequiredJobToWatch, JobToWatch, LOCAL_TASKS +from utils.admin_utils import admin_make_database_updates +from utils.ai_utils import GoogleTranslate +from utils.backend_utils import get_amount_per_task, check_task_is_supported, get_task +from utils.database_utils import update_sql_table, get_from_sql_table, \ + create_sql_table, get_or_add_user, update_user_balance +from utils.nostr_utils import get_event_by_id, get_referenced_event_by_id, send_event +from utils.output_utils import post_process_result +from utils.requestform_utils import create_requestform_from_nostr_event +from utils.zap_utils import check_bolt11_ln_bits_is_paid, parse_bolt11_invoice, \ + check_for_zapplepay, decrypt_private_zap_message, create_bolt11_ln_bits + +use_logger = False +if use_logger: + init_logger(LogLevel.DEBUG) + +job_list = [] +jobs_on_hold_list = [] +dvm_config = DVMConfig() + + +def dvm(config): + dvm_config = config + keys = Keys.from_sk_str(dvm_config.PRIVATE_KEY) + pk = keys.public_key() + + print(f"Nostr DVM public key: {pk.to_bech32()}, Hex: {pk.to_hex()} ") + print(f"Supported DVM tasks: {dvm_config.SUPPORTED_TASKS}") + + client = Client(keys) + for relay in dvm_config.RELAY_LIST: + client.add_relay(relay) + client.connect() + + dm_zap_filter = Filter().pubkey(pk).kinds([EventDefinitions.KIND_ZAP]).since(Timestamp.now()) + dvm_filter = (Filter().kinds([EventDefinitions.KIND_NIP90_GENERIC, + EventDefinitions.KIND_NIP90_TRANSLATE_TEXT, + ]).since(Timestamp.now())) + client.subscribe([dm_zap_filter, dvm_filter]) + + create_sql_table() + admin_make_database_updates(config=dvm_config, client=client) + + class NotificationHandler(HandleNotification): + def handle(self, relay_url, nostr_event): + if EventDefinitions.KIND_NIP90_EXTRACT_TEXT <= nostr_event.kind() <= EventDefinitions.KIND_NIP90_GENERIC: + print(f"[Nostr] Received new NIP90 Job Request from {relay_url}: {nostr_event.as_json()}") + handle_nip90_job_event(nostr_event) + elif nostr_event.kind() == EventDefinitions.KIND_ZAP: + handle_zap(nostr_event) + + def handle_msg(self, relay_url, msg): + return + + def handle_nip90_job_event(event): + user = get_or_add_user(event.pubkey().to_hex()) + is_whitelisted = user[2] + is_blacklisted = user[3] + if is_whitelisted: + task_supported, task, duration = check_task_is_supported(event, client=client, get_duration=False, + config=dvm_config) + print(task) + else: + task_supported, task, duration = check_task_is_supported(event, client=client, get_duration=True, + config=dvm_config) + if is_blacklisted: + send_job_status_reaction(event, "error", client=client, config=dvm_config) + print("[Nostr] Request by blacklisted user, skipped") + + elif task_supported: + print("Received new Task: " + task) + print(duration) + amount = get_amount_per_task(task, duration, config=dvm_config) + if amount is None: + return + + if is_whitelisted: + print("[Nostr] Whitelisted for task " + task + ". Starting processing..") + send_job_status_reaction(event, "processing", True, 0, client=client, config=dvm_config) + do_work(event, is_from_bot=False) + # otherwise send payment request + else: + bid = 0 + for tag in event.tags(): + if tag.as_vec()[0] == 'bid': + bid = int(tag.as_vec()[1]) + + print("[Nostr][Payment required] New Nostr " + task + " Job event: " + event.as_json()) + if bid > 0: + bid_offer = int(bid / 1000) + if bid_offer >= amount: + send_job_status_reaction(event, "payment-required", False, + amount, # bid_offer + client=client, config=dvm_config) + + else: # If there is no bid, just request server rate from user + print("[Nostr] Requesting payment for Event: " + event.id().to_hex()) + send_job_status_reaction(event, "payment-required", + False, amount, client=client, config=dvm_config) + else: + print("Task not supported on this DVM, skipping..") + + def handle_zap(event): + zapped_event = None + invoice_amount = 0 + anon = False + sender = event.pubkey() + + try: + for tag in event.tags(): + if tag.as_vec()[0] == 'bolt11': + invoice_amount = parse_bolt11_invoice(tag.as_vec()[1]) + elif tag.as_vec()[0] == 'e': + zapped_event = get_event_by_id(tag.as_vec()[1], config=dvm_config) + elif tag.as_vec()[0] == 'description': + zap_request_event = Event.from_json(tag.as_vec()[1]) + sender = check_for_zapplepay(zap_request_event.pubkey().to_hex(), + zap_request_event.content()) + for ztag in zap_request_event.tags(): + if ztag.as_vec()[0] == 'anon': + if len(ztag.as_vec()) > 1: + print("Private Zap received.") + decrypted_content = decrypt_private_zap_message(ztag.as_vec()[1], + keys.secret_key(), + zap_request_event.pubkey()) + decrypted_private_event = Event.from_json(decrypted_content) + if decrypted_private_event.kind() == 9733: + sender = decrypted_private_event.pubkey().to_hex() + message = decrypted_private_event.content() + if message != "": + print("Zap Message: " + message) + else: + anon = True + print("Anonymous Zap received. Unlucky, I don't know from whom, and never will") + user = get_or_add_user(sender) + print(str(user)) + + if zapped_event is not None: + if zapped_event.kind() == EventDefinitions.KIND_FEEDBACK: # if a reaction by us got zapped + if not dvm_config.IS_BOT: + print("Zap received for NIP90 task: " + str(invoice_amount) + " Sats from " + str( + user[6])) + amount = 0 + job_event = None + for tag in zapped_event.tags(): + if tag.as_vec()[0] == 'amount': + amount = int(float(tag.as_vec()[1]) / 1000) + elif tag.as_vec()[0] == 'e': + job_event = get_event_by_id(tag.as_vec()[1], config=dvm_config) + + task_supported, task, duration = check_task_is_supported(job_event, client=client, + get_duration=False, + config=dvm_config) + if job_event is not None and task_supported: + if amount <= invoice_amount: + print("[Nostr] Payment-request fulfilled...") + send_job_status_reaction(job_event, "processing", client=client, + config=dvm_config) + indices = [i for i, x in enumerate(job_list) if + x.event_id == job_event.id().to_hex()] + index = -1 + if len(indices) > 0: + index = indices[0] + if index > -1: + if job_list[index].is_processed: # If payment-required appears a processing + job_list[index].is_paid = True + check_and_return_event(job_list[index].result, str(job_event.as_json()), + dvm_key=dvm_config.PRIVATE_KEY) + elif not (job_list[index]).is_processed: + # If payment-required appears before processing + job_list.pop(index) + print("Starting work...") + do_work(job_event, is_from_bot=False) + else: + print("Job not in List, but starting work...") + do_work(job_event, is_from_bot=False) + + else: + send_job_status_reaction(job_event, "payment-rejected", + False, invoice_amount, client=client, config=dvm_config) + print("[Nostr] Invoice was not paid sufficiently") + + elif zapped_event.kind() in EventDefinitions.ANY_RESULT: + print("Someone zapped the result of an exisiting Task. Nice") + elif not anon and not dvm_config.PASSIVE_MODE: + print("Note Zap received for Bot balance: " + str(invoice_amount) + " Sats from " + str( + user[6])) + update_user_balance(sender, invoice_amount, config=dvm_config) + + # a regular note + elif not anon and not dvm_config.PASSIVE_MODE: + print("Profile Zap received for Bot balance: " + str(invoice_amount) + " Sats from " + str( + user[6])) + update_user_balance(sender, invoice_amount, config=dvm_config) + + except Exception as e: + print(f"Error during content decryption: {e}") + + def do_work(job_event, is_from_bot=False): + if (( + EventDefinitions.KIND_NIP90_EXTRACT_TEXT <= job_event.kind() <= EventDefinitions.KIND_NIP90_GENERIC) + or job_event.kind() == EventDefinitions.KIND_DM): + + #We're building a request form here because we want to send larger tasks to a processing server later + request_form = create_requestform_from_nostr_event(job_event, is_from_bot, client, dvm_config) + task = get_task(job_event, client=client, dvmconfig=dvm_config) + # TASKS that take time or need GPU are moved to a backend server in the future + if task not in LOCAL_TASKS: + if task.startswith("unknown"): + print("Task not (yet) supported") + return + else: + print("[Nostr] Scheduling " + task + " Job event: " + job_event.as_json()) + print("We will employ a backend server here in the future") + + else: + print("[Nostr] Scheduling local " + task + " Job event: " + job_event.as_json()) + result = "" + print("Setting options...") + opts = [] + if request_form.get("optStr"): + for k, v in [option.split("=") for option in request_form["optStr"].split(";")]: + t = (k, v) + opts.append(t) + print(k + "=" + v) + print("...done.") + options = dict(opts) + + if task == "translation": + result = GoogleTranslate(options["text"], options["translation_lang"]) + # TODO ADD FURTHER LOCAL TASKS HERE + + check_and_return_event(result, str(job_event.as_json()), + dvm_key=dvm_config.PRIVATE_KEY) + + client.handle_notifications(NotificationHandler()) + + while True: + for job in job_list: + if job.bolt11 != "" and job.payment_hash != "" and not job.is_paid: + if str(check_bolt11_ln_bits_is_paid(job.payment_hash, dvm_config)) == "True": + job.is_paid = True + event = get_event_by_id(job.event_id, config=dvm_config) + if event != None: + send_job_status_reaction(event, "processing", True, 0, client=client, config=dvm_config) + print("do work from joblist") + + do_work(event, is_from_bot=False) + elif check_bolt11_ln_bits_is_paid(job.payment_hash, dvm_config) is None: # invoice expired + job_list.remove(job) + + if Timestamp.now().as_secs() > job.expires: + job_list.remove(job) + + for job in jobs_on_hold_list: + if check_event_has_not_unifinished_job_input(job.event, False, client=client, dvmconfig=dvm_config): + handle_nip90_job_event(job.event) + jobs_on_hold_list.remove(job) + + if Timestamp.now().as_secs() > job.timestamp + 60 * 20: # remove jobs to look for after 20 minutes.. + jobs_on_hold_list.remove(job) + + time.sleep(1.0) + + +def check_event_has_not_unifinished_job_input(nevent, append, client, dvmconfig): + task_supported, task, duration = check_task_is_supported(nevent, client, False, config=dvmconfig) + if not task_supported: + return False + + for tag in nevent.tags(): + if tag.as_vec()[0] == 'i': + if len(tag.as_vec()) < 3: + print("Job Event missing/malformed i tag, skipping..") + return False + else: + input = tag.as_vec()[1] + input_type = tag.as_vec()[2] + if input_type == "job": + evt = get_referenced_event_by_id(input, EventDefinitions.ANY_RESULT, client, config=dvmconfig) + if evt is None: + if append: + job = RequiredJobToWatch(event=nevent, timestamp=Timestamp.now().as_secs()) + jobs_on_hold_list.append(job) + send_job_status_reaction(nevent, "chain-scheduled", True, 0, client=client, + config=dvmconfig) + + return False + else: + return True + + +def send_job_status_reaction(original_event, status, is_paid=True, amount=0, client=None, content=None, config=None, + key=None): + dvmconfig = config + altdesc = "This is a reaction to a NIP90 DVM AI task. " + task = get_task(original_event, client=client, dvmconfig=dvmconfig) + if status == "processing": + altdesc = "NIP90 DVM AI task " + task + " started processing. " + reaction = altdesc + emoji.emojize(":thumbs_up:") + elif status == "success": + altdesc = "NIP90 DVM AI task " + task + " finished successfully. " + reaction = altdesc + emoji.emojize(":call_me_hand:") + elif status == "chain-scheduled": + altdesc = "NIP90 DVM AI task " + task + " Chain Task scheduled" + reaction = altdesc + emoji.emojize(":thumbs_up:") + elif status == "error": + altdesc = "NIP90 DVM AI task " + task + " had an error. " + if content is None: + reaction = altdesc + emoji.emojize(":thumbs_down:") + else: + reaction = altdesc + emoji.emojize(":thumbs_down:") + content + + elif status == "payment-required": + + altdesc = "NIP90 DVM AI task " + task + " requires payment of min " + str(amount) + " Sats. " + reaction = altdesc + emoji.emojize(":orange_heart:") + + elif status == "payment-rejected": + altdesc = "NIP90 DVM AI task " + task + " payment is below required amount of " + str(amount) + " Sats. " + reaction = altdesc + emoji.emojize(":thumbs_down:") + elif status == "user-blocked-from-service": + + altdesc = "NIP90 DVM AI task " + task + " can't be performed. User has been blocked from Service. " + reaction = altdesc + emoji.emojize(":thumbs_down:") + else: + reaction = emoji.emojize(":thumbs_down:") + + etag = Tag.parse(["e", original_event.id().to_hex()]) + ptag = Tag.parse(["p", original_event.pubkey().to_hex()]) + alttag = Tag.parse(["alt", altdesc]) + statustag = Tag.parse(["status", status]) + tags = [etag, ptag, alttag, statustag] + + if status == "success" or status == "error": # + for x in job_list: + if x.event_id == original_event.id(): + is_paid = x.is_paid + amount = x.amount + break + + bolt11 = "" + payment_hash = "" + expires = original_event.created_at().as_secs() + (60 * 60 * 24) + if status == "payment-required" or (status == "processing" and not is_paid): + if dvmconfig.LNBITS_INVOICE_KEY != "": + try: + bolt11, payment_hash = create_bolt11_ln_bits(amount, dvmconfig) + except Exception as e: + print(e) + + if not any(x.event_id == original_event.id().to_hex() for x in job_list): + job_list.append( + JobToWatch(event_id=original_event.id().to_hex(), timestamp=original_event.created_at().as_secs(), + amount=amount, + is_paid=is_paid, + status=status, result="", is_processed=False, bolt11=bolt11, payment_hash=payment_hash, + expires=expires, from_bot=False)) + print(str(job_list)) + if status == "payment-required" or status == "payment-rejected" or (status == "processing" and not is_paid) or ( + status == "success" and not is_paid): + + if dvmconfig.LNBITS_INVOICE_KEY != "": + amount_tag = Tag.parse(["amount", str(amount * 1000), bolt11]) + else: + amount_tag = Tag.parse(["amount", str(amount * 1000)]) # to millisats + tags.append(amount_tag) + if key is not None: + keys = Keys.from_sk_str(key) + else: + keys = Keys.from_sk_str(dvmconfig.PRIVATE_KEY) + event = EventBuilder(EventDefinitions.KIND_FEEDBACK, reaction, tags).to_event(keys) + + send_event(event, key=keys) + print("[Nostr] Sent Kind " + str(EventDefinitions.KIND_FEEDBACK) + " Reaction: " + status + " " + event.as_json()) + return event.as_json() + + +def send_nostr_reply_event(content, original_event_as_str, key=None): + originalevent = Event.from_json(original_event_as_str) + requesttag = Tag.parse(["request", original_event_as_str.replace("\\", "")]) + etag = Tag.parse(["e", originalevent.id().to_hex()]) + ptag = Tag.parse(["p", originalevent.pubkey().to_hex()]) + alttag = Tag.parse(["alt", "This is the result of a NIP90 DVM AI task with kind " + str( + originalevent.kind()) + ". The task was: " + originalevent.content()]) + statustag = Tag.parse(["status", "success"]) + replytags = [requesttag, etag, ptag, alttag, statustag] + for tag in originalevent.tags(): + if tag.as_vec()[0] == "i": + icontent = tag.as_vec()[1] + ikind = tag.as_vec()[2] + itag = Tag.parse(["i", icontent, ikind]) + replytags.append(itag) + + if key is None: + key = Keys.from_sk_str(dvm_config.PRIVATE_KEY) + + response_kind = originalevent.kind() + 1000 + event = EventBuilder(response_kind, str(content), replytags).to_event(key) + send_event(event, key=key) + print("[Nostr] " + str(response_kind) + " Job Response event sent: " + event.as_json()) + return event.as_json() + + +def respond_to_error(content, originaleventstr, is_from_bot=False, dvm_key=None): + print("ERROR") + if dvm_key is None: + keys = Keys.from_sk_str(dvm_config.PRIVATE_KEY) + else: + keys = Keys.from_sk_str(dvm_key) + + original_event = Event.from_json(originaleventstr) + sender = "" + task = "" + if not is_from_bot: + send_job_status_reaction(original_event, "error", content=content, key=dvm_key) + # TODO Send Zap back + else: + for tag in original_event.tags(): + if tag.as_vec()[0] == "p": + sender = tag.as_vec()[1] + elif tag.as_vec()[0] == "i": + task = tag.as_vec()[1] + + user = get_from_sql_table(sender) + is_whitelisted = user[2] + if not is_whitelisted: + amount = int(user[1]) + get_amount_per_task(task) + update_sql_table(sender, amount, user[2], user[3], user[4], user[5], user[6], + Timestamp.now().as_secs()) + message = "There was the following error : " + content + ". Credits have been reimbursed" + else: + # User didn't pay, so no reimbursement + message = "There was the following error : " + content + + evt = EventBuilder.new_encrypted_direct_msg(keys, PublicKey.from_hex(sender), message, None).to_event(keys) + send_event(evt, key=keys) + + +def check_and_return_event(data, original_event_str: str, dvm_key=""): + original_event = Event.from_json(original_event_str) + keys = Keys.from_sk_str(dvm_key) + + for x in job_list: + if x.event_id == original_event.id().to_hex(): + is_paid = x.is_paid + amount = x.amount + x.result = data + x.is_processed = True + if dvm_config.SHOWRESULTBEFOREPAYMENT and not is_paid: + send_nostr_reply_event(data, original_event_str, key=keys) + send_job_status_reaction(original_event, "success", amount, + config=dvm_config) # or payment-required, or both? + elif not dvm_config.SHOWRESULTBEFOREPAYMENT and not is_paid: + send_job_status_reaction(original_event, "success", amount, + config=dvm_config) # or payment-required, or both? + + if dvm_config.SHOWRESULTBEFOREPAYMENT and is_paid: + job_list.remove(x) + elif not dvm_config.SHOWRESULTBEFOREPAYMENT and is_paid: + job_list.remove(x) + send_nostr_reply_event(data, original_event_str, key=keys) + break + + post_processed_content = post_process_result(data, original_event) + send_nostr_reply_event(post_processed_content, original_event_str, key=keys) diff --git a/main.py b/main.py new file mode 100644 index 0000000..4acb86e --- /dev/null +++ b/main.py @@ -0,0 +1,48 @@ +import os +from pathlib import Path +from threading import Thread + +import dotenv +import utils.env as env +from utils.definitions import EventDefinitions + + +def run_nostr_dvm_with_local_config(): + from dvm import dvm, DVMConfig + from utils.nip89_utils import NIP89Announcement + + + dvmconfig = DVMConfig() + dvmconfig.PRIVATE_KEY = os.getenv(env.NOSTR_PRIVATE_KEY) + dvmconfig.SUPPORTED_TASKS = ["translation"] + dvmconfig.LNBITS_INVOICE_KEY = os.getenv(env.LNBITS_INVOICE_KEY) + dvmconfig.LNBITS_URL = os.getenv(env.LNBITS_HOST) + + nip89translation = NIP89Announcement() + nip89translation.kind = EventDefinitions.KIND_NIP90_TRANSLATE_TEXT + nip89translation.dtag = os.getenv(env.TASK_TRANSLATION_NIP89_DTAG) + nip89translation.pk = os.getenv(env.NOSTR_PRIVATE_KEY) + nip89translation.content = "{\"name\":\"NostrAI DVM Translator\",\"image\":\"https://cdn.nostr.build/i/feb98d8700abe7d6c67d9106a72a20354bf50805af79869638f5a32d24a5ac2a.jpg\",\"about\":\"Translates Text from given text/event/job, currently using Google Translation Services into language defined in param. \",\"nip90Params\":{\"language\":{\"required\":true,\"values\":[\"af\",\"am\",\"ar\",\"az\",\"be\",\"bg\",\"bn\",\"bs\",\"ca\",\"ceb\",\"co\",\"cs\",\"cy\",\"da\",\"de\",\"el\",\"eo\",\"es\",\"et\",\"eu\",\"fa\",\"fi\",\"fr\",\"fy\",\"ga\",\"gd\",\"gl\",\"gu\",\"ha\",\"haw\",\"hi\",\"hmn\",\"hr\",\"ht\",\"hu\",\"hy\",\"id\",\"ig\",\"is\",\"it\",\"he\",\"ja\",\"jv\",\"ka\",\"kk\",\"km\",\"kn\",\"ko\",\"ku\",\"ky\",\"la\",\"lb\",\"lo\",\"lt\",\"lv\",\"mg\",\"mi\",\"mk\",\"ml\",\"mn\",\"mr\",\"ms\",\"mt\",\"my\",\"ne\",\"nl\",\"no\",\"ny\",\"or\",\"pa\",\"pl\",\"ps\",\"pt\",\"ro\",\"ru\",\"sd\",\"si\",\"sk\",\"sl\",\"sm\",\"sn\",\"so\",\"sq\",\"sr\",\"st\",\"su\",\"sv\",\"sw\",\"ta\",\"te\",\"tg\",\"th\",\"tl\",\"tr\",\"ug\",\"uk\",\"ur\",\"uz\",\"vi\",\"xh\",\"yi\",\"yo\",\"zh\",\"zu\"]}}}" + dvmconfig.NIP89s.append(nip89translation) + + + nostr_dvm_thread = Thread(target=dvm, args=[dvmconfig]) + nostr_dvm_thread.start() + + +if __name__ == '__main__': + + env_path = Path('.env') + if env_path.is_file(): + print(f'loading environment from {env_path.resolve()}') + dotenv.load_dotenv(env_path, verbose=True, override=True) + else: + raise FileNotFoundError(f'.env file not found at {env_path} ') + + + + run_nostr_dvm_with_local_config() + + + + diff --git a/outputs/.gitkeep b/outputs/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..7ac6045 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,32 @@ +beautifulsoup4==4.12.2 +bech32==1.2.0 +blessed==1.20.0 +certifi==2023.7.22 +charset-normalizer==3.3.2 +emoji==2.8.0 +ffmpegio==0.8.5 +ffmpegio-core==0.8.5 +idna==3.4 +inquirer==3.1.3 +install==1.3.5 +nostr-sdk==0.0.4 +numpy==1.26.2 +packaging==23.2 +pandas==2.1.3 +Pillow==10.1.0 +pluggy==1.3.0 +pycryptodome==3.19.0 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-editor==1.0.4 +pytz==2023.3.post1 +pyuseragents==1.0.5 +readchar==4.0.5 +requests==2.31.0 +safeIO==1.2 +six==1.16.0 +soupsieve==2.5 +translatepy==2.3 +tzdata==2023.3 +urllib3==2.1.0 +wcwidth==0.2.10 diff --git a/test_client.py b/test_client.py new file mode 100644 index 0000000..4e7a4bf --- /dev/null +++ b/test_client.py @@ -0,0 +1,100 @@ + +import os +import time +import datetime as datetime +from pathlib import Path +from threading import Thread + +import dotenv +from nostr_sdk import Keys, Client, Tag, EventBuilder, Filter, HandleNotification, Timestamp, nip04_decrypt + +from utils.nostr_utils import send_event +from utils.definitions import EventDefinitions, RELAY_LIST + +import utils.env as env +#TODO HINT: Only use this path with a preiously whitelisted privkey, as zapping events is not implemented in the lib/code +def nostr_client_test_translation(input, kind, lang, sats, satsmax): + keys = Keys.from_sk_str(os.getenv(env.NOSTR_TEST_CLIENT_PRIVATE_KEY)) + if kind == "text": + iTag = Tag.parse(["i", input, "text"]) + elif kind == "event": + iTag = Tag.parse(["i", input, "event"]) + paramTag1 = Tag.parse(["param", "language", lang]) + + bidTag = Tag.parse(['bid', str(sats * 1000), str(satsmax * 1000)]) + relaysTag = Tag.parse(['relays', "wss://relay.damus.io", "wss://blastr.f7z.xyz", "wss://relayable.org", "wss://nostr-pub.wellorder.net"]) + alttag = Tag.parse(["alt", "This is a NIP90 DVM AI task to translate a given Input"]) + event = EventBuilder(EventDefinitions.KIND_NIP90_TRANSLATE_TEXT, str("Translate the given input."), [iTag, paramTag1, bidTag, relaysTag, alttag]).to_event(keys) + + relay_list = ["wss://relay.damus.io", "wss://blastr.f7z.xyz", "wss://relayable.org", + "wss://nostr-pub.wellorder.net"] + + + client = Client(keys) + for relay in relay_list: + client.add_relay(relay) + client.connect() + send_event(event, client, keys) + return event.as_json() + +def nostr_client(): + keys = Keys.from_sk_str(os.getenv(env.NOSTR_TEST_CLIENT_PRIVATE_KEY)) + sk = keys.secret_key() + pk = keys.public_key() + print(f"Nostr Client public key: {pk.to_bech32()}, Hex: {pk.to_hex()} ") + client = Client(keys) + for relay in RELAY_LIST: + client.add_relay(relay) + client.connect() + + dm_zap_filter = Filter().pubkey(pk).kinds([EventDefinitions.KIND_DM, + EventDefinitions.KIND_ZAP]).since(Timestamp.now()) # events to us specific + dvm_filter = (Filter().kinds([EventDefinitions.KIND_NIP90_RESULT_TRANSLATE_TEXT, + EventDefinitions.KIND_FEEDBACK]).since(Timestamp.now())) # public events + client.subscribe([dm_zap_filter, dvm_filter]) + + + #nostr_client_test_translation("This is the result of the DVM in spanish", "text", "es", 20, 20) + nostr_client_test_translation("44a0a8b395ade39d46b9d20038b3f0c8a11168e67c442e3ece95e4a1703e2beb", "event", "fr", 20, 20) + + + #nostr_client_test_image(sats=50, satsmax=10) + class NotificationHandler(HandleNotification): + def handle(self, relay_url, event): + print(f"Received new event from {relay_url}: {event.as_json()}") + if event.kind() == 7000: + print("[Nostr Client]: " + event.as_json()) + elif event.kind() > 6000 and event.kind() < 6999: + print("[Nostr Client]: " + event.as_json()) + print("[Nostr Client]: " + event.content()) + + elif event.kind() == 4: + dec_text = nip04_decrypt(sk, event.pubkey(), event.content()) + print("[Nostr Client]: " + f"Received new msg: {dec_text}") + + elif event.kind() == 9735: + print("[Nostr Client]: " + f"Received new zap:") + print(event.as_json()) + + + def handle_msg(self, relay_url, msg): + None + + client.handle_notifications(NotificationHandler()) + while True: + time.sleep(5.0) + + + +if __name__ == '__main__': + + env_path = Path('.env') + if env_path.is_file(): + print(f'loading environment from {env_path.resolve()}') + dotenv.load_dotenv(env_path, verbose=True, override=True) + else: + raise FileNotFoundError(f'.env file not found at {env_path} ') + + + nostr_dvm_thread = Thread(target=nostr_client()) + nostr_dvm_thread.start() diff --git a/utils/admin_utils.py b/utils/admin_utils.py new file mode 100644 index 0000000..9aae0b8 --- /dev/null +++ b/utils/admin_utils.py @@ -0,0 +1,67 @@ +# ADMINISTRARIVE DB MANAGEMENT +import time + +from nostr_sdk import Keys, EventBuilder, PublicKey + +from utils.database_utils import get_from_sql_table, list_db, clear_db, delete_from_sql_table, update_sql_table, \ + get_or_add_user, update_user_metadata +from utils.nip89_utils import nip89_announce_tasks +from utils.nostr_utils import send_event + + +def admin_make_database_updates(config=None, client=None): + # This is called on start of Server, Admin function to manually whitelist/blacklist/add balance/delete users + dvmconfig = config + + rebroadcast_nip89 = False + cleardb = False + listdatabase = False + deleteuser = False + whitelistuser = False + unwhitelistuser = False + blacklistuser = False + addbalance = False + additional_balance = 50 + + # publickey = PublicKey.from_bech32("npub1...").to_hex() + # use this if you have the npub + publickey = "asd123" + #use this if you have hex + + if whitelistuser: + user = get_or_add_user(publickey) + update_sql_table(user[0], user[1], True, False, user[4], user[5], user[6], user[7]) + user = get_from_sql_table(publickey) + print(str(user[6]) + " is whitelisted: " + str(user[2])) + + if unwhitelistuser: + user = get_from_sql_table(publickey) + update_sql_table(user[0], user[1], False, False, user[4], user[5], user[6], user[7]) + + if blacklistuser: + user = get_from_sql_table(publickey) + update_sql_table(user[0], user[1], False, True, user[4], user[5], user[6], user[7]) + + if addbalance: + user = get_from_sql_table(publickey) + update_sql_table(user[0], (int(user[1]) + additional_balance), user[2], user[3], user[4], user[5], user[6], + user[7]) + time.sleep(1.0) + message = str(additional_balance) + " Sats have been added to your balance. Your new balance is " + str( + (int(user[1]) + additional_balance)) + " Sats." + keys = Keys.from_sk_str(config.PRIVATE_KEY) + evt = EventBuilder.new_encrypted_direct_msg(keys, PublicKey.from_hex(publickey), message, + None).to_event(keys) + send_event(evt, key=keys) + + if deleteuser: + delete_from_sql_table(publickey) + + if cleardb: + clear_db() + + if listdatabase: + list_db() + + if rebroadcast_nip89: + nip89_announce_tasks(dvmconfig) diff --git a/utils/ai_utils.py b/utils/ai_utils.py new file mode 100644 index 0000000..cedcb7c --- /dev/null +++ b/utils/ai_utils.py @@ -0,0 +1,38 @@ + +#We can add multiple Tasks here and call them in the do_work function. + +#Also make sure to define your task in get supported tasks, and in get amount per task and listen to +#the according event type in the beginning of dvm.py and + + +def GoogleTranslate(text, translation_lang): + from translatepy.translators.google import GoogleTranslate + gtranslate = GoogleTranslate() + length = len(text) + + step = 0 + translated_text = "" + if length > 4999: + while step+5000 < length: + textpart = text[step:step+5000] + step = step + 5000 + try: + translated_text_part = str(gtranslate.translate(textpart, translation_lang)) + print("Translated Text part:\n\n " + translated_text_part) + except: + translated_text_part = "An error occured" + + translated_text = translated_text + translated_text_part + + if step < length: + textpart = text[step:length] + try: + translated_text_part = str(gtranslate.translate(textpart, translation_lang)) + print("Translated Text part:\n\n " + translated_text_part) + except: + translated_text_part = "An error occured" + + translated_text = translated_text + translated_text_part + + + return translated_text diff --git a/utils/backend_utils.py b/utils/backend_utils.py new file mode 100644 index 0000000..ec5efa2 --- /dev/null +++ b/utils/backend_utils.py @@ -0,0 +1,109 @@ + +import requests + +from utils.definitions import EventDefinitions +from utils.nostr_utils import get_event_by_id + + +def get_task(event, client, dvmconfig): + if event.kind() == EventDefinitions.KIND_NIP90_GENERIC: # use this for events that have no id yet + for tag in event.tags(): + if tag.as_vec()[0] == 'j': + return tag.as_vec()[1] + else: + return "unknown job: " + event.as_json() + elif event.kind() == EventDefinitions.KIND_DM: # dm + for tag in event.tags(): + if tag.as_vec()[0] == 'j': + return tag.as_vec()[1] + else: + return "unknown job: " + event.as_json() + + elif event.kind() == EventDefinitions.KIND_NIP90_TRANSLATE_TEXT: + return "translation" + + else: + return "unknown type" +def check_task_is_supported(event, client, get_duration = False, config=None): + dvmconfig = config + input_value = "" + input_type = "" + duration = 1 + + output_is_set = True + + for tag in event.tags(): + if tag.as_vec()[0] == 'i': + if len(tag.as_vec()) < 3: + print("Job Event missing/malformed i tag, skipping..") + return False, "", 0 + else: + input_value = tag.as_vec()[1] + input_type = tag.as_vec()[2] + if input_type == "event": + evt = get_event_by_id(input_value, config=dvmconfig) + if evt == None: + print("Event not found") + return False, "", 0 + + elif tag.as_vec()[0] == 'output': + output = tag.as_vec()[1] + output_is_set = True + if not (output == "text/plain" or output == "text/json" or output == "json" or output == "image/png" or "image/jpg" or output == ""): + print("Output format not supported, skipping..") + return False, "", 0 + + task = get_task(event, client=client, dvmconfig=dvmconfig) + if not output_is_set: + print("No output set") + if task not in dvmconfig.SUPPORTED_TASKS: # The Tasks this DVM supports (can be extended) + return False, task, duration + elif task == "translation" and ( + input_type != "event" and input_type != "job" and input_type != "text"): # The input types per task + return False, task, duration + if task == "translation" and input_type != "text" and len(event.content()) > 4999: # Google Services have a limit of 5000 signs + return False, task, duration + if input_type == 'url' and check_url_is_readable(input_value) is None: + print("url not readable") + return False, task, duration + + return True, task, duration + +def check_url_is_readable(url): + if not str(url).startswith("http"): + return None + # If it's a YouTube oder Overcast link, we suppose we support it + if (str(url).replace("http://", "").replace("https://", "").replace("www.", "").replace("youtu.be/", + "youtube.com?v=")[ + 0:11] == "youtube.com" and str(url).find("live") == -1) or str(url).startswith('https://x.com') or str(url).startswith('https://twitter.com') : + return "video" + + elif str(url).startswith("https://overcast.fm/"): + return "audio" + + # If link is comaptible with one of these file formats, it's fine. + req = requests.get(url) + content_type = req.headers['content-type'] + if content_type == 'audio/x-wav' or str(url).endswith(".wav") or content_type == 'audio/mpeg' or str(url).endswith( + ".mp3") or content_type == 'audio/ogg' or str(url).endswith(".ogg"): + return "audio" + elif content_type == 'image/png' or str(url).endswith(".png") or content_type == 'image/jpg' or str(url).endswith( + ".jpg") or content_type == 'image/jpeg' or str(url).endswith(".jpeg") or str(url).endswith(".pdf") or content_type == 'image/png' or str( + url).endswith(".png"): + return "image" + elif content_type == 'video/mp4' or str(url).endswith(".mp4") or content_type == 'video/avi' or str(url).endswith( + ".avi") or content_type == 'video/mov' or str(url).endswith(".mov"): + return "video" + # Otherwise we will not offer to do the job. + return None + +def get_amount_per_task(task, duration = 0, config=None): + dvmconfig = config + if task == "translation": + amount = dvmconfig.COSTPERUNIT_TRANSLATION + else: + print("[Nostr] Task " + task + " is currently not supported by this instance, skipping") + return None + return amount + + diff --git a/utils/database_utils.py b/utils/database_utils.py new file mode 100644 index 0000000..03951e5 --- /dev/null +++ b/utils/database_utils.py @@ -0,0 +1,202 @@ +# DATABASE LOGIC +import os +import sqlite3 +import time + +from _sqlite3 import Error +from datetime import timedelta +from logging import Filter + +from nostr_sdk import Timestamp, Keys, PublicKey, EventBuilder, Metadata, Filter + +from utils import env +from utils.definitions import NEW_USER_BALANCE +from utils.nostr_utils import send_event + +def create_sql_table(): + try: + con = sqlite3.connect(os.getenv(env.USER_DB_PATH)) + cur = con.cursor() + cur.execute(""" CREATE TABLE IF NOT EXISTS users ( + npub text PRIMARY KEY, + sats integer NOT NULL, + iswhitelisted boolean, + isblacklisted boolean, + nip05 text, + lud16 text, + name text, + lastactive integer + ); """) + cur.execute("SELECT name FROM sqlite_master") + con.close() + + except Error as e: + print(e) + + +def add_sql_table_column(): + try: + con = sqlite3.connect(os.getenv(env.USER_DB_PATH)) + cur = con.cursor() + cur.execute(""" ALTER TABLE users ADD COLUMN lastactive 'integer' """) + con.close() + except Error as e: + print(e) + + +def add_to_sql_table(npub, sats, iswhitelisted, isblacklisted, nip05, lud16, name, lastactive): + try: + con = sqlite3.connect(os.getenv(env.USER_DB_PATH)) + cur = con.cursor() + data = (npub, sats, iswhitelisted, isblacklisted, nip05, lud16, name, lastactive) + cur.execute("INSERT or IGNORE INTO users VALUES(?, ?, ?, ?, ?, ?, ?, ?)", data) + con.commit() + con.close() + except Error as e: + print(e) + + +def update_sql_table(npub, sats, iswhitelisted, isblacklisted, nip05, lud16, name, lastactive): + try: + con = sqlite3.connect(os.getenv(env.USER_DB_PATH)) + cur = con.cursor() + data = (sats, iswhitelisted, isblacklisted, nip05, lud16, name, lastactive, npub) + + cur.execute(""" UPDATE users + SET sats = ? , + iswhitelisted = ? , + isblacklisted = ? , + nip05 = ? , + lud16 = ? , + name = ? , + lastactive = ? + WHERE npub = ?""", data) + con.commit() + con.close() + except Error as e: + print(e) + + +def get_from_sql_table(npub): + try: + con = sqlite3.connect(os.getenv(env.USER_DB_PATH)) + cur = con.cursor() + cur.execute("SELECT * FROM users WHERE npub=?", (npub,)) + row = cur.fetchone() + con.close() + return row + + except Error as e: + print(e) + + +def delete_from_sql_table(npub): + try: + con = sqlite3.connect(os.getenv(env.USER_DB_PATH)) + cur = con.cursor() + cur.execute("DELETE FROM users WHERE npub=?", (npub,)) + con.commit() + con.close() + except Error as e: + print(e) + + +def clear_db(): + try: + con = sqlite3.connect(os.getenv(env.USER_DB_PATH)) + cur = con.cursor() + cur.execute("SELECT * FROM users WHERE npub IS NULL OR npub = '' ") + rows = cur.fetchall() + for row in rows: + print(row) + delete_from_sql_table(row[0]) + con.close() + return rows + except Error as e: + print(e) + + +def list_db(): + try: + con = sqlite3.connect(os.getenv(env.USER_DB_PATH)) + cur = con.cursor() + cur.execute("SELECT * FROM users ORDER BY sats DESC") + rows = cur.fetchall() + for row in rows: + print(row) + con.close() + except Error as e: + print(e) + + +def update_user_balance(sender, sats, config=None): + user = get_from_sql_table(sender) + if user is None: + add_to_sql_table(sender, (int(sats) + NEW_USER_BALANCE), False, False, + "", "", "", Timestamp.now().as_secs()) + print("NEW USER: " + sender + " Zap amount: " + str(sats) + " Sats.") + else: + user = get_from_sql_table(sender) + print(str(sats)) + nip05 =user[4] + lud16 = user[5] + name = user[6] + + if nip05 is None: + nip05 = "" + if lud16 is None: + lud16 = "" + if name is None: + name = "" + + new_balance = int(user[1]) + int(sats) + update_sql_table(sender, new_balance, user[2], user[3], nip05, lud16, name, + Timestamp.now().as_secs()) + print("UPDATE USER BALANCE: " + str(name) + " Zap amount: " + str(sats) + " Sats.") + + + if config is not None: + keys = Keys.from_sk_str(config.PRIVATE_KEY) + time.sleep(1.0) + + message = ("Added "+ str(sats) + " Sats to balance. New balance is " + str(new_balance) + " Sats. " ) + + + evt = EventBuilder.new_encrypted_direct_msg(keys, PublicKey.from_hex(sender), message, + None).to_event(keys) + send_event(evt, key=keys) + + +def get_or_add_user(sender): + user = get_from_sql_table(sender) + + if user is None: + add_to_sql_table(sender, NEW_USER_BALANCE, False, False, None, + None, None, Timestamp.now().as_secs()) + user = get_from_sql_table(sender) + print(user) + + return user + +def update_user_metadata(sender, client): + user = get_from_sql_table(sender) + name = user[6] + lud16 = user[5] + nip05 = user[4] + try: + profile_filter = Filter().kind(0).author(sender).limit(1) + events = client.get_events_of([profile_filter], timedelta(seconds=3)) + if len(events) > 0: + ev = events[0] + metadata = Metadata.from_json(ev.content()) + name = metadata.get_display_name() + if str(name) == "" or name is None: + name = metadata.get_name() + nip05 = metadata.get_nip05() + lud16 = metadata.get_lud16() + except: + print("Couldn't get meta information") + update_sql_table(user[0], user[1], user[2], user[3], nip05, lud16, + name, Timestamp.now().as_secs()) + user = get_from_sql_table(user[0]) + return user diff --git a/utils/definitions.py b/utils/definitions.py new file mode 100644 index 0000000..6857ae2 --- /dev/null +++ b/utils/definitions.py @@ -0,0 +1,81 @@ +from dataclasses import dataclass + +from nostr_sdk import Event +NEW_USER_BALANCE = 250 + +LOCAL_TASKS = ["conversion", "summarization","note-recommendation", "inactive-following", "image-upscale", "translation"] +# Tasks performed by the DVM and not send to nova-server (can change later) + +RELAY_LIST = ["wss://relay.damus.io", "wss://nostr-pub.wellorder.net", "wss://nos.lol", "wss://nostr.wine", + "wss://relay.nostfiles.dev", "wss://nostr.mom", "wss://nostr.oxtr.dev", "wss://relay.nostr.bg", "wss://relay.f7z.io"] +class EventDefinitions: + KIND_DM: int = 4 + KIND_ZAP: int = 9735 + KIND_NIP94_METADATA: int = 1063 + KIND_FEEDBACK: int = 7000 + KIND_NIP90_EXTRACT_TEXT = 5000 + KIND_NIP90_RESULT_EXTRACT_TEXT = 6000 + KIND_NIP90_SUMMARIZE_TEXT = 5001 + KIND_NIP90_RESULT_SUMMARIZE_TEXT = 6001 + KIND_NIP90_TRANSLATE_TEXT = 5002 + KIND_NIP90_RESULT_TRANSLATE_TEXT = 6002 + KIND_NIP90_GENERATE_IMAGE = 5100 + KIND_NIP90_RESULT_GENERATE_IMAGE = 6100 + KIND_NIP90_RECOMMEND_NOTES = 65006 + KIND_NIP90_RESULT_RECOMMEND_NOTES = 65001 + KIND_NIP90_RECOMMEND_USERS = 65007 + KIND_NIP90_RESULT_RECOMMEND_USERS = 65001 + KIND_NIP90_CONVERT_VIDEO = 5200 + KIND_NIP90_RESULT_CONVERT_VIDEO = 6200 + KIND_NIP90_GENERIC = 5999 + KIND_NIP90_RESULT_GENERIC = 6999 + ANY_RESULT = [KIND_NIP90_RESULT_EXTRACT_TEXT, + KIND_NIP90_RESULT_SUMMARIZE_TEXT, + KIND_NIP90_RESULT_TRANSLATE_TEXT, + KIND_NIP90_RESULT_GENERATE_IMAGE, + KIND_NIP90_RESULT_RECOMMEND_NOTES, + KIND_NIP90_RESULT_RECOMMEND_USERS, + KIND_NIP90_RESULT_CONVERT_VIDEO, + KIND_NIP90_RESULT_GENERIC] + + +class DVMConfig: + SUPPORTED_TASKS = [] + PRIVATE_KEY: str + + RELAY_LIST = ["wss://relay.damus.io", "wss://nostr-pub.wellorder.net", "wss://nos.lol", "wss://nostr.wine", + "wss://relay.nostfiles.dev", "wss://nostr.mom", "wss://nostr.oxtr.dev", "wss://relay.nostr.bg", "wss://relay.f7z.io"] + RELAY_TIMEOUT = 5 + LNBITS_INVOICE_KEY = '' + LNBITS_URL = 'https://lnbits.com' + REQUIRES_NIP05: bool = False + + + SHOWRESULTBEFOREPAYMENT: bool = True # if this is true show results even when not paid right after autoprocess + NEW_USER_BALANCE: int = 250 # Free credits for new users + + COSTPERUNIT_TRANSLATION: int = 20 # Still need to multiply this by duration + + NIP89s: list = [] + + + + +@dataclass +class JobToWatch: + event_id: str + timestamp: int + is_paid: bool + amount: int + status: str + result: str + is_processed: bool + bolt11: str + payment_hash: str + expires: int + from_bot: bool + +@dataclass +class RequiredJobToWatch: + event: Event + timestamp: int \ No newline at end of file diff --git a/utils/env.py b/utils/env.py new file mode 100644 index 0000000..7df959c --- /dev/null +++ b/utils/env.py @@ -0,0 +1,11 @@ +NOSTR_PRIVATE_KEY = "NOSTR_PRIVATE_KEY" +NOSTR_TEST_CLIENT_PRIVATE_KEY = "NOSTR_TEST_CLIENT_PRIVATE_KEY" + +USER_DB_PATH = "USER_DB_PATH" + +LNBITS_INVOICE_KEY = "LNBITS_INVOICE_KEY" +LNBITS_HOST = "LNBITS_HOST" + +TASK_TRANSLATION_NIP89_DTAG = "TASK_TRANSLATION_NIP89_DTAG" + + diff --git a/utils/nip89_utils.py b/utils/nip89_utils.py new file mode 100644 index 0000000..220e53c --- /dev/null +++ b/utils/nip89_utils.py @@ -0,0 +1,19 @@ +from nostr_sdk import Tag, Keys, EventBuilder +from utils.nostr_utils import send_event + +class NIP89Announcement: + kind: int + dtag: str + pk: str + content: str + +def nip89_announce_tasks(dvmconfig): + for nip89 in dvmconfig.NIP89s: + k_tag = Tag.parse(["k", str(nip89.kind)]) + d_tag = Tag.parse(["d", nip89.dtag]) + keys = Keys.from_sk_str(nip89.pk) + content = nip89.content + event = EventBuilder(31990, content, [k_tag, d_tag]).to_event(keys) + send_event(event, key=keys) + + print("Announced NIP 89") \ No newline at end of file diff --git a/utils/nostr_utils.py b/utils/nostr_utils.py new file mode 100644 index 0000000..1ee2834 --- /dev/null +++ b/utils/nostr_utils.py @@ -0,0 +1,92 @@ +from datetime import timedelta +from nostr_sdk import Keys, Filter, Client, Alphabet, EventId, Options + +from utils.definitions import RELAY_LIST + + +def get_event_by_id(event_id, client=None, config=None): + is_new_client = False + if client is None: + keys = Keys.from_sk_str(config.PRIVATE_KEY) + client = Client(keys) + for relay in config.RELAY_LIST: + client.add_relay(relay) + client.connect() + is_new_client = True + + split = event_id.split(":") + if len(split) == 3: + id_filter = Filter().author(split[1]).custom_tag(Alphabet.D, [split[2]]) + events = client.get_events_of([id_filter], timedelta(seconds=config.RELAY_TIMEOUT)) + else: + id_filter = Filter().id(event_id).limit(1) + events = client.get_events_of([id_filter], timedelta(seconds=config.RELAY_TIMEOUT)) + if is_new_client: + client.disconnect() + if len(events) > 0: + return events[0] + else: + return None + +def get_referenced_event_by_id(event_id, kinds=None, client=None, config=None): + if kinds is None: + kinds = [] + is_new_client = False + if client is None: + keys = Keys.from_sk_str(config.PRIVATE_KEY) + client = Client(keys) + for relay in config.RELAY_LIST: + client.add_relay(relay) + client.connect() + is_new_client = True + if kinds is None: + kinds = [] + if len(kinds) > 0: + job_id_filter = Filter().kinds(kinds).event(EventId.from_hex(event_id)).limit(1) + else: + job_id_filter = Filter().event(EventId.from_hex(event_id)).limit(1) + + events = client.get_events_of([job_id_filter], timedelta(seconds=config.RELAY_TIMEOUT)) + + if is_new_client: + client.disconnect() + if len(events) > 0: + return events[0] + else: + return None + +def send_event(event, client=None, key=None, config=None): + relays = [] + is_new_client = False + + for tag in event.tags(): + if tag.as_vec()[0] == 'relays': + relays = tag.as_vec()[1].split(',') + + if client is None: + print(key.secret_key().to_hex()) + + opts = Options().wait_for_send(False).send_timeout(timedelta(seconds=5)).skip_disconnected_relays(True) + client = Client.with_opts(key, opts) + for relay in RELAY_LIST: + client.add_relay(relay) + + client.connect() + is_new_client = True + + for relay in relays: + if relay not in RELAY_LIST: + client.add_relay(relay) + client.connect() + + + event_id = client.send_event(event) + + for relay in relays: + if relay not in RELAY_LIST: + client.remove_relay(relay) + + if is_new_client: + client.disconnect() + + return event_id \ No newline at end of file diff --git a/utils/output_utils.py b/utils/output_utils.py new file mode 100644 index 0000000..06536f6 --- /dev/null +++ b/utils/output_utils.py @@ -0,0 +1,90 @@ +import json +import datetime as datetime +from types import NoneType + +import pandas + + +def post_process_result(anno, original_event): + print("post-processing...") + if isinstance(anno, pandas.DataFrame): # if input is an anno we parse it to required output format + for tag in original_event.tags(): + print(tag.as_vec()[0]) + if tag.as_vec()[0] == "output": + print("HAS OUTPUT TAG") + output_format = tag.as_vec()[1] + print("requested output is " + str(tag.as_vec()[1]) + "...") + try: + if output_format == "text/plain": + result = "" + for each_row in anno['name']: + if each_row is not None: + for i in str(each_row).split('\n'): + result = result + i + "\n" + result = replace_broken_words( + str(result).replace("\"", "").replace('[', "").replace(']', "").lstrip(None)) + return result + + elif output_format == "text/vtt": + print(str(anno)) + result = "WEBVTT\n\n" + for element in anno: + name = element["name"] # name + start = float(element["from"]) + convertstart = str(datetime.timedelta(seconds=start)) + end = float(element["to"]) + convertend = str(datetime.timedelta(seconds=end)) + print(str(convertstart) + " --> " + str(convertend)) + cleared_name = str(name).lstrip("\'").rstrip("\'") + result = result + str(convertstart) + " --> " + str( + convertend) + "\n" + cleared_name + "\n\n" + result = replace_broken_words( + str(result).replace("\"", "").replace('[', "").replace(']', "").lstrip(None)) + return result + + elif output_format == "text/json" or output_format == "json": + # result = json.dumps(json.loads(anno.data.to_json(orient="records"))) + result = replace_broken_words(json.dumps(anno.data.tolist())) + return result + # TODO add more + else: + result = "" + for element in anno.data: + element["name"] = str(element["name"]).lstrip() + element["from"] = (format(float(element["from"]), '.2f')).lstrip() # name + element["to"] = (format(float(element["to"]), '.2f')).lstrip() # name + result = result + "(" + str(element["from"]) + "," + str(element["to"]) + ")" + " " + str( + element["name"]) + "\n" + + print(result) + result = replace_broken_words(result) + return result + + except Exception as e: + print(e) + result = replace_broken_words(str(anno.data)) + return result + + else: + result = "" + for element in anno.data: + element["name"] = str(element["name"]).lstrip() + element["from"] = (format(float(element["from"]), '.2f')).lstrip() # name + element["to"] = (format(float(element["to"]), '.2f')).lstrip() # name + result = result + "(" + str(element["from"]) + "," + str(element["to"]) + ")" + " " + str( + element["name"]) + "\n" + + print(result) + result = replace_broken_words(result) + return result + elif isinstance(anno, NoneType): + return "An error occurred" + else: + result = replace_broken_words(anno) #TODO + return result + + +def replace_broken_words(text): + result = (text.replace("Noster", "Nostr").replace("Nostra", "Nostr").replace("no stir", "Nostr"). + replace("Nostro", "Nostr").replace("Impub", "npub").replace("sets", "Sats")) + return result diff --git a/utils/requestform_utils.py b/utils/requestform_utils.py new file mode 100644 index 0000000..28d4c9f --- /dev/null +++ b/utils/requestform_utils.py @@ -0,0 +1,65 @@ +import os +import re +from configparser import ConfigParser +import time + +from nostr_sdk import PublicKey + +from utils.definitions import EventDefinitions +from utils.backend_utils import get_task +from utils.nostr_utils import get_referenced_event_by_id, get_event_by_id +from utils.definitions import LOCAL_TASKS +import utils.env as env + + +def create_requestform_from_nostr_event(event, is_bot=False, client=None, dvmconfig=None): + task = get_task(event, client=client, dvmconfig=dvmconfig) + + request_form = {"jobID": event.id().to_hex(), "frameSize": 0, "stride": 0, + "leftContext": 0, "rightContext": 0, + "startTime": "0", "endTime": "0"} + + if task == "translation": + input_type = "event" + text = "" + translation_lang = "en" + for tag in event.tags(): + if tag.as_vec()[0] == 'i': + input_type = tag.as_vec()[2] + + elif tag.as_vec()[0] == 'param': + param = tag.as_vec()[1] + if param == "language": # check for paramtype + translation_lang = str(tag.as_vec()[2]).split('-')[0] + elif param == "lang": # check for paramtype + translation_lang = str(tag.as_vec()[2]).split('-')[0] + + if input_type == "event": + for tag in event.tags(): + if tag.as_vec()[0] == 'i': + evt = get_event_by_id(tag.as_vec()[1], config=dvmconfig) + text = evt.content() + break + + elif input_type == "text": + for tag in event.tags(): + if tag.as_vec()[0] == 'i': + text = tag.as_vec()[1] + break + + elif input_type == "job": + for tag in event.tags(): + if tag.as_vec()[0] == 'i': + evt = get_referenced_event_by_id(tag.as_vec()[1], + [EventDefinitions.KIND_NIP90_RESULT_EXTRACT_TEXT, + EventDefinitions.KIND_NIP90_RESULT_SUMMARIZE_TEXT], + client, + config=dvmconfig) + text = evt.content() + break + + request_form["optStr"] = ('translation_lang=' + translation_lang + ';text=' + + text.replace('\U0001f919', "").replace("=", "equals"). + replace(";", ",")) + + return request_form diff --git a/utils/zap_utils.py b/utils/zap_utils.py new file mode 100644 index 0000000..4923761 --- /dev/null +++ b/utils/zap_utils.py @@ -0,0 +1,96 @@ +# LIGHTNING FUNCTIONS +import json + +import requests +from Crypto.Cipher import AES +from bech32 import bech32_decode, convertbits +from nostr_sdk import PublicKey, nostr_sdk + + +def parse_bolt11_invoice(invoice): + def get_index_of_first_letter(ip): + index = 0 + for c in ip: + if c.isalpha(): + return index + else: + index = index + 1 + return len(ip) + + remaining_invoice = invoice[4:] + index = get_index_of_first_letter(remaining_invoice) + identifier = remaining_invoice[index] + number_string = remaining_invoice[:index] + number = float(number_string) + if identifier == 'm': + number = number * 100000000 * 0.001 + elif identifier == 'u': + number = number * 100000000 * 0.000001 + elif identifier == 'n': + number = number * 100000000 * 0.000000001 + elif identifier == 'p': + number = number * 100000000 * 0.000000000001 + + return int(number) + +def create_bolt11_ln_bits(sats, config): + url = config.LNBITS_URL + "/api/v1/payments" + data = {'out': False, 'amount': sats, 'memo': "Nostr-DVM"} + headers = {'X-API-Key': config.LNBITS_INVOICE_KEY, 'Content-Type': 'application/json', 'charset': 'UTF-8'} + try: + res = requests.post(url, json=data, headers=headers) + obj = json.loads(res.text) + return obj["payment_request"], obj["payment_hash"] + except Exception as e: + print(e) + return None + +def check_bolt11_ln_bits_is_paid(payment_hash, config): + url = config.LNBITS_URL + "/api/v1/payments/" + payment_hash + headers = {'X-API-Key': config.LNBITS_INVOICE_KEY, 'Content-Type': 'application/json', 'charset': 'UTF-8'} + try: + res = requests.get(url, headers=headers) + obj = json.loads(res.text) + return obj["paid"] + except Exception as e: + #print("Exception checking invoice is paid:" + e) + return None + + +# DECRYPT ZAPS +def check_for_zapplepay(sender, content): + try: + # Special case Zapplepay + if sender == PublicKey.from_bech32("npub1wxl6njlcgygduct7jkgzrvyvd9fylj4pqvll6p32h59wyetm5fxqjchcan").to_hex(): + real_sender_bech32 = content.replace("From: nostr:", "") + sender = PublicKey.from_bech32(real_sender_bech32).to_hex() + return sender + + except Exception as e: + print(e) + return sender + + +def decrypt_private_zap_message(msg, privkey, pubkey): + shared_secret = nostr_sdk.generate_shared_key(privkey, pubkey) + if len(shared_secret) != 16 and len(shared_secret) != 32: + return "invalid shared secret size" + parts = msg.split("_") + if len(parts) != 2: + return "invalid message format" + try: + _, encrypted_msg = bech32_decode(parts[0]) + encrypted_bytes = convertbits(encrypted_msg, 5, 8, False) + _, iv = bech32_decode(parts[1]) + iv_bytes = convertbits(iv, 5, 8, False) + except Exception as e: + return e + try: + cipher = AES.new(bytearray(shared_secret), AES.MODE_CBC, bytearray(iv_bytes)) + decrypted_bytes = cipher.decrypt(bytearray(encrypted_bytes)) + plaintext = decrypted_bytes.decode("utf-8") + decoded = plaintext.rsplit("}", 1)[0] + "}" # weird symbols at the end + return decoded + except Exception as ex: + return str(ex) +