From 215916c1ef05939b8b72e2a0bbe0ba4202472941 Mon Sep 17 00:00:00 2001 From: Believethehype Date: Fri, 24 Nov 2023 17:20:29 +0100 Subject: [PATCH] added dall-e, reworked bot, added nip89config --- .env_example | 3 + .idea/dataSources.xml | 36 +++++ backends/nova_server.py | 2 +- bot.py | 139 +++++++++++++----- dvm.py | 11 +- interfaces/dvmtaskinterface.py | 17 +-- main.py | 20 ++- playground.py | 80 ++++++++-- tasks/imagegeneration_openai_dalle.py | 112 ++++++++++++++ ...erationsdxl.py => imagegeneration_sdxl.py} | 10 +- tasks/textextractionpdf.py | 5 +- tasks/translation.py | 5 +- test_dvm_client.py | 4 +- utils/backend_utils.py | 2 +- utils/database_utils.py | 58 ++++++-- utils/nip89_utils.py | 5 + 16 files changed, 410 insertions(+), 99 deletions(-) create mode 100644 tasks/imagegeneration_openai_dalle.py rename tasks/{imagegenerationsdxl.py => imagegeneration_sdxl.py} (95%) diff --git a/.env_example b/.env_example index b88e846..6468622 100644 --- a/.env_example +++ b/.env_example @@ -20,8 +20,11 @@ TASK_TEXT_EXTRACTION_NIP89_DTAG = "asdd" TASK_TRANSLATION_NIP89_DTAG = "abcded" TASK_IMAGE_GENERATION_NIP89_DTAG = "fgdfgdf" TASK_IMAGE_GENERATION_NIP89_DTAG2 = "fdgdfg" +TASK_IMAGE_GENERATION_NIP89_DTAG3 = "asdasd" + #Backend Specific Options for tasks that require them #nova-server is a local backend supporting some AI modules and needs to be installed separately, #if dvms supporting it should be used +OPENAI_API_KEY = "your-openai-api-key" NOVA_SERVER = "127.0.0.1:37318" \ No newline at end of file diff --git a/.idea/dataSources.xml b/.idea/dataSources.xml index 211f211..a9eee01 100644 --- a/.idea/dataSources.xml +++ b/.idea/dataSources.xml @@ -8,5 +8,41 @@ jdbc:sqlite:$PROJECT_DIR$/nostrzaps.db $ProjectFileDir$ + + sqlite.xerial + true + org.sqlite.JDBC + jdbc:sqlite:$PROJECT_DIR$/db/bot.db + $ProjectFileDir$ + + + file://$APPLICATION_CONFIG_DIR$/jdbc-drivers/Xerial SQLiteJDBC/3.43.0/org/xerial/sqlite-jdbc/3.43.0.0/sqlite-jdbc-3.43.0.0.jar + + + + + sqlite.xerial + true + org.sqlite.JDBC + jdbc:sqlite:$PROJECT_DIR$/db/Dall-E 3.db + $ProjectFileDir$ + + + file://$APPLICATION_CONFIG_DIR$/jdbc-drivers/Xerial SQLiteJDBC/3.43.0/org/xerial/sqlite-jdbc/3.43.0.0/sqlite-jdbc-3.43.0.0.jar + + + + + sqlite.xerial + true + org.sqlite.JDBC + jdbc:sqlite:$PROJECT_DIR$/db/Translator.db + $ProjectFileDir$ + + + file://$APPLICATION_CONFIG_DIR$/jdbc-drivers/Xerial SQLiteJDBC/3.43.0/org/xerial/sqlite-jdbc/3.43.0.0/sqlite-jdbc-3.43.0.0.jar + + + \ No newline at end of file diff --git a/backends/nova_server.py b/backends/nova_server.py index 885e352..a961082 100644 --- a/backends/nova_server.py +++ b/backends/nova_server.py @@ -71,7 +71,7 @@ def check_nova_server_status(jobID, address): result = "" url_fetch = 'http://' + address + '/fetch_result' print("Fetching Results from NOVA-Server...") - data = {"jobID": jobID} + data = {"jobID": jobID, "delete_after_download": True} response = requests.post(url_fetch, headers=headers, data=data) content_type = response.headers['content-type'] print("Content-type: " + str(content_type)) diff --git a/bot.py b/bot.py index 60d49b9..493415f 100644 --- a/bot.py +++ b/bot.py @@ -1,22 +1,22 @@ import json import time from datetime import timedelta -from threading import Thread from nostr_sdk import Keys, Client, Timestamp, Filter, nip04_decrypt, HandleNotification, EventBuilder, PublicKey, \ Event, Options from utils.admin_utils import admin_make_database_updates -from utils.database_utils import get_or_add_user, update_user_balance, create_sql_table +from utils.backend_utils import get_amount_per_task +from utils.database_utils import get_or_add_user, update_user_balance, create_sql_table, update_sql_table, User from utils.definitions import EventDefinitions from utils.nostr_utils import send_event, get_event_by_id from utils.zap_utils import parse_amount_from_bolt11_invoice, check_for_zapplepay, decrypt_private_zap_message class Bot: - job_list: list - def __init__(self, dvm_config, admin_config=None): + self.NAME = "Bot" + dvm_config.DB = "db/" + self.NAME + ".db" self.dvm_config = dvm_config self.admin_config = admin_config self.keys = Keys.from_sk_str(dvm_config.PRIVATE_KEY) @@ -24,23 +24,22 @@ class Bot: skip_disconnected_relays = True opts = (Options().wait_for_send(wait_for_send).send_timeout(timedelta(seconds=self.dvm_config.RELAY_TIMEOUT)) .skip_disconnected_relays(skip_disconnected_relays)) - self.client = Client.with_opts(self.keys, opts) - self.job_list = [] pk = self.keys.public_key() - self.dvm_config.DB = "db/bot.db" - print("Nostr BOT public key: " + str(pk.to_bech32()) + " Hex: " + str(pk.to_hex()) + " Supported DVM tasks: " + + print("Nostr BOT public key: " + str(pk.to_bech32()) + " Hex: " + str(pk.to_hex()) + " Name: " + self.NAME + " Supported DVM tasks: " + ', '.join(p.NAME + ":" + p.TASK for p in self.dvm_config.SUPPORTED_DVMS) + "\n") for relay in self.dvm_config.RELAY_LIST: self.client.add_relay(relay) self.client.connect() - dm_zap_filter = Filter().pubkey(pk).kinds([EventDefinitions.KIND_DM, EventDefinitions.KIND_ZAP]).since( + zap_filter = Filter().pubkey(pk).kinds([EventDefinitions.KIND_ZAP]).since(Timestamp.now()) + dm_filter = Filter().pubkey(pk).kinds([EventDefinitions.KIND_DM]).since( Timestamp.now()) - self.client.subscribe([dm_zap_filter]) + + self.client.subscribe([zap_filter, dm_filter]) create_sql_table(self.dvm_config.DB) admin_make_database_updates(adminconfig=self.admin_config, dvmconfig=self.dvm_config, client=self.client) @@ -52,59 +51,127 @@ class Bot: def handle(self, relay_url, nostr_event): if EventDefinitions.KIND_DM: - print( - "[Bot] " + f"Received new DM from {relay_url}: {nostr_event.as_json()}") handle_dm(nostr_event) elif nostr_event.kind() == EventDefinitions.KIND_ZAP: - print("yay zap") handle_zap(nostr_event) def handle_msg(self, relay_url, msg): return def handle_dm(nostr_event): + sender = nostr_event.pubkey().to_hex() + try: decrypted_text = nip04_decrypt(self.keys.secret_key(), nostr_event.pubkey(), nostr_event.content()) - # TODO more advanced logic, more parsing, just very basic test functions for now + user = get_or_add_user(db=self.dvm_config.DB, npub=sender, client=self.client) + + # user = User + # user.npub = sender + # user.balance = 250 + # user.iswhitelisted = False + # user.isblacklisted = False + # user.name = "Test" + # user.nip05 = "Test@test" + # user.lud16 = "Test@test" + + # We do a selection of tasks now, maybe change this later, Idk. if decrypted_text[0].isdigit(): index = int(decrypted_text.split(' ')[0]) - 1 - i_tag = decrypted_text.replace(decrypted_text.split(' ')[0] + " ", "") + task = self.dvm_config.SUPPORTED_DVMS[index].TASK + print("["+ self.NAME + "] Request from " + str(user.name) + " (" + str(user.nip05) + ", Balance: "+ str(user.balance)+ " Sats) Task: " + str(task)) - keys = Keys.from_sk_str(self.dvm_config.SUPPORTED_DVMS[index].PK) - params = { - "sender": nostr_event.pubkey().to_hex(), - "input": i_tag, - "task": self.dvm_config.SUPPORTED_DVMS[index].TASK - } - message = json.dumps(params) - evt = EventBuilder.new_encrypted_direct_msg(self.keys, keys.public_key(), - message, None).to_event(self.keys) - send_event(evt, client=self.client, dvm_config=dvm_config) + required_amount = self.dvm_config.SUPPORTED_DVMS[index].COST + if user.isblacklisted: + # For some reason an admin might blacklist npubs, e.g. for abusing the service + evt = EventBuilder.new_encrypted_direct_msg(self.keys, nostr_event.pubkey(), + "Your are currently blocked from all services.", + None).to_event(self.keys) + send_event(evt, client=self.client, dvm_config=dvm_config) + + elif user.iswhitelisted or user.balance >= required_amount or required_amount == 0: + if not user.iswhitelisted: + + balance = max(user.balance - required_amount, 0) + update_sql_table(db=self.dvm_config.DB, npub=user.npub, balance=balance, + iswhitelisted=user.iswhitelisted, isblacklisted=user.isblacklisted, + nip05=user.nip05, lud16=user.lud16, name=user.name, + lastactive=Timestamp.now().as_secs()) + + evt = EventBuilder.new_encrypted_direct_msg(self.keys, nostr_event.pubkey(), + "Your Job is now scheduled. New balance is " + + str(balance) + + " Sats.\nI will DM you once I'm done " + "processing.", + nostr_event.id()).to_event(self.keys) + else: + evt = EventBuilder.new_encrypted_direct_msg(self.keys, nostr_event.pubkey(), + "Your Job is now scheduled. As you are " + "whitelisted, your balance remains at" + + str(user.balance) + " Sats.\n" + "I will DM you once I'm " + "done processing.", + nostr_event.id()).to_event(self.keys) + + print("[" + self.NAME + "] Replying " + user.name + " with \"scheduled\" confirmation") + time.sleep(2.0) + send_event(evt, client=self.client, dvm_config=dvm_config) + + i_tag = decrypted_text.replace(decrypted_text.split(' ')[0] + " ", "") + # TODO more advanced logic, more parsing, params etc, just very basic test functions for now + dvm_keys = Keys.from_sk_str(self.dvm_config.SUPPORTED_DVMS[index].PK) + params = { + "sender": nostr_event.pubkey().to_hex(), + "input": i_tag, + "task": self.dvm_config.SUPPORTED_DVMS[index].TASK + } + message = json.dumps(params) + evt = EventBuilder.new_encrypted_direct_msg(self.keys, dvm_keys.public_key(), + message, None).to_event(self.keys) + print("[" + self.NAME + "] Forwarding task " + self.dvm_config.SUPPORTED_DVMS[index].TASK + + " for user " + user.name + " to " + self.dvm_config.SUPPORTED_DVMS[index].NAME) + send_event(evt, client=self.client, dvm_config=dvm_config) + else: + print("payment-required") + time.sleep(2.0) + evt = EventBuilder.new_encrypted_direct_msg(self.keys, nostr_event.pubkey(), + "Balance required, please zap me with at least " + str( + int(required_amount - user.balance)) + + " Sats, then try again.", + nostr_event.id()).to_event(self.keys) + time.sleep(2.0) + send_event(evt, client=self.client, dvm_config=dvm_config) + + + # TODO if we receive the result from one of the dvms, need some better management, maybe check for keys elif decrypted_text.startswith('{"result":'): + dvm_result = json.loads(decrypted_text) - # user = get_or_add_user(db=self.dvm_config.DB, npub=dvm_result["sender"], client=self.client) - # print("BOT received and forwarded to " + user.name + ": " + str(decrypted_text)) + user_npub_hex = dvm_result["sender"] + user = get_or_add_user(db=self.dvm_config.DB, npub=user_npub_hex, client=self.client) + print("[" + self.NAME + "] Received results, message to orignal sender " + user.name) reply_event = EventBuilder.new_encrypted_direct_msg(self.keys, - PublicKey.from_hex(dvm_result["sender"]), + PublicKey.from_hex(user.npub), dvm_result["result"], None).to_event(self.keys) + time.sleep(2.0) send_event(reply_event, client=self.client, dvm_config=dvm_config) - - else: + print("Message from " + user.name + ": " + decrypted_text) message = "DVMs that I support:\n\n" index = 1 for p in self.dvm_config.SUPPORTED_DVMS: - message += str(index) + " " + p.NAME + " " + p.TASK + "\n" + message += str(index) + " " + p.NAME + " " + p.TASK + " " + str(p.COST) + " Sats" + "\n" index += 1 evt = EventBuilder.new_encrypted_direct_msg(self.keys, nostr_event.pubkey(), message + "\nSelect an Index and provide an input (" "e.g. 1 A purple ostrich)", - nostr_event.id()).to_event(self.keys) + None).to_event(self.keys) + #nostr_event.id()).to_event(self.keys) + time.sleep(3) send_event(evt, client=self.client, dvm_config=dvm_config) except Exception as e: print(e) @@ -126,11 +193,11 @@ class Bot: zap_request_event = Event.from_json(tag.as_vec()[1]) sender = check_for_zapplepay(zap_request_event.pubkey().to_hex(), zap_request_event.content()) - for ztag in zap_request_event.tags(): - if ztag.as_vec()[0] == 'anon': - if len(ztag.as_vec()) > 1: + for z_tag in zap_request_event.tags(): + if z_tag.as_vec()[0] == 'anon': + if len(z_tag.as_vec()) > 1: print("Private Zap received.") - decrypted_content = decrypt_private_zap_message(ztag.as_vec()[1], + decrypted_content = decrypt_private_zap_message(z_tag.as_vec()[1], self.keys.secret_key(), zap_request_event.pubkey()) decrypted_private_event = Event.from_json(decrypted_content) diff --git a/dvm.py b/dvm.py index b392181..9c0db4b 100644 --- a/dvm.py +++ b/dvm.py @@ -240,10 +240,6 @@ class DVM: print(decrypted_text) if str(ob['input']).startswith("http"): input_type = "url" - #elif str(ob['input']).startswith("nostr:nevent"): - # ob['input'] = str(ob['input']).replace("nostr:", "") - # ob['input'] = EventId.from_bech32(ob['input']).to_hex() - # input_type = "event" j_tag = Tag.parse(["j", self.dvm_config.SUPPORTED_DVMS[0].TASK]) i_tag = Tag.parse(["i", ob['input'], input_type]) @@ -370,12 +366,11 @@ class DVM: elif tag.as_vec()[0] == "i": task = tag.as_vec()[1] - user = get_from_sql_table(self.dvm_config.DB, sender) + user = get_or_add_user(self.dvm_config.DB, sender, self.client) if not user.iswhitelisted: amount = int(user.balance) + get_amount_per_task(task, self.dvm_config) - update_sql_table(self.dvm_config.DB, sender, amount, user.iswhitelisted, user.isblacklisted, user.nip05, user.lud16, - user.name, - Timestamp.now().as_secs()) + update_sql_table(self.dvm_config.DB, sender, amount, user.iswhitelisted, user.isblacklisted, + user.nip05, user.lud16, user.name, Timestamp.now().as_secs()) message = "There was the following error : " + content + ". Credits have been reimbursed" else: # User didn't pay, so no reimbursement diff --git a/interfaces/dvmtaskinterface.py b/interfaces/dvmtaskinterface.py index 32efa2c..0c4f0a3 100644 --- a/interfaces/dvmtaskinterface.py +++ b/interfaces/dvmtaskinterface.py @@ -3,7 +3,7 @@ from threading import Thread from utils.admin_utils import AdminConfig from utils.dvmconfig import DVMConfig -from utils.nip89_utils import NIP89Announcement +from utils.nip89_utils import NIP89Announcement, NIP89Config from dvm import DVM @@ -17,19 +17,20 @@ class DVMTaskInterface: dvm_config: DVMConfig admin_config: AdminConfig - def NIP89_announcement(self, d_tag, content): + def NIP89_announcement(self, nip89config: NIP89Config): nip89 = NIP89Announcement() nip89.name = self.NAME nip89.kind = self.KIND nip89.pk = self.PK - nip89.dtag = d_tag - nip89.content = content + nip89.dtag = nip89config.DTAG + nip89.content = nip89config.CONTENT return nip89 def run(self): nostr_dvm_thread = Thread(target=self.DVM, args=[self.dvm_config, self.admin_config]) nostr_dvm_thread.start() + def is_input_supported(self, input_type, input_content) -> bool: """Check if input is supported for current Task.""" pass @@ -50,12 +51,4 @@ class DVMTaskInterface: opts = json.loads(request_form["options"]) print(opts) - # old format, deprecated, will remove - elif request_form.get("optStr"): - opts = [] - for k, v in [option.split("=") for option in request_form["optStr"].split(";")]: - t = (k, v) - opts.append(t) - print("...done.") - return dict(opts) diff --git a/main.py b/main.py index d968c40..718b06d 100644 --- a/main.py +++ b/main.py @@ -1,4 +1,7 @@ import os +import signal +import sys +import time from pathlib import Path from threading import Thread @@ -6,7 +9,7 @@ import dotenv from nostr_sdk import Keys from bot import Bot -from playground import build_pdf_extractor, build_translator, build_unstable_diffusion, build_sketcher +from playground import build_pdf_extractor, build_translator, build_unstable_diffusion, build_sketcher, build_dalle from utils.dvmconfig import DVMConfig @@ -34,6 +37,9 @@ def run_nostr_dvm_with_local_config(): sketcher = build_sketcher("Sketcher", [bot_publickey]) sketcher.run() + dalle = build_dalle("Dall-E 3", [bot_publickey]) + dalle.run() + # We will run an optional bot that can communicate with the DVMs # Note this is very basic for now and still under development bot_config = DVMConfig() @@ -41,20 +47,26 @@ def run_nostr_dvm_with_local_config(): bot_config.LNBITS_INVOICE_KEY = os.getenv("LNBITS_INVOICE_KEY") bot_config.LNBITS_URL = os.getenv("LNBITS_HOST") # Finally we add some of the DVMs we created before to the Bot and start it. - bot_config.SUPPORTED_DVMS = [sketcher, unstable_artist, translator] + bot_config.SUPPORTED_DVMS = [sketcher, unstable_artist, dalle, translator] bot = Bot nostr_dvm_thread = Thread(target=bot, args=[bot_config]) nostr_dvm_thread.start() + # Keep the main function alive for libraries like openai + try: + while True: + time.sleep(10) + except KeyboardInterrupt: + print('Stay weird!') + os.kill(os.getpid(), signal.SIGKILL) + if __name__ == '__main__': - env_path = Path('.env') if env_path.is_file(): print(f'loading environment from {env_path.resolve()}') dotenv.load_dotenv(env_path, verbose=True, override=True) else: raise FileNotFoundError(f'.env file not found at {env_path} ') - run_nostr_dvm_with_local_config() diff --git a/playground.py b/playground.py index 6dfab48..d3978c1 100644 --- a/playground.py +++ b/playground.py @@ -1,11 +1,32 @@ import json import os -from tasks.imagegenerationsdxl import ImageGenerationSDXL +from tasks.imagegeneration_openai_dalle import ImageGenerationDALLE +from tasks.imagegeneration_sdxl import ImageGenerationSDXL from tasks.textextractionpdf import TextExtractionPDF from tasks.translation import Translation from utils.admin_utils import AdminConfig from utils.dvmconfig import DVMConfig +from utils.nip89_utils import NIP89Config +""" +This File is a playground to create DVMs. It shows some examples of DVMs that make use of the modules in the tasks folder +These DVMs should be considered examples and will be extended in the future. env variables are used to not commit keys, +but if used privatley, these can also be directly filled in this file. The main.py function calls some of the functions +defined here and starts the DVMs. + +Note that the admin_config is optional, and if given commands as defined in admin_utils will be called at start of the +DVM. For example the NIP89 event can be rebroadcasted (store the d_tag somewhere). + +DM_ALLOWED is used to tell the DVM to which npubs it should listen to. We use this here to listen to our bot, +as defined in main.py to perform jobs on it's behalf and reply. + +if LNBITS_INVOICE_KEY is not set (=""), the DVM is still zappable but a lud16 address in required in the profile. + +additional options can be set, for example to preinitalize vaiables or give parameters that are required to perform a +task, for example an address or an API key. + + +""" # Generate an optional Admin Config, in this case, whenever we give our DVMs this config, they will (re)broadcast # their NIP89 announcement @@ -20,7 +41,6 @@ def build_pdf_extractor(name, dm_allowed_keys): dvm_config.LNBITS_URL = os.getenv("LNBITS_HOST") dvm_config.DM_ALLOWED = dm_allowed_keys # Add NIP89 - d_tag = os.getenv("TASK_TEXT_EXTRACTION_NIP89_DTAG") nip90params = {} nip89info = { "name": name, @@ -28,7 +48,11 @@ def build_pdf_extractor(name, dm_allowed_keys): "about": "I extract text from pdf documents", "nip90Params": nip90params } - return TextExtractionPDF(name=name, dvm_config=dvm_config, nip89d_tag=d_tag, nip89info=json.dumps(nip89info), + + nip89config = NIP89Config() + nip89config.DTAG = os.getenv("TASK_TEXT_EXTRACTION_NIP89_DTAG") + nip89config.CONTENT = json.dumps(nip89info) + return TextExtractionPDF(name=name, dvm_config=dvm_config, nip89config=nip89config, admin_config=admin_config) @@ -39,7 +63,6 @@ def build_translator(name, dm_allowed_keys): dvm_config.LNBITS_URL = os.getenv("LNBITS_HOST") dvm_config.DM_ALLOWED = dm_allowed_keys - d_tag = os.getenv("TASK_TRANSLATION_NIP89_DTAG") nip90params = { "language": { "required": False, @@ -59,7 +82,10 @@ def build_translator(name, dm_allowed_keys): "input into the language defined in params.", "nip90Params": nip90params } - return Translation(name=name, dvm_config=dvm_config, nip89d_tag=d_tag, nip89info=json.dumps(nip89info), + nip89config = NIP89Config() + nip89config.DTAG = os.getenv("TASK_TRANSLATION_NIP89_DTAG") + nip89config.CONTENT = json.dumps(nip89info) + return Translation(name=name, dvm_config=dvm_config, nip89config=nip89config, admin_config=admin_config) @@ -74,7 +100,6 @@ def build_unstable_diffusion(name, dm_allowed_keys): # address it should use. These parameters can be freely defined in the task component options = {'default_model': "unstable", 'nova_server': os.getenv("NOVA_SERVER")} - d_tag = os.getenv("TASK_IMAGE_GENERATION_NIP89_DTAG") nip90params = { "negative_prompt": { "required": False, @@ -91,7 +116,10 @@ def build_unstable_diffusion(name, dm_allowed_keys): "about": "I draw images based on a prompt with a Model called unstable diffusion", "nip90Params": nip90params } - return ImageGenerationSDXL(name=name, dvm_config=dvm_config, nip89d_tag=d_tag, nip89info=json.dumps(nip89info), + nip89config = NIP89Config() + nip89config.DTAG = os.getenv("TASK_IMAGE_GENERATION_NIP89_DTAG") + nip89config.CONTENT = json.dumps(nip89info) + return ImageGenerationSDXL(name=name, dvm_config=dvm_config, nip89config=nip89config, admin_config=admin_config, options=options) @@ -102,7 +130,6 @@ def build_sketcher(name, dm_allowed_keys): dvm_config.LNBITS_URL = os.getenv("LNBITS_HOST") dvm_config.DM_ALLOWED = dm_allowed_keys - d_tag = os.getenv("TASK_IMAGE_GENERATION_NIP89_DTAG2") nip90params = { "negative_prompt": { "required": False, @@ -124,6 +151,39 @@ def build_sketcher(name, dm_allowed_keys): # address it should use. These parameters can be freely defined in the task component options = {'default_model': "mohawk", 'default_lora': "timburton", 'nova_server': os.getenv("NOVA_SERVER")} + nip89config = NIP89Config() + nip89config.DTAG = os.getenv("TASK_IMAGE_GENERATION_NIP89_DTAG2") + nip89config.CONTENT = json.dumps(nip89info) # We add an optional AdminConfig for this one, and tell the dvm to rebroadcast its NIP89 - return ImageGenerationSDXL(name=name, dvm_config=dvm_config, nip89d_tag=d_tag, nip89info=json.dumps(nip89info), - admin_config=admin_config, options=options) \ No newline at end of file + return ImageGenerationSDXL(name=name, dvm_config=dvm_config, nip89config=nip89config, + admin_config=admin_config, options=options) + +def build_dalle(name, dm_allowed_keys): + dvm_config = DVMConfig() + dvm_config.PRIVATE_KEY = os.getenv("NOSTR_PRIVATE_KEY3") + dvm_config.LNBITS_INVOICE_KEY = os.getenv("LNBITS_INVOICE_KEY") + dvm_config.LNBITS_URL = os.getenv("LNBITS_HOST") + dvm_config.DM_ALLOWED = dm_allowed_keys + + nip90params = { + "size": { + "required": False, + "values": ["1024:1024", "1024x1792", "1792x1024"] + } + } + nip89info = { + "name": name, + "image": "https://image.nostr.build/c33ca6fc4cc038ca4adb46fdfdfda34951656f87ee364ef59095bae1495ce669.jpg", + "about": "I use OpenAI's DALL·E 3", + "nip90Params": nip90params + } + + # A module might have options it can be initialized with, here we set a default model, lora and the nova-server + # address it should use. These parameters can be freely defined in the task component + + nip89config = NIP89Config() + nip89config.DTAG = os.getenv("TASK_IMAGE_GENERATION_NIP89_DTAG3") + nip89config.CONTENT = json.dumps(nip89info) + # We add an optional AdminConfig for this one, and tell the dvm to rebroadcast its NIP89 + return ImageGenerationDALLE(name=name, dvm_config=dvm_config, nip89config=nip89config, + admin_config=admin_config) \ No newline at end of file diff --git a/tasks/imagegeneration_openai_dalle.py b/tasks/imagegeneration_openai_dalle.py new file mode 100644 index 0000000..8d81aee --- /dev/null +++ b/tasks/imagegeneration_openai_dalle.py @@ -0,0 +1,112 @@ +import json +import os +import time +from multiprocessing.pool import ThreadPool +from threading import Thread + +from backends.nova_server import check_nova_server_status, send_request_to_nova_server +from dvm import DVM +from interfaces.dvmtaskinterface import DVMTaskInterface +from utils.admin_utils import AdminConfig +from utils.definitions import EventDefinitions +from utils.dvmconfig import DVMConfig +from utils.nip89_utils import NIP89Config + +""" +This File contains a Module to transform Text input on NOVA-Server and receive results back. + +Accepted Inputs: Prompt (text) +Outputs: An url to an Image +Params: -model # models: juggernaut, dynavision, colossusProject, newreality, unstable + -lora # loras (weights on top of models) voxel, +""" + + +class ImageGenerationDALLE(DVMTaskInterface): + NAME: str = "" + KIND: int = EventDefinitions.KIND_NIP90_GENERATE_IMAGE + TASK: str = "text-to-image" + COST: int = 120 + PK: str + DVM = DVM + + def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config, admin_config: AdminConfig = None, + options=None): + self.NAME = name + self.PK = dvm_config.PRIVATE_KEY + + dvm_config.SUPPORTED_DVMS = [self] + dvm_config.DB = "db/" + self.NAME + ".db" + dvm_config.NIP89 = self.NIP89_announcement(nip89config) + self.dvm_config = dvm_config + self.admin_config = admin_config + self.options = options + + def is_input_supported(self, input_type, input_content): + if input_type != "text": + return False + return True + + def create_request_form_from_nostr_event(self, event, client=None, dvm_config=None): + request_form = {"jobID": event.id().to_hex() + "_" + self.NAME.replace(" ", "")} + prompt = "" + width = "1024" + height = "1024" + model = "dall-e-3" + quality = "standard" + + for tag in event.tags(): + if tag.as_vec()[0] == 'i': + input_type = tag.as_vec()[2] + if input_type == "text": + prompt = tag.as_vec()[1] + + elif tag.as_vec()[0] == 'param': + print("Param: " + tag.as_vec()[1] + ": " + tag.as_vec()[2]) + if tag.as_vec()[1] == "size": + if len(tag.as_vec()) > 3: + width = (tag.as_vec()[2]) + height = (tag.as_vec()[3]) + elif len(tag.as_vec()) == 3: + split = tag.as_vec()[2].split("x") + if len(split) > 1: + width = split[0] + height = split[1] + elif tag.as_vec()[1] == "model": + model = tag.as_vec()[2] + elif tag.as_vec()[1] == "quality": + quality = tag.as_vec()[2] + + options = { + "prompt": prompt, + "size": width + "x" + height, + "model": model, + "quality": quality, + "number": 1 + } + request_form['options'] = json.dumps(options) + + return request_form + + def process(self, request_form): + try: + options = DVMTaskInterface.set_options(request_form) + + from openai import OpenAI + client = OpenAI() + print("Job " + request_form['jobID'] + " sent to OpenAI API..") + + response = client.images.generate( + model=options['model'], + prompt=options['prompt'], + size=options['size'], + quality=options['quality'], + n=int(options['number']), + ) + + image_url = response.data[0].url + return image_url + + except Exception as e: + print("Error in Module") + raise Exception(e) diff --git a/tasks/imagegenerationsdxl.py b/tasks/imagegeneration_sdxl.py similarity index 95% rename from tasks/imagegenerationsdxl.py rename to tasks/imagegeneration_sdxl.py index 24e6027..3ad712e 100644 --- a/tasks/imagegenerationsdxl.py +++ b/tasks/imagegeneration_sdxl.py @@ -1,7 +1,5 @@ import json -import os from multiprocessing.pool import ThreadPool -from threading import Thread from backends.nova_server import check_nova_server_status, send_request_to_nova_server from dvm import DVM @@ -9,6 +7,7 @@ from interfaces.dvmtaskinterface import DVMTaskInterface from utils.admin_utils import AdminConfig from utils.definitions import EventDefinitions from utils.dvmconfig import DVMConfig +from utils.nip89_utils import NIP89Config """ This File contains a Module to transform Text input on NOVA-Server and receive results back. @@ -28,13 +27,13 @@ class ImageGenerationSDXL(DVMTaskInterface): PK: str DVM = DVM - def __init__(self, name, dvm_config: DVMConfig, nip89d_tag: str, nip89info: str, admin_config: AdminConfig = None, options=None): + def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config, admin_config: AdminConfig = None, options=None): self.NAME = name self.PK = dvm_config.PRIVATE_KEY dvm_config.SUPPORTED_DVMS = [self] dvm_config.DB = "db/" + self.NAME + ".db" - dvm_config.NIP89 = self.NIP89_announcement(nip89d_tag, nip89info) + dvm_config.NIP89 = self.NIP89_announcement(nip89config) self.dvm_config = dvm_config self.admin_config = admin_config self.options = options @@ -94,7 +93,6 @@ class ImageGenerationSDXL(DVMTaskInterface): ratio_height = split[1] # if size is set it will overwrite ratio. elif tag.as_vec()[1] == "size": - if len(tag.as_vec()) > 3: width = (tag.as_vec()[2]) height = (tag.as_vec()[3]) @@ -150,7 +148,7 @@ class ImageGenerationSDXL(DVMTaskInterface): # Call the process route of NOVA-Server with our request form. response = send_request_to_nova_server(request_form, self.options['nova_server']) if bool(json.loads(response)['success']): - print("Job " + request_form['jobID'] + " sent to nova-server") + print("Job " + request_form['jobID'] + " sent to NOVA-server") pool = ThreadPool(processes=1) thread = pool.apply_async(check_nova_server_status, (request_form['jobID'], self.options['nova_server'])) diff --git a/tasks/textextractionpdf.py b/tasks/textextractionpdf.py index 3e4453a..59b6c52 100644 --- a/tasks/textextractionpdf.py +++ b/tasks/textextractionpdf.py @@ -8,6 +8,7 @@ from interfaces.dvmtaskinterface import DVMTaskInterface from utils.admin_utils import AdminConfig from utils.definitions import EventDefinitions from utils.dvmconfig import DVMConfig +from utils.nip89_utils import NIP89Config from utils.nostr_utils import get_event_by_id """ @@ -27,13 +28,13 @@ class TextExtractionPDF(DVMTaskInterface): PK: str DVM = DVM - def __init__(self, name, dvm_config: DVMConfig, nip89d_tag: str, nip89info: str, admin_config: AdminConfig = None): + def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config, admin_config: AdminConfig = None): self.NAME = name self.PK = dvm_config.PRIVATE_KEY dvm_config.SUPPORTED_DVMS = [self] dvm_config.DB = "db/" + self.NAME + ".db" - dvm_config.NIP89 = self.NIP89_announcement(nip89d_tag, nip89info) + dvm_config.NIP89 = self.NIP89_announcement(nip89config) self.dvm_config = dvm_config self.admin_config = admin_config diff --git a/tasks/translation.py b/tasks/translation.py index 331d984..08d5cc1 100644 --- a/tasks/translation.py +++ b/tasks/translation.py @@ -6,6 +6,7 @@ from interfaces.dvmtaskinterface import DVMTaskInterface from utils.admin_utils import AdminConfig from utils.definitions import EventDefinitions from utils.dvmconfig import DVMConfig +from utils.nip89_utils import NIP89Config from utils.nostr_utils import get_referenced_event_by_id, get_event_by_id """ @@ -25,13 +26,13 @@ class Translation(DVMTaskInterface): PK: str DVM = DVM - def __init__(self, name, dvm_config: DVMConfig, nip89d_tag: str, nip89info: str, admin_config: AdminConfig = None): + def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config, admin_config: AdminConfig = None): self.NAME = name self.PK = dvm_config.PRIVATE_KEY dvm_config.SUPPORTED_DVMS = [self] dvm_config.DB = "db/" + self.NAME + ".db" - dvm_config.NIP89 = self.NIP89_announcement(nip89d_tag, nip89info) + dvm_config.NIP89 = self.NIP89_announcement(nip89config) self.dvm_config = dvm_config self.admin_config = admin_config diff --git a/test_dvm_client.py b/test_dvm_client.py index 8455a28..55d15a7 100644 --- a/test_dvm_client.py +++ b/test_dvm_client.py @@ -84,11 +84,11 @@ def nostr_client(): EventDefinitions.KIND_FEEDBACK]).since(Timestamp.now())) # public events client.subscribe([dm_zap_filter, dvm_filter]) - #nostr_client_test_translation("This is the result of the DVM in spanish", "text", "es", 20, 20) + nostr_client_test_translation("This is the result of the DVM in spanish", "text", "es", 20, 20) #nostr_client_test_translation("note1p8cx2dz5ss5gnk7c59zjydcncx6a754c0hsyakjvnw8xwlm5hymsnc23rs", "event", "es", 20,20) #nostr_client_test_translation("44a0a8b395ade39d46b9d20038b3f0c8a11168e67c442e3ece95e4a1703e2beb", "event", "zh", 20, 20) - nostr_client_test_image("a beautiful purple ostrich watching the sunset") + #nostr_client_test_image("a beautiful purple ostrich watching the sunset") class NotificationHandler(HandleNotification): def handle(self, relay_url, event): print(f"Received new event from {relay_url}: {event.as_json()}") diff --git a/utils/backend_utils.py b/utils/backend_utils.py index 22617d0..b9b1281 100644 --- a/utils/backend_utils.py +++ b/utils/backend_utils.py @@ -123,5 +123,5 @@ def get_amount_per_task(task, dvm_config, duration=1): amount = dvm.COST * duration return amount else: - print("[Nostr] Task " + task + " is currently not supported by this instance, skipping") + print("["+dvm_config.SUPPORTED_DVMS[0].NAME +"] Task " + task + " is currently not supported by this instance, skipping") return None diff --git a/utils/database_utils.py b/utils/database_utils.py index ebbc58d..4c13f04 100644 --- a/utils/database_utils.py +++ b/utils/database_utils.py @@ -1,4 +1,5 @@ # DATABASE LOGIC +import json import sqlite3 import time @@ -7,7 +8,7 @@ from dataclasses import dataclass from datetime import timedelta from logging import Filter -from nostr_sdk import Timestamp, Keys, PublicKey, EventBuilder, Metadata, Filter +from nostr_sdk import Timestamp, Keys, PublicKey, EventBuilder, Metadata, Filter, Options, Client from utils.definitions import NEW_USER_BALANCE from utils.nostr_utils import send_event @@ -68,11 +69,11 @@ def add_to_sql_table(db, npub, sats, iswhitelisted, isblacklisted, nip05, lud16, print(e) -def update_sql_table(db, npub, sats, iswhitelisted, isblacklisted, nip05, lud16, name, lastactive): +def update_sql_table(db, npub, balance, iswhitelisted, isblacklisted, nip05, lud16, name, lastactive): try: con = sqlite3.connect(db) cur = con.cursor() - data = (sats, iswhitelisted, isblacklisted, nip05, lud16, name, lastactive, npub) + data = (balance, iswhitelisted, isblacklisted, nip05, lud16, name, lastactive, npub) cur.execute(""" UPDATE users SET sats = ? , @@ -97,8 +98,7 @@ def get_from_sql_table(db, npub): row = cur.fetchone() con.close() if row is None: - user = None - return user + return None else: user = User user.npub = row[0] @@ -192,29 +192,55 @@ def update_user_balance(db, sender, sats, client, config): def get_or_add_user(db, npub, client): user = get_from_sql_table(db, npub) if user is None: - name, nip05, lud16 = fetch_user_metadata(npub, client) - print("Adding User: " + name + " (" + npub +")") + print("Adding User: " + npub + " (" + npub + ")") add_to_sql_table(db, npub, NEW_USER_BALANCE, False, False, nip05, lud16, name, Timestamp.now().as_secs()) user = get_from_sql_table(db, npub) - else: - # update Name, Nip05 and lud16 lnaddress - user.name, user.nip05, user.lud16 = fetch_user_metadata(npub, client) - update_sql_table(db, user.npub, user.balance, user.iswhitelisted, user.isblacklisted, user.nip05, user.lud16, - user.name, Timestamp.now().as_secs()) + return user return user -def fetch_user_metadata(sender, client) -> (str, str, str): +def fetch_user_metadata(npub, client): + name = "" + nip05 = "" + lud16 = "" + + # Get metadata + pk = PublicKey.from_hex(npub) + print(f"\nGetting profile metadata for {pk.to_bech32()}...") + filter = Filter().kind(0).author(pk).limit(1) + events = client.get_events_of([filter], timedelta(seconds=3)) + if len(events) > 0: + latest_entry = events[0] + newest = 0 + for entry in events: + if entry.created_at().as_secs() > newest: + newest = entry.created_at().as_secs() + latest_entry = entry + + print(latest_entry.content()) + profile = json.loads(latest_entry.content()) + if profile.get("name"): + name = profile['name'] + if profile.get("nip05"): + nip05 = profile['nip05'] + if profile.get("lud16"): + lud16 = profile['lud16'] + + return name, nip05, lud16 + + +def fetch_user_metadata2(sender, client) -> (str, str, str): name = "" nip05 = "" lud16 = "" try: pk = PublicKey.from_hex(sender) + print(f"\nGetting profile metadata for {pk.to_bech32()}...") profile_filter = Filter().kind(0).author(pk).limit(1) - events = client.get_events_of([profile_filter], timedelta(seconds=3)) + events = client.get_events_of([profile_filter], timedelta(seconds=1)) if len(events) > 0: ev = events[0] metadata = Metadata.from_json(ev.content()) @@ -223,9 +249,11 @@ def fetch_user_metadata(sender, client) -> (str, str, str): name = metadata.get_name() nip05 = metadata.get_nip05() lud16 = metadata.get_lud16() + else: + print("Profile not found") + return name, nip05, lud16 except: print("Couldn't get meta information") return name, nip05, lud16 - diff --git a/utils/nip89_utils.py b/utils/nip89_utils.py index ddcd11f..be918dd 100644 --- a/utils/nip89_utils.py +++ b/utils/nip89_utils.py @@ -11,6 +11,11 @@ class NIP89Announcement: content: str +class NIP89Config: + DTAG: str + CONTENT: str + + def nip89_announce_tasks(dvm_config, client): k_tag = Tag.parse(["k", str(dvm_config.NIP89.kind)]) d_tag = Tag.parse(["d", dvm_config.NIP89.dtag])