added generate image via nserver, refactor

.env_example

@@ -1,9 +1,14 @@
-NOSTR_PRIVATE_KEY = nostrSecretkeyinhex
+NOSTR_PRIVATE_KEY = "nostrSecretkeyinhex"
-NOSTR_TEST_CLIENT_PRIVATE_KEY = nostrSecretkeyinhex_forthetestclient
+NOSTR_TEST_CLIENT_PRIVATE_KEY = "nostrSecretkeyinhex_forthetestclient"
-USER_DB_PATH = nostrzaps.db
+USER_DB_PATH = "nostrzaps.db"
 
-LNBITS_INVOICE_KEY = lnbitswalletinvoicekey
+# Optional LNBITS options to create invoices (if empty, it will use the lud16 from profile)
-LNBITS_HOST = https://lnbits.com
+LNBITS_INVOICE_KEY = ""
+LNBITS_HOST = "https://lnbits.com"
 
 TASK_TEXTEXTRACTION_NIP89_DTAG = "asdd"
-TASK_TRANSLATION_NIP89_DTAG = abcded
+TASK_TRANSLATION_NIP89_DTAG = "abcded"
+TASK_IMAGEGENERATION_NIP89_DTAG = "fgdfgdf"
+
+#Backend Specific Options for tasks that require them
+NOVA_SERVER = "127.0.0.1:37318"

README.md

@@ -1,4 +1,4 @@
-# Nostr AI Data Vending Machine
+# NostrAI Data Vending Machine
 
 This example DVM implementation in Python currently supports simple translations using Google translate, as well as extraction of text from links with pdf files.
 

backends/README.md

@@ -0,0 +1,10 @@
+# NostrAI Data Vending Machine Backends
+
+Each DVM task might either run locally or use a specific backend.
+Especially for GPU tasks it might make sense to outsource some tasks on other machines.
+Backends can also be API calls to (paid) services. This directory contains basic calling functions to such backends.
+Modules in the folder "tasks" might use these functions to call a specific backend.
+
+Using backends might require some extra work like running/hosting a server or acquiring an API key.
+
+

backends/nova_server.py

@@ -0,0 +1,105 @@
+import io
+import json
+import os
+import time
+import zipfile
+import pandas as pd
+import requests
+import PIL.Image as Image
+
+from utils.output_utils import uploadMediaToHoster
+
+"""
+This file contains basic calling functions for ML tasks that are outsourced to nova-server 
+(https://github.com/hcmlab/nova-server). nova-server is an Open-Source backend that enables running models locally, by
+accepting a request form. Modules are deployed in in separate virtual environments so dependencies won't conflict. 
+
+Setup nova-server:
+https://hcmlab.github.io/nova-server/docbuild/html/tutorials/introduction.html
+
+"""
+
+"""
+send_request_to_nova_server(request_form)
+Function to send a request_form to the server, containing all the information we parsed from the Nostr event and added
+in the module that is calling the server
+
+"""
+
+
+def send_request_to_nova_server(request_form, address):
+    print("Sending job to NOVA-Server")
+    url = ('http://' + address + '/' + str(request_form["mode"]).lower())
+    headers = {'Content-type': 'application/x-www-form-urlencoded'}
+    response = requests.post(url, headers=headers, data=request_form)
+    return response.content
+
+
+"""
+check_nova_server_status(request_form)
+Function that requests the status of the current process with the jobID (we use the Nostr event as jobID).
+When the Job is successfully finished we grab the result and depending on the type return the output
+We throw an exception on error
+"""
+
+
+def check_nova_server_status(jobID, address):
+    headers = {'Content-type': 'application/x-www-form-urlencoded'}
+    url_status = 'http://' + address + '/job_status'
+    url_log = 'http://' + address + '/log'
+
+    print("Sending Status Request to NOVA-Server")
+    data = {"jobID": jobID}
+
+    status = 0
+    length = 0
+    while status != 2 and status != 3:
+        response_status = requests.post(url_status, headers=headers, data=data)
+        response_log = requests.post(url_log, headers=headers, data=data)
+        status = int(json.loads(response_status.text)['status'])
+
+        log = str(response_log.content)[length:]
+        length = len(str(response_log.content))
+        if log != "":
+            print(log + " Status: " + str(status))
+        # WAITING = 0, RUNNING = 1, FINISHED = 2, ERROR = 3
+        time.sleep(1.0)
+
+    if status == 2:
+        try:
+            result = ""
+            url_fetch = 'http://' + address + '/fetch_result'
+            print("Fetching Results from NOVA-Server...")
+            data = {"jobID": jobID}
+            response = requests.post(url_fetch, headers=headers, data=data)
+            content_type = response.headers['content-type']
+            print(content_type)
+            if content_type == "image/jpeg":
+                image = Image.open(io.BytesIO(response.content))
+                image.save("./outputs/image.jpg")
+                result = uploadMediaToHoster("./outputs/image.jpg")
+                os.remove("./outputs/image.jpg")
+            elif content_type == 'text/plain; charset=utf-8':
+                result = response.content.decode('utf-8')
+            elif content_type == "zip":
+                zf = zipfile.ZipFile(io.BytesIO(response.content), "r")
+
+                for fileinfo in zf.infolist():
+                    if fileinfo.filename.endswith(".annotation~"):
+                        try:
+                            anno_string = zf.read(fileinfo).decode('utf-8', errors='replace')
+                            columns = ['from', 'to', 'name', 'conf']
+                            result = pd.DataFrame([row.split(';') for row in anno_string.split('\n')],
+                                                  columns=columns)
+                            print(result)
+                            with open("response.zip", "wb") as f:
+                                f.write(response.content)
+                        except:
+                            zf.extractall()
+
+            return result
+        except Exception as e:
+            print("Couldn't fetch result: " + str(e))
+
+    elif status == 3:
+        return "error"

dvm.py

@@ -22,13 +22,14 @@ jobs_on_hold_list = []
 dvm_config = DVMConfig()
 
 
-def dvm(config):
+def DVM(config):
     dvm_config = config
     keys = Keys.from_sk_str(dvm_config.PRIVATE_KEY)
     pk = keys.public_key()
 
     print(f"Nostr DVM public key: {pk.to_bech32()}, Hex: {pk.to_hex()} ")
-    print('Supported DVM tasks: ' + ', '.join(p.TASK for p in dvm_config.SUPPORTED_TASKS))
+    print('Supported DVM tasks: ' + ', '.join(p.NAME + ":" + p.TASK for p in dvm_config.SUPPORTED_TASKS))
+
 
     client = Client(keys)
     for relay in dvm_config.RELAY_LIST:
@@ -207,6 +208,7 @@ def dvm(config):
                 or job_event.kind() == EventDefinitions.KIND_DM):
 
             task = get_task(job_event, client=client, dvmconfig=dvm_config)
+            result = ""
             for dvm in dvm_config.SUPPORTED_TASKS:
                 try:
                     if task == dvm.TASK:
@@ -215,7 +217,11 @@ def dvm(config):
                         check_and_return_event(result, str(job_event.as_json()), dvm_key=dvm_config.PRIVATE_KEY)
 
                 except Exception as e:
+                    print(e)
                     respond_to_error(e, job_event.as_json(), is_from_bot, dvm_config.PRIVATE_KEY)
+                    return
+
+
 
     def check_event_has_not_unfinished_job_input(nevent, append, client, dvmconfig):
         task_supported, task, duration = check_task_is_supported(nevent, client, False, config=dvmconfig)
@@ -246,44 +252,43 @@ def dvm(config):
     def send_job_status_reaction(original_event, status, is_paid=True, amount=0, client=None, content=None, config=None,
                                  key=None):
         dvmconfig = config
-        altdesc = "This is a reaction to a NIP90 DVM AI task. "
+        alt_description = "This is a reaction to a NIP90 DVM AI task. "
         task = get_task(original_event, client=client, dvmconfig=dvmconfig)
         if status == "processing":
-            altdesc = "NIP90 DVM AI task " + task + " started processing. "
+            alt_description = "NIP90 DVM AI task " + task + " started processing. "
-            reaction = altdesc + emoji.emojize(":thumbs_up:")
+            reaction = alt_description + emoji.emojize(":thumbs_up:")
         elif status == "success":
-            altdesc = "NIP90 DVM AI task " + task + " finished successfully. "
+            alt_description = "NIP90 DVM AI task " + task + " finished successfully. "
-            reaction = altdesc + emoji.emojize(":call_me_hand:")
+            reaction = alt_description + emoji.emojize(":call_me_hand:")
         elif status == "chain-scheduled":
-            altdesc = "NIP90 DVM AI task " + task + " Chain Task scheduled"
+            alt_description = "NIP90 DVM AI task " + task + " Chain Task scheduled"
-            reaction = altdesc + emoji.emojize(":thumbs_up:")
+            reaction = alt_description + emoji.emojize(":thumbs_up:")
         elif status == "error":
-            altdesc = "NIP90 DVM AI task " + task + " had an error. "
+            alt_description = "NIP90 DVM AI task " + task + " had an error. "
             if content is None:
-                reaction = altdesc + emoji.emojize(":thumbs_down:")
+                reaction = alt_description + emoji.emojize(":thumbs_down:")
             else:
-                reaction = altdesc + emoji.emojize(":thumbs_down:") + content
+                reaction = alt_description + emoji.emojize(":thumbs_down:") + content
 
         elif status == "payment-required":
 
-            altdesc = "NIP90 DVM AI task " + task + " requires payment of min " + str(amount) + " Sats. "
+            alt_description = "NIP90 DVM AI task " + task + " requires payment of min " + str(amount) + " Sats. "
-            reaction = altdesc + emoji.emojize(":orange_heart:")
+            reaction = alt_description + emoji.emojize(":orange_heart:")
 
         elif status == "payment-rejected":
-            altdesc = "NIP90 DVM AI task " + task + " payment is below required amount of " + str(amount) + " Sats. "
+            alt_description = "NIP90 DVM AI task " + task + " payment is below required amount of " + str(amount) + " Sats. "
-            reaction = altdesc + emoji.emojize(":thumbs_down:")
+            reaction = alt_description + emoji.emojize(":thumbs_down:")
         elif status == "user-blocked-from-service":
-
+            alt_description = "NIP90 DVM AI task " + task + " can't be performed. User has been blocked from Service. "
-            altdesc = "NIP90 DVM AI task " + task + " can't be performed. User has been blocked from Service. "
+            reaction = alt_description + emoji.emojize(":thumbs_down:")
-            reaction = altdesc + emoji.emojize(":thumbs_down:")
         else:
             reaction = emoji.emojize(":thumbs_down:")
 
-        etag = Tag.parse(["e", original_event.id().to_hex()])
+        e_tag = Tag.parse(["e", original_event.id().to_hex()])
-        ptag = Tag.parse(["p", original_event.pubkey().to_hex()])
+        p_tag = Tag.parse(["p", original_event.pubkey().to_hex()])
-        alttag = Tag.parse(["alt", altdesc])
+        alt_tag = Tag.parse(["alt", alt_description])
-        statustag = Tag.parse(["status", status])
+        status_tag = Tag.parse(["status", status])
-        tags = [etag, ptag, alttag, statustag]
+        tags = [e_tag, p_tag, alt_tag, status_tag]
 
         if status == "success" or status == "error":  #
             for x in job_list:
@@ -354,8 +359,13 @@ def dvm(config):
                     send_nostr_reply_event(data, original_event_str, key=keys)
                 break
 
-        post_processed_content = post_process_result(data, original_event)
+
-        send_nostr_reply_event(post_processed_content, original_event_str, key=keys)
+        try:
+            post_processed_content = post_process_result(data, original_event)
+            send_nostr_reply_event(post_processed_content, original_event_str, key=keys)
+        except Exception as e:
+            respond_to_error(e, original_event_str, False, dvm_config.PRIVATE_KEY)
+
 
     def send_nostr_reply_event(content, original_event_as_str, key=None):
         originalevent = Event.from_json(original_event_as_str)
@@ -395,7 +405,7 @@ def dvm(config):
         sender = ""
         task = ""
         if not is_from_bot:
-            send_job_status_reaction(original_event, "error", content=content, key=dvm_key)
+            send_job_status_reaction(original_event, "error", content=str(content), key=dvm_key)
             # TODO Send Zap back
         else:
             for tag in original_event.tags():
@@ -407,7 +417,8 @@ def dvm(config):
             user = get_from_sql_table(sender)
             if not user.iswhitelisted:
                 amount = int(user.balance) + get_amount_per_task(task, dvm_config)
-                update_sql_table(sender, amount, user.iswhitelisted, user.isblacklisted, user.nip05, user.lud16, user.name,
+                update_sql_table(sender, amount, user.iswhitelisted, user.isblacklisted, user.nip05, user.lud16,
+                                 user.name,
                                  Timestamp.now().as_secs())
                 message = "There was the following error : " + content + ". Credits have been reimbursed"
             else:

interfaces/dvmtaskinterface.py

@@ -3,7 +3,7 @@ class DVMTaskInterface:
     TASK: str
     COST: int
 
-    def NIP89_announcement(self):
+    def NIP89_announcement(self, d_tag, content):
         """Define the NIP89 Announcement"""
         pass
 

main.py

@@ -4,36 +4,74 @@ from threading import Thread
 
 import dotenv
 import utils.env as env
-from tasks.textextractionPDF import TextExtractionPDF
+from tasks.imagegenerationsdxl import ImageGenerationSDXL
+from tasks.textextractionpdf import TextExtractionPDF
 from tasks.translation import Translation
-from utils.definitions import EventDefinitions
-
 
 def run_nostr_dvm_with_local_config():
-    from dvm import dvm, DVMConfig
+    from dvm import DVM, DVMConfig
 
-    dvmconfig = DVMConfig()
+    dvm_config = DVMConfig()
-    dvmconfig.PRIVATE_KEY = os.getenv(env.NOSTR_PRIVATE_KEY)
+    dvm_config.PRIVATE_KEY = os.getenv(env.NOSTR_PRIVATE_KEY)
 
-    #Spawn two DVMs
+    # Spawn the DVMs
-    PDFextactor = TextExtractionPDF("PDF Extractor", env.NOSTR_PRIVATE_KEY)
+    # Add  NIP89 events for each DVM (set rebroad_cast = True for the next start in admin_utils)
-    Translator = Translation("Translator", env.NOSTR_PRIVATE_KEY)
+    # Add the dtag here or in your .env file, so you can update your dvm later and change the content as needed.
+    # Get a dtag and the content at vendata.io
 
-    #Add  the 2 DVMS to the config
+    # Spawn DVM1 Kind 5000 Text Ectractor from PDFs
-    dvmconfig.SUPPORTED_TASKS = [PDFextactor, Translator]
+    pdfextactor = TextExtractionPDF("PDF Extractor", os.getenv(env.NOSTR_PRIVATE_KEY))
+    d_tag = os.getenv(env.TASK_TEXTEXTRACTION_NIP89_DTAG)
+    content = "{\"name\":\"" + pdfextactor.NAME + ("\",\"image\":\"https://image.nostr.build"
+                                                   "/c33ca6fc4cc038ca4adb46fdfdfda34951656f87ee364ef59095bae1495ce669"
+                                                   ".jpg\",\"about\":\"I extract Text from pdf documents\","
+                                                   "\"nip90Params\":{}}")
+    dvm_config.NIP89s.append(pdfextactor.NIP89_announcement(d_tag, content))
 
-    # Add  NIP89 events for both DVMs (set rebroad_cast = True in admin_utils)
+    # Spawn DVM2 Kind 5002 Text Translation
-    # Add the dtag in your .env file so you can update your dvm later and change the content in the module file as needed.
+    translator = Translation("Translator", os.getenv(env.NOSTR_PRIVATE_KEY))
-    # Get a dtag at vendata.io
+    d_tag = os.getenv(env.TASK_TRANSLATION_NIP89_DTAG)
-    dvmconfig.NIP89s.append(PDFextactor.NIP89_announcement())
+    content = "{\"name\":\"" + translator.NAME + ("\",\"image\":\"https://image.nostr.build"
-    dvmconfig.NIP89s.append(Translator.NIP89_announcement())
+                                                  "/c33ca6fc4cc038ca4adb46fdfdfda34951656f87ee364ef59095bae1495ce669"
+                                                  ".jpg\",\"about\":\"I translate text from given text/event/job, "
+                                                  "currently using Google Translation Services into language defined "
+                                                  "in param.  \",\"nip90Params\":{\"language\":{\"required\":true,"
+                                                  "\"values\":[\"af\",\"am\",\"ar\",\"az\",\"be\",\"bg\",\"bn\","
+                                                  "\"bs\",\"ca\",\"ceb\",\"co\",\"cs\",\"cy\",\"da\",\"de\",\"el\","
+                                                  "\"eo\",\"es\",\"et\",\"eu\",\"fa\",\"fi\",\"fr\",\"fy\",\"ga\","
+                                                  "\"gd\",\"gl\",\"gu\",\"ha\",\"haw\",\"hi\",\"hmn\",\"hr\",\"ht\","
+                                                  "\"hu\",\"hy\",\"id\",\"ig\",\"is\",\"it\",\"he\",\"ja\",\"jv\","
+                                                  "\"ka\",\"kk\",\"km\",\"kn\",\"ko\",\"ku\",\"ky\",\"la\",\"lb\","
+                                                  "\"lo\",\"lt\",\"lv\",\"mg\",\"mi\",\"mk\",\"ml\",\"mn\",\"mr\","
+                                                  "\"ms\",\"mt\",\"my\",\"ne\",\"nl\",\"no\",\"ny\",\"or\",\"pa\","
+                                                  "\"pl\",\"ps\",\"pt\",\"ro\",\"ru\",\"sd\",\"si\",\"sk\",\"sl\","
+                                                  "\"sm\",\"sn\",\"so\",\"sq\",\"sr\",\"st\",\"su\",\"sv\",\"sw\","
+                                                  "\"ta\",\"te\",\"tg\",\"th\",\"tl\",\"tr\",\"ug\",\"uk\",\"ur\","
+                                                  "\"uz\",\"vi\",\"xh\",\"yi\",\"yo\",\"zh\",\"zu\"]}}}")
+    dvm_config.NIP89s.append(translator.NIP89_announcement(d_tag, content))
 
-    #SET Lnbits Invoice Key and Server if DVM should provide invoices directly, else make sure you have a lnaddress on the profile
+    # Spawn DVM3 Kind 5100 Image Generation This one uses a specific backend called nova-server. If you want to use
-    dvmconfig.LNBITS_INVOICE_KEY = os.getenv(env.LNBITS_INVOICE_KEY)
+    # it see the instructions in backends/nova_server
-    dvmconfig.LNBITS_URL = os.getenv(env.LNBITS_HOST)
+    artist = ImageGenerationSDXL("Unstable Diffusion", os.getenv(env.NOSTR_PRIVATE_KEY))
+    d_tag = os.getenv(env.TASK_IMAGEGENERATION_NIP89_DTAG)
+    content = "{\"name\":\"" + artist.NAME + ("\",\"image\":\"https://image.nostr.build"
+                                              "/c33ca6fc4cc038ca4adb46fdfdfda34951656f87ee364ef59095bae1495ce669.jpg"
+                                              "\",\"about\":\"I draw images based on a prompt with Stable Diffusion "
+                                              "XL 1.0.\",\"nip90Params\":{}}")
+    dvm_config.NIP89s.append(artist.NIP89_announcement(d_tag, content))
 
-    #Start the DVM
+
-    nostr_dvm_thread = Thread(target=dvm, args=[dvmconfig])
+
+    # Add the DVMS you want to use to the config
+    dvm_config.SUPPORTED_TASKS = [pdfextactor, translator, artist]
+
+    # SET Lnbits Invoice Key and Server if DVM should provide invoices directly, else make sure you have a lnaddress
+    # on the profile
+    dvm_config.LNBITS_INVOICE_KEY = os.getenv(env.LNBITS_INVOICE_KEY)
+    dvm_config.LNBITS_URL = os.getenv(env.LNBITS_HOST)
+
+    # Start the Server
+    nostr_dvm_thread = Thread(target=DVM, args=[dvm_config])
     nostr_dvm_thread.start()
 
 

requirements.txt

@@ -21,6 +21,7 @@ python-dateutil==2.8.2
 python-dotenv==1.0.0
 python-editor==1.0.4
 pytz==2023.3.post1
+PyUpload~=0.1.4
 pyuseragents==1.0.5
 readchar==4.0.5
 requests==2.31.0
@@ -31,3 +32,4 @@ translatepy==2.3
 tzdata==2023.3
 urllib3==2.1.0
 wcwidth==0.2.10
+

tasks/README.md

@@ -0,0 +1,13 @@
+# NostrAI Data Vending Machine Tasks
+
+Here Tasks can be defined. Tasks need to follow the DVMTaskInterface as defined in interfaces. 
+Tasks can either happen locally (especially if they are fast) or they can call an alternative backend.
+Reusable backend functions can be defined in backends (e.g. API calls)
+
+Current List of Tasks:
+
+| Module              | Kind | Description                               | Backend                   |
+|---------------------|------|-------------------------------------------|---------------------------|
+| Translation         | 5002 | Translates Inputs to another language     | Local, calling Google API |
+| TextExtractionPDF   | 5001 | Extracts Text from a PDF file             | Local                     |
+| ImageGenerationSDXL | 5100 | Generates an Image with StableDiffusionXL | nova-server               |

tasks/imagegenerationsdxl.py

@@ -0,0 +1,120 @@
+import os
+from multiprocessing.pool import ThreadPool
+from backends.nova_server import check_nova_server_status, send_request_to_nova_server
+from interfaces.dvmtaskinterface import DVMTaskInterface
+from utils.definitions import EventDefinitions
+from utils.nip89_utils import NIP89Announcement
+
+
+"""
+This File contains a Module to transform Text input on NOVA-Server and receive results back. 
+
+Accepted Inputs: Prompt (text)
+Outputs: An url to an Image
+"""
+
+
+class ImageGenerationSDXL(DVMTaskInterface):
+    KIND: int = EventDefinitions.KIND_NIP90_GENERATE_IMAGE
+    TASK: str = "text-to-image"
+    COST: int = 0
+
+    def __init__(self, name, pk):
+        self.NAME = name
+        self.PK = pk
+
+    def NIP89_announcement(self, d_tag, content):
+        nip89 = NIP89Announcement()
+        nip89.kind = self.KIND
+        nip89.pk = self.PK
+        nip89.dtag = d_tag
+        nip89.content = content
+        return nip89
+
+    def is_input_supported(self, input_type, input_content):
+        if input_type != "text":
+            return False
+        return True
+
+    def create_request_form_from_nostr_event(self, event, client=None, dvm_config=None):
+        request_form = {"jobID": event.id().to_hex() + "_"+ self.NAME.replace(" ", "")}
+        request_form["mode"] = "PROCESS"
+        request_form["trainerFilePath"] = 'modules\\stablediffusionxl\\stablediffusionxl.trainer'
+
+        prompt = ""
+        negative_prompt = ""
+        #model = "stabilityai/stable-diffusion-xl-base-1.0"
+        model = "unstable"
+        # models: juggernautXL, dynavisionXL, colossusProjectXL, newrealityXL, unstable
+        ratio_width = "1"
+        ratio_height = "1"
+        width = ""
+        height = ""
+        lora = ""
+        lora_weight = ""
+        strength = ""
+        guidance_scale = ""
+        for tag in event.tags():
+            if tag.as_vec()[0] == 'i':
+                input_type = tag.as_vec()[2]
+                if input_type == "text":
+                    prompt = tag.as_vec()[1]
+
+            elif tag.as_vec()[0] == 'param':
+                print(tag.as_vec()[2])
+                if tag.as_vec()[1] == "negative_prompt":
+                    negative_prompt = tag.as_vec()[2]
+                elif tag.as_vec()[1] == "lora":
+                    lora = tag.as_vec()[2]
+                elif tag.as_vec()[1] == "lora_weight":
+                    lora_weight = tag.as_vec()[2]
+                elif tag.as_vec()[1] == "strength":
+                    strength = tag.as_vec()[2]
+                elif tag.as_vec()[1] == "guidance_scale":
+                    guidance_scale = tag.as_vec()[2]
+                elif tag.as_vec()[1] == "ratio":
+                    if len(tag.as_vec()) > 3:
+                        ratio_width = (tag.as_vec()[2])
+                        ratio_height = (tag.as_vec()[3])
+                    elif len(tag.as_vec()) == 3:
+                        split = tag.as_vec()[2].split(":")
+                        ratio_width = split[0]
+                        ratio_height = split[1]
+                    #if size is set it will overwrite ratio.
+                elif tag.as_vec()[1] == "size":
+
+                    if len(tag.as_vec()) > 3:
+                         width = (tag.as_vec()[2])
+                         height = (tag.as_vec()[3])
+                    elif len(tag.as_vec()) == 3:
+                        split = tag.as_vec()[2].split("x")
+                        if len(split) > 1:
+                            width = split[0]
+                            height = split[1]
+                            print(width)
+                            print(height)
+                elif tag.as_vec()[1] == "model":
+                    model = tag.as_vec()[2]
+
+        prompt = prompt.replace(";", ",")
+        request_form['data'] = '[{"id":"input_prompt","type":"input","src":"request:text","data":"' + prompt + '","active":"True"},{"id":"negative_prompt","type":"input","src":"request:text","data":"' + negative_prompt + '","active":"True"},{"id":"output_image","type":"output","src":"request:image","active":"True"}]'
+        request_form["optStr"] = ('model=' + model + ';ratio=' + str(ratio_width) + '-' + str(ratio_height) + ';size=' +
+                                  str(width) + '-' + str(height) + ';strength=' + str(strength) + ';guidance_scale=' +
+                                  str(guidance_scale) + ';lora=' + lora + ';lora_weight=' + lora_weight)
+
+        return request_form
+
+    def process(self, request_form):
+        try:
+            # Call the process route of NOVA-Server with our request form.
+            success = send_request_to_nova_server(request_form, os.environ["NOVA_SERVER"])
+            print(success)
+
+            pool = ThreadPool(processes=1)
+            thread = pool.apply_async(check_nova_server_status, (request_form['jobID'], os.environ["NOVA_SERVER"]))
+            print("Wait for results of NOVA-Server...")
+            result = thread.get()
+            return str(result)
+
+        except Exception as e:
+            raise Exception(e)

tasks/textextractionpdf.py

@@ -2,12 +2,16 @@ import os
 import re
 
 from interfaces.dvmtaskinterface import DVMTaskInterface
-from utils import env
 from utils.definitions import EventDefinitions
 from utils.nip89_utils import NIP89Announcement
-from utils.nostr_utils import get_event_by_id, get_referenced_event_by_id
+from utils.nostr_utils import get_event_by_id
 
+"""
+This File contains a Module to extract Text from a PDF file locally on the DVM Machine
 
+Accepted Inputs: Url to pdf file, Event containing an URL to a PDF file
+Outputs: Text containing the extracted contents of the PDF file
+"""
 class TextExtractionPDF(DVMTaskInterface):
     KIND: int = EventDefinitions.KIND_NIP90_EXTRACT_TEXT
     TASK: str = "pdf-to-text"
@@ -17,16 +21,16 @@ class TextExtractionPDF(DVMTaskInterface):
         self.NAME = name
         self.PK = pk
 
-    def NIP89_announcement(self):
+    def NIP89_announcement(self, d_tag, content):
         nip89 = NIP89Announcement()
         nip89.kind = self.KIND
         nip89.pk = self.PK
-        nip89.dtag = os.getenv(env.TASK_TEXTEXTRACTION_NIP89_DTAG)
+        nip89.dtag = d_tag
-        nip89.content = "{\"name\":\"" + self.NAME + "\",\"image\":\"https://image.nostr.build/c33ca6fc4cc038ca4adb46fdfdfda34951656f87ee364ef59095bae1495ce669.jpg\",\"about\":\"I extract Text from pdf documents\",\"nip90Params\":{}}"
+        nip89.content = content
         return nip89
 
     def is_input_supported(self, input_type, input_content):
-        if input_type != "url":
+        if input_type != "url" and input_type != "event":
             return False
         return True
 
@@ -45,23 +49,21 @@ class TextExtractionPDF(DVMTaskInterface):
 
         if input_type == "url":
             url = input_content
+        # if event contains url to pdf, we checked for a pdf link before
         elif input_type == "event":
             evt = get_event_by_id(input_content, config=dvm_config)
             url = re.search("(?P<url>https?://[^\s]+)", evt.content()).group("url")
-        elif input_type == "job":
-            evt = get_referenced_event_by_id(input_content, [EventDefinitions.KIND_NIP90_RESULT_GENERATE_IMAGE],
-                                             client, config=dvm_config)
-
-            url = re.search("(?P<url>https?://[^\s]+)", evt.content()).group("url")
 
         request_form["optStr"] = 'url=' + url
         return request_form
 
     def process(self, request_form):
-        options = DVMTaskInterface.setOptions(request_form)
         from pypdf import PdfReader
         from pathlib import Path
         import requests
+
+        options = DVMTaskInterface.setOptions(request_form)
+
         try:
             file_path = Path('temp.pdf')
             response = requests.get(options["url"])

tasks/translation.py

@@ -1,12 +1,19 @@
 import os
 
 from interfaces.dvmtaskinterface import DVMTaskInterface
-from utils import env
 from utils.definitions import EventDefinitions
 from utils.nip89_utils import NIP89Announcement
 from utils.nostr_utils import get_referenced_event_by_id, get_event_by_id
 
 
+"""
+This File contains a Module to call Google Translate Services locally on the DVM Machine
+
+Accepted Inputs: Text, Events, Jobs (Text Extraction, Summary, Translation)
+Outputs: Text containing the Translation in the desired language.
+"""
+
+
 class Translation(DVMTaskInterface):
     KIND: int = EventDefinitions.KIND_NIP90_TRANSLATE_TEXT
     TASK: str = "translation"
@@ -16,12 +23,12 @@ class Translation(DVMTaskInterface):
         self.NAME = name
         self.PK = pk
 
-    def NIP89_announcement(self):
+    def NIP89_announcement(self, d_tag, content):
         nip89 = NIP89Announcement()
         nip89.kind = self.KIND
         nip89.pk = self.PK
-        nip89.dtag = os.getenv(env.TASK_TRANSLATION_NIP89_DTAG)
+        nip89.dtag = d_tag
-        nip89.content = "{\"name\":\"" + self.NAME + "\",\"image\":\"https://image.nostr.build/c33ca6fc4cc038ca4adb46fdfdfda34951656f87ee364ef59095bae1495ce669.jpg\",\"about\":\"I translate text from given text/event/job, currently using Google Translation Services into language defined in param.  \",\"nip90Params\":{\"language\":{\"required\":true,\"values\":[\"af\",\"am\",\"ar\",\"az\",\"be\",\"bg\",\"bn\",\"bs\",\"ca\",\"ceb\",\"co\",\"cs\",\"cy\",\"da\",\"de\",\"el\",\"eo\",\"es\",\"et\",\"eu\",\"fa\",\"fi\",\"fr\",\"fy\",\"ga\",\"gd\",\"gl\",\"gu\",\"ha\",\"haw\",\"hi\",\"hmn\",\"hr\",\"ht\",\"hu\",\"hy\",\"id\",\"ig\",\"is\",\"it\",\"he\",\"ja\",\"jv\",\"ka\",\"kk\",\"km\",\"kn\",\"ko\",\"ku\",\"ky\",\"la\",\"lb\",\"lo\",\"lt\",\"lv\",\"mg\",\"mi\",\"mk\",\"ml\",\"mn\",\"mr\",\"ms\",\"mt\",\"my\",\"ne\",\"nl\",\"no\",\"ny\",\"or\",\"pa\",\"pl\",\"ps\",\"pt\",\"ro\",\"ru\",\"sd\",\"si\",\"sk\",\"sl\",\"sm\",\"sn\",\"so\",\"sq\",\"sr\",\"st\",\"su\",\"sv\",\"sw\",\"ta\",\"te\",\"tg\",\"th\",\"tl\",\"tr\",\"ug\",\"uk\",\"ur\",\"uz\",\"vi\",\"xh\",\"yi\",\"yo\",\"zh\",\"zu\"]}}}"
+        nip89.content = content
         return nip89
 
     def is_input_supported(self, input_type, input_content):
@@ -65,7 +72,8 @@ class Translation(DVMTaskInterface):
                 if tag.as_vec()[0] == 'i':
                     evt = get_referenced_event_by_id(tag.as_vec()[1],
                                                      [EventDefinitions.KIND_NIP90_RESULT_EXTRACT_TEXT,
-                                                      EventDefinitions.KIND_NIP90_RESULT_SUMMARIZE_TEXT],
+                                                      EventDefinitions.KIND_NIP90_RESULT_SUMMARIZE_TEXT,
+                                                      EventDefinitions.KIND_NIP90_RESULT_TRANSLATE_TEXT],
                                                      client,
                                                      config=dvm_config)
                     text = evt.content()
@@ -77,8 +85,9 @@ class Translation(DVMTaskInterface):
         return request_form
 
     def process(self, request_form):
-        options = DVMTaskInterface.setOptions(request_form)
         from translatepy.translators.google import GoogleTranslate
+
+        options = DVMTaskInterface.setOptions(request_form)
         gtranslate = GoogleTranslate()
         length = len(options["text"])
 

test_client.py

@@ -1,4 +1,3 @@
-
 import os
 import time
 import datetime as datetime
@@ -12,7 +11,9 @@ from utils.nostr_utils import send_event
 from utils.definitions import EventDefinitions, RELAY_LIST
 
 import utils.env as env
-#TODO HINT: Only use this path with a preiously whitelisted privkey, as zapping events is not implemented in the lib/code
+
+
+# TODO HINT: Best use this path with a previously whitelisted privkey, as zapping events is not implemented in the lib/code
 def nostr_client_test_translation(input, kind, lang, sats, satsmax):
     keys = Keys.from_sk_str(os.getenv(env.NOSTR_TEST_CLIENT_PRIVATE_KEY))
     if kind == "text":
@@ -22,19 +23,46 @@ def nostr_client_test_translation(input, kind, lang, sats, satsmax):
     paramTag1 = Tag.parse(["param", "language", lang])
 
     bidTag = Tag.parse(['bid', str(sats * 1000), str(satsmax * 1000)])
-    relaysTag = Tag.parse(['relays', "wss://relay.damus.io", "wss://blastr.f7z.xyz", "wss://relayable.org", "wss://nostr-pub.wellorder.net"])
+    relaysTag = Tag.parse(['relays', "wss://relay.damus.io", "wss://blastr.f7z.xyz", "wss://relayable.org",
+                           "wss://nostr-pub.wellorder.net"])
     alttag = Tag.parse(["alt", "This is a NIP90 DVM AI task to translate a given Input"])
-    event = EventBuilder(EventDefinitions.KIND_NIP90_TRANSLATE_TEXT, str("Translate the given input."), [iTag, paramTag1, bidTag, relaysTag, alttag]).to_event(keys)
+    event = EventBuilder(EventDefinitions.KIND_NIP90_TRANSLATE_TEXT, str("Translate the given input."),
+                         [iTag, paramTag1, bidTag, relaysTag, alttag]).to_event(keys)
 
     relay_list = ["wss://relay.damus.io", "wss://blastr.f7z.xyz", "wss://relayable.org",
                   "wss://nostr-pub.wellorder.net"]
 
-
     client = Client(keys)
     for relay in relay_list:
         client.add_relay(relay)
     client.connect()
-    send_event(event, client,  keys)
+    send_event(event, client, keys)
+    return event.as_json()
+
+
+def nostr_client_test_image(prompt):
+    keys = Keys.from_sk_str(os.getenv(env.NOSTR_TEST_CLIENT_PRIVATE_KEY))
+
+    iTag = Tag.parse(["i", prompt, "text"])
+    outTag = Tag.parse(["output", "image/png;format=url"])
+    paramTag1 = Tag.parse(["param", "size", "1024x1024"])
+    tTag = Tag.parse(["t", "bitcoin"])
+
+    bidTag = Tag.parse(['bid', str(1000 * 1000), str(1000 * 1000)])
+    relaysTag = Tag.parse(['relays', "wss://relay.damus.io", "wss://blastr.f7z.xyz", "wss://relayable.org",
+                           "wss://nostr-pub.wellorder.net"])
+    alttag = Tag.parse(["alt", "This is a NIP90 DVM AI task to translate a given Input"])
+    event = EventBuilder(EventDefinitions.KIND_NIP90_GENERATE_IMAGE, str("Generate an Image."),
+                         [iTag, outTag, tTag, paramTag1, bidTag, relaysTag, alttag]).to_event(keys)
+
+    relay_list = ["wss://relay.damus.io", "wss://blastr.f7z.xyz", "wss://relayable.org",
+                  "wss://nostr-pub.wellorder.net"]
+
+    client = Client(keys)
+    for relay in relay_list:
+        client.add_relay(relay)
+    client.connect()
+    send_event(event, client, keys)
     return event.as_json()
 
 def nostr_client():
@@ -48,17 +76,19 @@ def nostr_client():
     client.connect()
 
     dm_zap_filter = Filter().pubkey(pk).kinds([EventDefinitions.KIND_DM,
-                                             EventDefinitions.KIND_ZAP]).since(Timestamp.now())  # events to us specific
+                                               EventDefinitions.KIND_ZAP]).since(
+        Timestamp.now())  # events to us specific
     dvm_filter = (Filter().kinds([EventDefinitions.KIND_NIP90_RESULT_TRANSLATE_TEXT,
-                                 EventDefinitions.KIND_FEEDBACK]).since(Timestamp.now()))  # public events
+                                  EventDefinitions.KIND_FEEDBACK]).since(Timestamp.now()))  # public events
     client.subscribe([dm_zap_filter, dvm_filter])
 
+    # nostr_client_test_translation("This is the result of the DVM in spanish", "text", "es", 20, 20)
+    #nostr_client_test_translation("44a0a8b395ade39d46b9d20038b3f0c8a11168e67c442e3ece95e4a1703e2beb", "event", "es", 20,
+    #                              20)
 
-    #nostr_client_test_translation("This is the result of the DVM in spanish", "text", "es", 20, 20)
+    # nostr_client_test_translation("9c5d6d054e1b7a34a6a4b26ac59469c96e77f7cba003a30456fa6a57974ea86d", "event", "zh", 20, 20)
-    nostr_client_test_translation("44a0a8b395ade39d46b9d20038b3f0c8a11168e67c442e3ece95e4a1703e2beb", "event", "fr", 20, 20)
 
-
+    nostr_client_test_image("a beautiful purple ostrich watching the sunset")
-    #nostr_client_test_image(sats=50, satsmax=10)
     class NotificationHandler(HandleNotification):
         def handle(self, relay_url, event):
             print(f"Received new event from {relay_url}: {event.as_json()}")
@@ -76,16 +106,14 @@ def nostr_client():
                 print("[Nostr Client]: " + f"Received new zap:")
                 print(event.as_json())
 
-
         def handle_msg(self, relay_url, msg):
-            None
+            return
 
     client.handle_notifications(NotificationHandler())
     while True:
         time.sleep(5.0)
 
 
-
 if __name__ == '__main__':
 
     env_path = Path('.env')
@@ -95,6 +123,5 @@ if __name__ == '__main__':
     else:
         raise FileNotFoundError(f'.env file not found at {env_path} ')
 
-
     nostr_dvm_thread = Thread(target=nostr_client())
     nostr_dvm_thread.start()

utils/backend_utils.py

@@ -1,10 +1,10 @@
 import requests
-
-from tasks.textextractionPDF import TextExtractionPDF
 from utils.definitions import EventDefinitions
 from utils.nostr_utils import get_event_by_id
 
+from tasks.textextractionpdf import TextExtractionPDF
 from tasks.translation import Translation
+from tasks.imagegenerationsdxl import ImageGenerationSDXL
 
 
 def get_task(event, client, dvmconfig):
@@ -35,9 +35,9 @@ def get_task(event, client, dvmconfig):
                     evt = get_event_by_id(tag.as_vec()[1], config=dvmconfig)
                     if evt is not None:
                         if evt.kind() == 1063:
-                            for tag in evt.tags():
+                            for tg in evt.tags():
-                                if tag.as_vec()[0] == 'url':
+                                if tg.as_vec()[0] == 'url':
-                                    file_type = check_url_is_readable(tag.as_vec()[1])
+                                    file_type = check_url_is_readable(tg.as_vec()[1])
                                     if file_type == "pdf":
                                         return "pdf-to-text"
                                     else:
@@ -45,9 +45,10 @@ def get_task(event, client, dvmconfig):
                         else:
                             return "unknown type"
 
-
     elif event.kind() == EventDefinitions.KIND_NIP90_TRANSLATE_TEXT:
         return Translation.TASK
+    elif event.kind() == EventDefinitions.KIND_NIP90_GENERATE_IMAGE:
+        return ImageGenerationSDXL.TASK
 
     else:
         return "unknown type"
@@ -121,7 +122,6 @@ def check_url_is_readable(url):
 
 
 def get_amount_per_task(task, dvm_config, duration=1):
-    print(dvm_config.SUPPORTED_TASKS)
     for dvm in dvm_config.SUPPORTED_TASKS:
         if dvm.TASK == task:
             amount = dvm.COST * duration

utils/env.py

@@ -8,5 +8,6 @@ LNBITS_HOST = "LNBITS_HOST"
 
 TASK_TRANSLATION_NIP89_DTAG = "TASK_TRANSLATION_NIP89_DTAG"
 TASK_TEXTEXTRACTION_NIP89_DTAG = "TASK_TEXTEXTRACTION_NIP89_DTAG"
+TASK_IMAGEGENERATION_NIP89_DTAG = "TASK_IMAGEGENERATION_NIP89_DTAG"
 
 

utils/output_utils.py

@@ -1,10 +1,15 @@
 import json
 import datetime as datetime
+import os
 from types import NoneType
+import requests
+from pyupload.uploader import CatboxUploader
 
 import pandas
 
-
+'''
+Post process results to either given output format or a Nostr readable plain text.
+'''
 def post_process_result(anno, original_event):
     print("post-processing...")
     if isinstance(anno, pandas.DataFrame):  # if input is an anno we parse it to required output format
@@ -84,7 +89,52 @@ def post_process_result(anno, original_event):
         return result
 
 
+'''
+Convenience function to replace words like Noster with Nostr
+'''
 def replace_broken_words(text):
     result = (text.replace("Noster", "Nostr").replace("Nostra", "Nostr").replace("no stir", "Nostr").
               replace("Nostro", "Nostr").replace("Impub", "npub").replace("sets", "Sats"))
     return result
+
+
+'''
+Function to upload to Nostr.build and if it fails to Nostrfiles.dev
+Larger files than these hosters allow and fallback is catbox currently.
+Will probably need to switch to another system in the future.
+'''
+def uploadMediaToHoster(filepath):
+    print("Uploading image: " + filepath)
+    try:
+        files = {'file': open(filepath, 'rb')}
+        file_stats = os.stat(filepath)
+        sizeinmb = file_stats.st_size / (1024*1024)
+        print("Filesize of Uploaded media: " + str(sizeinmb) + " Mb.")
+        if sizeinmb > 25:
+            uploader = CatboxUploader(filepath)
+            result = uploader.execute()
+            return result
+        else:
+            url = 'https://nostr.build/api/v2/upload/files'
+            response = requests.post(url, files=files)
+            json_object = json.loads(response.text)
+            result = json_object["data"][0]["url"]
+            return result
+    except:
+        try:
+            file = {'file': open(filepath, 'rb')}
+            url = 'https://nostrfiles.dev/upload_image'
+            response = requests.post(url, files=file)
+            json_object = json.loads(response.text)
+            print(json_object["url"])
+            return json_object["url"]
+            # fallback filehoster
+        except:
+
+            try:
+                uploader = CatboxUploader(filepath)
+                result = uploader.execute()
+                print(result)
+                return result
+            except:
+                return "Upload not possible, all hosters didn't work"