separate dbs per DVM, fix for shared joblists

This commit is contained in:
Believethehype 2023-11-21 10:03:04 +01:00
parent 86dafcc320
commit 50f4076416
10 changed files with 138 additions and 121 deletions

1
.gitignore vendored
View File

@ -160,3 +160,4 @@ cython_debug/
#.idea/
nostrzaps.db
.DS_Store
*.db

66
dvm.py
View File

@ -3,7 +3,8 @@ from nostr_sdk import PublicKey, Keys, Client, Tag, Event, EventBuilder, Filter,
import time
import emoji
from utils.definitions import EventDefinitions, DVMConfig, RequiredJobToWatch, JobToWatch
from utils.definitions import EventDefinitions, RequiredJobToWatch, JobToWatch
from utils.dvmconfig import DVMConfig
from utils.admin_utils import admin_make_database_updates
from utils.backend_utils import get_amount_per_task, check_task_is_supported, get_task
from utils.database_utils import update_sql_table, get_from_sql_table, \
@ -17,19 +18,22 @@ use_logger = False
if use_logger:
init_logger(LogLevel.DEBUG)
job_list = []
jobs_on_hold_list = []
class DVM:
dvm_config: DVMConfig
keys: Keys
client: Client
job_list: list
jobs_on_hold_list: list
def __init__(self, config):
self.dvm_config = config
self.keys = Keys.from_sk_str(config.PRIVATE_KEY)
self.client = Client(self.keys)
self.job_list = []
self.jobs_on_hold_list = []
pk = self.keys.public_key()
@ -49,7 +53,7 @@ class DVM:
dvm_filter = (Filter().kinds(kinds).since(Timestamp.now()))
self.client.subscribe([dm_zap_filter, dvm_filter])
create_sql_table()
create_sql_table(self.dvm_config.DB)
admin_make_database_updates(config=self.dvm_config, client=self.client)
class NotificationHandler(HandleNotification):
@ -68,7 +72,9 @@ class DVM:
return
def handle_nip90_job_event(nip90_event):
user = get_or_add_user(nip90_event.pubkey().to_hex())
print(str(self.dvm_config.DB))
user = get_or_add_user(self.dvm_config.DB, nip90_event.pubkey().to_hex())
print("got user")
task_supported, task, duration = check_task_is_supported(nip90_event, client=self.client,
get_duration=(not user.iswhitelisted),
config=self.dvm_config)
@ -149,7 +155,7 @@ class DVM:
else:
anon = True
print("Anonymous Zap received. Unlucky, I don't know from whom, and never will")
user = get_or_add_user(sender)
user = get_or_add_user(self.dvm_config.DB, sender)
print(str(user))
if zapped_event is not None:
@ -172,20 +178,20 @@ class DVM:
print("[Nostr] Payment-request fulfilled...")
send_job_status_reaction(job_event, "processing", client=self.client,
config=self.dvm_config)
indices = [i for i, x in enumerate(job_list) if
indices = [i for i, x in enumerate(self.job_list) if
x.event_id == job_event.id().to_hex()]
index = -1
if len(indices) > 0:
index = indices[0]
if index > -1:
if job_list[index].is_processed: # If payment-required appears a processing
job_list[index].is_paid = True
check_and_return_event(job_list[index].result,
if self.job_list[index].is_processed: # If payment-required appears a processing
self.job_list[index].is_paid = True
check_and_return_event(self.job_list[index].result,
str(job_event.as_json()),
dvm_key=self.dvm_config.PRIVATE_KEY)
elif not (job_list[index]).is_processed:
elif not (self.job_list[index]).is_processed:
# If payment-required appears before processing
job_list.pop(index)
self.job_list.pop(index)
print("Starting work...")
do_work(job_event, is_from_bot=False)
else:
@ -203,13 +209,13 @@ class DVM:
elif not anon:
print("Note Zap received for Bot balance: " + str(invoice_amount) + " Sats from " + str(
user.name))
update_user_balance(sender, invoice_amount, config=self.dvm_config)
update_user_balance(self.dvm_config.DB, sender, invoice_amount, config=self.dvm_config)
# a regular note
elif not anon:
print("Profile Zap received for Bot balance: " + str(invoice_amount) + " Sats from " + str(
user.name))
update_user_balance(sender, invoice_amount, config=self.dvm_config)
update_user_balance(self.dvm_config.DB, sender, invoice_amount, config=self.dvm_config)
except Exception as e:
print(f"Error during content decryption: {e}")
@ -233,7 +239,7 @@ class DVM:
if evt is None:
if append:
job = RequiredJobToWatch(event=nevent, timestamp=Timestamp.now().as_secs())
jobs_on_hold_list.append(job)
self.jobs_on_hold_list.append(job)
send_job_status_reaction(nevent, "chain-scheduled", True, 0, client=client,
config=dvmconfig)
@ -245,7 +251,7 @@ class DVM:
original_event = Event.from_json(original_event_str)
keys = Keys.from_sk_str(dvm_key)
for x in job_list:
for x in self.job_list:
if x.event_id == original_event.id().to_hex():
is_paid = x.is_paid
amount = x.amount
@ -260,9 +266,9 @@ class DVM:
config=self.dvm_config) # or payment-required, or both?
if self.dvm_config.SHOWRESULTBEFOREPAYMENT and is_paid:
job_list.remove(x)
self.job_list.remove(x)
elif not self.dvm_config.SHOWRESULTBEFOREPAYMENT and is_paid:
job_list.remove(x)
self.job_list.remove(x)
send_nostr_reply_event(data, original_event_str, key=keys)
break
@ -317,10 +323,10 @@ class DVM:
elif tag.as_vec()[0] == "i":
task = tag.as_vec()[1]
user = get_from_sql_table(sender)
user = get_from_sql_table(self.dvm_config.DB, sender)
if not user.iswhitelisted:
amount = int(user.balance) + get_amount_per_task(task, self.dvm_config)
update_sql_table(sender, amount, user.iswhitelisted, user.isblacklisted, user.nip05, user.lud16,
update_sql_table(self.dvm_config.DB, sender, amount, user.iswhitelisted, user.isblacklisted, user.nip05, user.lud16,
user.name,
Timestamp.now().as_secs())
message = "There was the following error : " + content + ". Credits have been reimbursed"
@ -378,7 +384,7 @@ class DVM:
tags = [e_tag, p_tag, alt_tag, status_tag]
if status == "success" or status == "error": #
for x in job_list:
for x in self.job_list:
if x.event_id == original_event.id():
is_paid = x.is_paid
amount = x.amount
@ -394,8 +400,8 @@ class DVM:
except Exception as e:
print(e)
if not any(x.event_id == original_event.id().to_hex() for x in job_list):
job_list.append(
if not any(x.event_id == original_event.id().to_hex() for x in self.job_list):
self.job_list.append(
JobToWatch(event_id=original_event.id().to_hex(),
timestamp=original_event.created_at().as_secs(),
amount=amount,
@ -403,7 +409,7 @@ class DVM:
status=status, result="", is_processed=False, bolt11=bolt11,
payment_hash=payment_hash,
expires=expires, from_bot=False))
print(str(job_list))
print(str(self.job_list))
if status == "payment-required" or status == "payment-rejected" or (
status == "processing" and not is_paid) or (
status == "success" and not is_paid):
@ -448,7 +454,7 @@ class DVM:
self.client.handle_notifications(NotificationHandler())
while True:
for job in job_list:
for job in self.job_list:
if job.bolt11 != "" and job.payment_hash != "" and not job.is_paid:
if str(check_bolt11_ln_bits_is_paid(job.payment_hash, self.dvm_config)) == "True":
job.is_paid = True
@ -462,26 +468,26 @@ class DVM:
do_work(event, is_from_bot=False)
elif check_bolt11_ln_bits_is_paid(job.payment_hash, self.dvm_config) is None: # invoice expired
try:
job_list.remove(job)
self.job_list.remove(job)
except:
continue
if Timestamp.now().as_secs() > job.expires:
try:
job_list.remove(job)
self.job_list.remove(job)
except:
continue
for job in jobs_on_hold_list:
for job in self.jobs_on_hold_list:
if check_event_has_not_unfinished_job_input(job.event, False, client=self.client,
dvmconfig=self.dvm_config):
handle_nip90_job_event(nip90_event=job.event)
try:
jobs_on_hold_list.remove(job)
self.jobs_on_hold_list.remove(job)
except:
continue
if Timestamp.now().as_secs() > job.timestamp + 60 * 20: # remove jobs to look for after 20 minutes..
jobs_on_hold_list.remove(job)
self.jobs_on_hold_list.remove(job)
time.sleep(2.0)

15
main.py
View File

@ -1,6 +1,5 @@
import os
from pathlib import Path
from threading import Thread
import dotenv
import utils.env as env
@ -10,7 +9,7 @@ from tasks.translation import Translation
def run_nostr_dvm_with_local_config():
from dvm import DVM, DVMConfig
from utils.dvmconfig import DVMConfig
# Spawn the DVMs
# Add NIP89 events for each DVM (set rebroadcast = True for the next start in admin_utils)
@ -67,12 +66,12 @@ def run_nostr_dvm_with_local_config():
dvm_config.LNBITS_INVOICE_KEY = os.getenv(env.LNBITS_INVOICE_KEY)
dvm_config.LNBITS_URL = os.getenv(env.LNBITS_HOST)
#unstableartist = ImageGenerationSDXL("Unstable Diffusion", dvm_config, "unstable")
#d_tag = os.getenv(env.TASK_IMAGEGENERATION_NIP89_DTAG)
#content = "{\"name\":\"" + unstableartist.NAME + ("\",\"image\":\"https://image.nostr.build"
# "/c33ca6fc4cc038ca4adb46fdfdfda34951656f87ee364ef59095bae1495ce669.jpg"
# "\",\"about\":\"I draw images based on a prompt with a Model called unstable diffusion.\",\"nip90Params\":{}}")
#dvm_config.NIP89s.append(unstableartist.NIP89_announcement(d_tag, content))
unstableartist = ImageGenerationSDXL("Unstable Diffusion", dvm_config, "unstable")
d_tag = os.getenv(env.TASK_IMAGEGENERATION_NIP89_DTAG)
content = "{\"name\":\"" + unstableartist.NAME + ("\",\"image\":\"https://image.nostr.build"
"/c33ca6fc4cc038ca4adb46fdfdfda34951656f87ee364ef59095bae1495ce669.jpg"
"\",\"about\":\"I draw images based on a prompt with a Model called unstable diffusion.\",\"nip90Params\":{}}")
dvm_config.NIP89s.append(unstableartist.NIP89_announcement(d_tag, content))
# Spawn another Instance of text-to-image but use a different model and lora this time.
dvm_config = DVMConfig()

View File

@ -6,7 +6,6 @@ from backends.nova_server import check_nova_server_status, send_request_to_nova_
from dvm import DVM
from interfaces.dvmtaskinterface import DVMTaskInterface
from utils.definitions import EventDefinitions
from utils.nip89_utils import NIP89Announcement
"""
@ -27,6 +26,7 @@ class ImageGenerationSDXL(DVMTaskInterface):
def __init__(self, name, dvm_config, default_model=None, default_lora=None):
self.NAME = name
dvm_config.SUPPORTED_TASKS = [self]
dvm_config.DB = "db/" + self.NAME + ".db"
self.PK = dvm_config.PRIVATE_KEY
self.default_model = default_model
self.default_lora = default_lora

View File

@ -5,7 +5,6 @@ from threading import Thread
from dvm import DVM
from interfaces.dvmtaskinterface import DVMTaskInterface
from utils.definitions import EventDefinitions
from utils.nip89_utils import NIP89Announcement
from utils.nostr_utils import get_event_by_id
"""
@ -14,6 +13,8 @@ This File contains a Module to extract Text from a PDF file locally on the DVM M
Accepted Inputs: Url to pdf file, Event containing an URL to a PDF file
Outputs: Text containing the extracted contents of the PDF file
"""
class TextExtractionPDF(DVMTaskInterface):
NAME: str
KIND: int = EventDefinitions.KIND_NIP90_EXTRACT_TEXT
@ -21,18 +22,16 @@ class TextExtractionPDF(DVMTaskInterface):
COST: int = 20
PK: str
def __init__(self, name, dvm_config):
self.NAME = name
dvm_config.SUPPORTED_TASKS = [self]
dvm_config.DB = "db/" + self.NAME + ".db"
self.PK = dvm_config.PRIVATE_KEY
dvm = DVM
nostr_dvm_thread = Thread(target=dvm, args=[dvm_config])
nostr_dvm_thread.start()
def is_input_supported(self, input_type, input_content):
if input_type != "url" and input_type != "event":
return False

View File

@ -1,13 +1,10 @@
import os
from threading import Thread
from dvm import DVM
from interfaces.dvmtaskinterface import DVMTaskInterface
from utils.definitions import EventDefinitions
from utils.nip89_utils import NIP89Announcement
from utils.nostr_utils import get_referenced_event_by_id, get_event_by_id
"""
This File contains a Module to call Google Translate Services locally on the DVM Machine
@ -26,6 +23,7 @@ class Translation(DVMTaskInterface):
def __init__(self, name, dvm_config):
self.NAME = name
dvm_config.SUPPORTED_TASKS = [self]
dvm_config.DB = "db/" + self.NAME + ".db"
self.PK = dvm_config.PRIVATE_KEY
dvm = DVM

View File

@ -8,10 +8,13 @@ from utils.database_utils import get_from_sql_table, list_db, delete_from_sql_ta
from utils.nip89_utils import nip89_announce_tasks
from utils.nostr_utils import send_event
class AdminConfig:
REBROADCASTNIP89: bool = False
def admin_make_database_updates(config=None, client=None):
# This is called on start of Server, Admin function to manually whitelist/blacklist/add balance/delete users
dvmconfig = config
db = config.DB
rebroadcast_nip89 = False
cleandb = False
@ -28,27 +31,27 @@ def admin_make_database_updates(config=None, client=None):
#use this if you have hex
if whitelistuser:
user = get_or_add_user(publickey)
update_sql_table(user.npub, user.balance, True, False, user.nip05, user.lud16, user.name, user.lastactive)
user = get_from_sql_table(publickey)
user = get_or_add_user(db, publickey)
update_sql_table(db, user.npub, user.balance, True, False, user.nip05, user.lud16, user.name, user.lastactive)
user = get_from_sql_table(db, publickey)
print(str(user.name) + " is whitelisted: " + str(user.iswhitelisted))
if unwhitelistuser:
user = get_from_sql_table(publickey)
update_sql_table(user.npub, user.balance, False, False, user.nip05, user.lud16, user.name, user.lastactive)
user = get_from_sql_table(db, publickey)
update_sql_table(db, user.npub, user.balance, False, False, user.nip05, user.lud16, user.name, user.lastactive)
if blacklistuser:
user = get_from_sql_table(publickey)
update_sql_table(user.npub, user.balance, False, True, user.nip05, user.lud16, user.name, user.lastactive)
user = get_from_sql_table(db, publickey)
update_sql_table(db, user.npub, user.balance, False, True, user.nip05, user.lud16, user.name, user.lastactive)
if deleteuser:
delete_from_sql_table(publickey)
delete_from_sql_table(db, publickey)
if cleandb:
clean_db()
clean_db(db)
if listdatabase:
list_db()
list_db(db)
if rebroadcast_nip89:
nip89_announce_tasks(dvmconfig)

View File

@ -1,5 +1,4 @@
# DATABASE LOGIC
import os
import sqlite3
import time
@ -10,10 +9,10 @@ from logging import Filter
from nostr_sdk import Timestamp, Keys, PublicKey, EventBuilder, Metadata, Filter
from utils import env
from utils.definitions import NEW_USER_BALANCE
from utils.nostr_utils import send_event
@dataclass
class User:
npub: str
@ -26,10 +25,9 @@ class User:
lastactive: int
def create_sql_table():
def create_sql_table(db):
try:
con = sqlite3.connect(os.getenv(env.USER_DB_PATH))
con = sqlite3.connect(db)
cur = con.cursor()
cur.execute(""" CREATE TABLE IF NOT EXISTS users (
npub text PRIMARY KEY,
@ -48,9 +46,9 @@ def create_sql_table():
print(e)
def add_sql_table_column():
def add_sql_table_column(db):
try:
con = sqlite3.connect(os.getenv(env.USER_DB_PATH))
con = sqlite3.connect(db)
cur = con.cursor()
cur.execute(""" ALTER TABLE users ADD COLUMN lastactive 'integer' """)
con.close()
@ -58,9 +56,11 @@ def add_sql_table_column():
print(e)
def add_to_sql_table(npub, sats, iswhitelisted, isblacklisted, nip05, lud16, name, lastactive):
def add_to_sql_table(db, npub, sats, iswhitelisted, isblacklisted, nip05, lud16, name, lastactive):
try:
con = sqlite3.connect(os.getenv(env.USER_DB_PATH))
print("ADD: " + str(db))
con = sqlite3.connect(db)
print("Connected")
cur = con.cursor()
data = (npub, sats, iswhitelisted, isblacklisted, nip05, lud16, name, lastactive)
cur.execute("INSERT or IGNORE INTO users VALUES(?, ?, ?, ?, ?, ?, ?, ?)", data)
@ -70,9 +70,9 @@ def add_to_sql_table(npub, sats, iswhitelisted, isblacklisted, nip05, lud16, nam
print(e)
def update_sql_table(npub, sats, iswhitelisted, isblacklisted, nip05, lud16, name, lastactive):
def update_sql_table(db, npub, sats, iswhitelisted, isblacklisted, nip05, lud16, name, lastactive):
try:
con = sqlite3.connect(os.getenv(env.USER_DB_PATH))
con = sqlite3.connect(db)
cur = con.cursor()
data = (sats, iswhitelisted, isblacklisted, nip05, lud16, name, lastactive, npub)
@ -91,32 +91,39 @@ def update_sql_table(npub, sats, iswhitelisted, isblacklisted, nip05, lud16, nam
print(e)
def get_from_sql_table(npub):
def get_from_sql_table(db, npub):
try:
con = sqlite3.connect(os.getenv(env.USER_DB_PATH))
con = sqlite3.connect(db)
print("Connecting to DB")
cur = con.cursor()
cur.execute("SELECT * FROM users WHERE npub=?", (npub,))
row = cur.fetchone()
con.close()
user = User
user.npub = row[0]
user.balance = row[1]
user.iswhitelisted = row[2]
user.isblacklisted = row[3]
user.nip05 = row[4]
user.lud16 = row[5]
user.name = row[6]
user.lastactive = row[7]
print(row)
if row is None:
user = None
print("returning None")
return user
else:
user = User
user.npub = row[0]
user.balance = row[1]
user.iswhitelisted = row[2]
user.isblacklisted = row[3]
user.nip05 = row[4]
user.lud16 = row[5]
user.name = row[6]
user.lastactive = row[7]
return user
return user
except Error as e:
print(e)
def delete_from_sql_table(npub):
def delete_from_sql_table(db, npub):
try:
con = sqlite3.connect(os.getenv(env.USER_DB_PATH))
con = sqlite3.connect(db)
cur = con.cursor()
cur.execute("DELETE FROM users WHERE npub=?", (npub,))
con.commit()
@ -125,24 +132,24 @@ def delete_from_sql_table(npub):
print(e)
def clean_db():
def clean_db(db):
try:
con = sqlite3.connect(os.getenv(env.USER_DB_PATH))
con = sqlite3.connect(db)
cur = con.cursor()
cur.execute("SELECT * FROM users WHERE npub IS NULL OR npub = '' ")
rows = cur.fetchall()
for row in rows:
print(row)
delete_from_sql_table(row[0])
delete_from_sql_table(db, row[0])
con.close()
return rows
except Error as e:
print(e)
def list_db():
def list_db(db):
try:
con = sqlite3.connect(os.getenv(env.USER_DB_PATH))
con = sqlite3.connect(db)
cur = con.cursor()
cur.execute("SELECT * FROM users ORDER BY sats DESC")
rows = cur.fetchall()
@ -153,17 +160,16 @@ def list_db():
print(e)
def update_user_balance(sender, sats, config=None):
user = get_from_sql_table(sender)
def update_user_balance(db, sender, sats, config=None):
user = get_from_sql_table(db, sender)
if user is None:
add_to_sql_table(sender, (int(sats) + NEW_USER_BALANCE), False, False,
add_to_sql_table(db, sender, (int(sats) + NEW_USER_BALANCE), False, False,
"", "", "", Timestamp.now().as_secs())
print("NEW USER: " + sender + " Zap amount: " + str(sats) + " Sats.")
else:
user = get_from_sql_table(sender)
user = get_from_sql_table(db, sender)
print(str(sats))
if user.nip05 is None:
user.nip05 = ""
if user.lud16 is None:
@ -172,36 +178,36 @@ def update_user_balance(sender, sats, config=None):
user.name = ""
new_balance = int(user.balance) + int(sats)
update_sql_table(sender, new_balance, user.iswhitelisted, user.isblacklisted, user.nip05, user.lud16, user.name,
update_sql_table(db, sender, new_balance, user.iswhitelisted, user.isblacklisted, user.nip05, user.lud16,
user.name,
Timestamp.now().as_secs())
print("UPDATE USER BALANCE: " + str(user.name) + " Zap amount: " + str(sats) + " Sats.")
if config is not None:
keys = Keys.from_sk_str(config.PRIVATE_KEY)
time.sleep(1.0)
message = ("Added "+ str(sats) + " Sats to balance. New balance is " + str(new_balance) + " Sats. " )
message = ("Added " + str(sats) + " Sats to balance. New balance is " + str(new_balance) + " Sats. ")
evt = EventBuilder.new_encrypted_direct_msg(keys, PublicKey.from_hex(sender), message,
None).to_event(keys)
send_event(evt, key=keys)
def get_or_add_user(sender):
user = get_from_sql_table(sender)
def get_or_add_user(db, sender):
user = get_from_sql_table(db, sender)
if user is None:
add_to_sql_table(sender, NEW_USER_BALANCE, False, False, None,
print("Adding User")
add_to_sql_table(db, sender, NEW_USER_BALANCE, False, False, None,
None, None, Timestamp.now().as_secs())
user = get_from_sql_table(sender)
user = get_from_sql_table(db, sender)
print(user)
return user
def update_user_metadata(sender, client):
user = get_from_sql_table(sender)
def update_user_metadata(db, sender, client):
user = get_from_sql_table(db, sender)
try:
profile_filter = Filter().kind(0).author(sender).limit(1)
events = client.get_events_of([profile_filter], timedelta(seconds=3))
@ -215,8 +221,7 @@ def update_user_metadata(sender, client):
user.lud16 = metadata.get_lud16()
except:
print("Couldn't get meta information")
update_sql_table(user.npub, user.balance, user.iswhitelisted, user.isblacklisted, user.nip05, user.lud16,
update_sql_table(db, user.npub, user.balance, user.iswhitelisted, user.isblacklisted, user.nip05, user.lud16,
user.name, Timestamp.now().as_secs())
user = get_from_sql_table(user.npub)
user = get_from_sql_table(db, user.npub)
return user

View File

@ -43,22 +43,6 @@ class EventDefinitions:
KIND_NIP90_RESULT_GENERIC]
class DVMConfig:
SUPPORTED_TASKS = []
PRIVATE_KEY: str = os.getenv(env.NOSTR_PRIVATE_KEY)
RELAY_LIST = ["wss://relay.damus.io", "wss://nostr-pub.wellorder.net", "wss://nos.lol", "wss://nostr.wine",
"wss://relay.nostfiles.dev", "wss://nostr.mom", "wss://nostr.oxtr.dev", "wss://relay.nostr.bg",
"wss://relay.f7z.io"]
RELAY_TIMEOUT = 5
LNBITS_INVOICE_KEY = ''
LNBITS_URL = 'https://lnbits.com'
REQUIRES_NIP05: bool = False
SHOWRESULTBEFOREPAYMENT: bool = True # if this is true show results even when not paid right after autoprocess
NIP89s: list = []
@dataclass

22
utils/dvmconfig.py Normal file
View File

@ -0,0 +1,22 @@
import os
from utils import env
class DVMConfig:
SUPPORTED_TASKS = []
PRIVATE_KEY: str = os.getenv(env.NOSTR_PRIVATE_KEY)
RELAY_LIST = ["wss://relay.damus.io", "wss://nostr-pub.wellorder.net", "wss://nos.lol", "wss://nostr.wine",
"wss://relay.nostfiles.dev", "wss://nostr.mom", "wss://nostr.oxtr.dev", "wss://relay.nostr.bg",
"wss://relay.f7z.io"]
RELAY_TIMEOUT = 5
LNBITS_INVOICE_KEY = ''
LNBITS_URL = 'https://lnbits.com'
REQUIRES_NIP05: bool = False
DB: str
SHOWRESULTBEFOREPAYMENT: bool = True # if this is true show results even when not paid right after autoprocess
NIP89s: list = []