highperfocused/nostrdvm
1
0
Fork 0
mirror of https://github.com/believethehype/nostrdvm.git synced 2025-09-25 20:56:44 +02:00
Code Issues Packages Projects Releases Wiki Activity

added libretranslate api

Browse Source 
Untested tho
This commit is contained in:
Believethehype
2023-11-29 10:01:20 +01:00
parent c2795bf7b7
commit 8225478a24
5 changed files with 171 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
.env_example
View File

@@ -6,6 +6,10 @@ BOT_PRIVATE_KEY = "The private key for a test bot that communicates with dvms"
#These are all for the playground and can be replaced and adjusted however needed
NOSTR_PRIVATE_KEY = "a secret hexkey for some demo dvms"
NOSTR_PRIVATE_KEY2 = "another secret hexkey for demo dvm with another key"
NOSTR_PRIVATE_KEY3 = "another secret hexkey for demo dvm with another key"
NOSTR_PRIVATE_KEY4 = "another secret hexkey for demo dvm with another key"
NOSTR_PRIVATE_KEY5 = "another secret hexkey for demo dvm with another key"
BOT_PRIVATE_KEY = "The private key for a test bot that communicates with dvms"
NOSTR_TEST_CLIENT_PRIVATE_KEY = "a secret hex key for the test dvm client"
@@ -24,6 +28,8 @@ TASK_IMAGE_GENERATION_NIP89_DTAG2 = "fdgdfg"
TASK_IMAGE_GENERATION_NIP89_DTAG3 = "asdasd"
#Backend Specific Options for tasks that require them
#Backend Specific Options for tasks that require inputs, such as Endpoints or API Keys
OPENAI_API_KEY = "" # Enter your OpenAI API Key to use DVMs with OpenAI services
LIBRE_TRANSLATE_ENDPOINT = "" # Url to LibreTranslate Endpoint e.g. https://libretranslate.com
LIBRE_TRANSLATE_API_KEY = "" # API Key, if required (You can host your own instance where you don't need it)
NOVA_SERVER = "" # Enter the address of a nova-server instance, locally or on a machine in your network host:port

16
main.py
View File

@@ -9,8 +9,9 @@ import dotenv
from nostr_sdk import Keys
from bot import Bot
from playground import build_pdf_extractor, build_translator, build_unstable_diffusion, build_sketcher, build_dalle, \
build_whisperx
from playground import build_pdf_extractor, build_googletranslator, build_unstable_diffusion, build_sketcher, \
build_dalle, \
build_whisperx, build_libretranslator
from utils.dvmconfig import DVMConfig
@@ -31,11 +32,18 @@ def run_nostr_dvm_with_local_config():
# If we don't add it to the bot, the bot will not provide access to the DVM
pdfextractor.run()
# Spawn DVM2 Kind 5002 Local Text Translation, calling the free Google API.
translator = build_translator("Translator")
# Spawn DVM2 Kind 5002 Local Text TranslationGoogle, calling the free Google API.
translator = build_googletranslator("Google Translator")
bot_config.SUPPORTED_DVMS.append(translator) # We add translator to the bot
translator.run()
# Spawn DVM3 Kind 5002 Local Text TranslationLibre, calling the free LibreTranslateApi, as an alternative.
if os.getenv("LIBRE_TRANSLATE_ENDPOINT") is not None and os.getenv("LIBRE_TRANSLATE_ENDPOINT") != "":
libre_translator = build_libretranslator("Libre Translator")
bot_config.SUPPORTED_DVMS.append(libre_translator) # We add translator to the bot
libre_translator.run()
# Spawn DVM3 Kind 5100 Image Generation This one uses a specific backend called nova-server.
# If you want to use it, see the instructions in backends/nova_server
if os.getenv("NOVA_SERVER") is not None and os.getenv("NOVA_SERVER") != "":

58
playground.py
View File

@@ -8,7 +8,8 @@ from tasks.imagegeneration_openai_dalle import ImageGenerationDALLE
from tasks.imagegeneration_sdxl import ImageGenerationSDXL
from tasks.textextraction_whisperx import SpeechToTextWhisperX
from tasks.textextractionpdf import TextExtractionPDF
from tasks.translation import Translation
from tasks.translation_google import TranslationGoogle
from tasks.translation_libretranslate import TranslationLibre
from utils.admin_utils import AdminConfig
from utils.dvmconfig import DVMConfig
from utils.nip89_utils import NIP89Config
@@ -37,6 +38,8 @@ task, for example an address or an API key.
# their NIP89 announcement
admin_config = AdminConfig()
admin_config.REBROADCAST_NIP89 = False
# Set rebroadcast to true once you have set your NIP89 descriptions and d tags. You only need to rebroadcast once you
# want to update your NIP89 descriptions
@@ -62,7 +65,7 @@ def build_pdf_extractor(name):
admin_config=admin_config)
def build_translator(name):
def build_googletranslator(name):
dvm_config = DVMConfig()
dvm_config.PRIVATE_KEY = os.getenv("NOSTR_PRIVATE_KEY")
dvm_config.LNBITS_INVOICE_KEY = os.getenv("LNBITS_INVOICE_KEY")
@@ -83,21 +86,55 @@ def build_translator(name):
nip89info = {
"name": name,
"image": "https://image.nostr.build/c33ca6fc4cc038ca4adb46fdfdfda34951656f87ee364ef59095bae1495ce669.jpg",
"about": "I translate text from given text/event/job. Currently using Google Translation Services to translate "
"about": "I translate text from given text/event/job. Currently using Google TranslationGoogle Services to translate "
"input into the language defined in params.",
"nip90Params": nip90params
}
nip89config = NIP89Config()
nip89config.DTAG = os.getenv("TASK_TRANSLATION_NIP89_DTAG")
nip89config.CONTENT = json.dumps(nip89info)
return Translation(name=name, dvm_config=dvm_config, nip89config=nip89config,
admin_config=admin_config)
return TranslationGoogle(name=name, dvm_config=dvm_config, nip89config=nip89config,
admin_config=admin_config)
def build_libretranslator(name):
dvm_config = DVMConfig()
dvm_config.PRIVATE_KEY = os.getenv("NOSTR_PRIVATE_KEY5")
dvm_config.LNBITS_INVOICE_KEY = os.getenv("LNBITS_INVOICE_KEY")
dvm_config.LNBITS_URL = os.getenv("LNBITS_HOST")
options = {'libre_end_point': os.getenv("LIBRE_TRANSLATE_ENDPOINT"),
'libre_api_key': os.getenv("LIBRE_TRANSLATE_API_KEY")}
nip90params = {
"language": {
"required": False,
"values": ["en", "az", "be", "bg", "bn", "bs", "ca", "ceb", "co", "cs", "cy", "da", "de", "el", "eo", "es",
"et", "eu", "fa", "fi", "fr", "fy", "ga", "gd", "gl", "gu", "ha", "haw", "hi", "hmn", "hr", "ht",
"hu", "hy", "id", "ig", "is", "it", "he", "ja", "jv", "ka", "kk", "km", "kn", "ko", "ku", "ky",
"la", "lb", "lo", "lt", "lv", "mg", "mi", "mk", "ml", "mn", "mr", "ms", "mt", "my", "ne", "nl",
"no", "ny", "or", "pa", "pl", "ps", "pt", "ro", "ru", "sd", "si", "sk", "sl", "sm", "sn", "so",
"sq", "sr", "st", "su", "sv", "sw", "ta", "te", "tg", "th", "tl", "tr", "ug", "uk", "ur", "uz",
"vi", "xh", "yi", "yo", "zh", "zu"]
}
}
nip89info = {
"name": name,
"image": "https://image.nostr.build/c33ca6fc4cc038ca4adb46fdfdfda34951656f87ee364ef59095bae1495ce669.jpg",
"about": "I translate text from given text/event/job using LibreTranslate Services to translate "
"input into the language defined in params.",
"nip90Params": nip90params
}
nip89config = NIP89Config()
nip89config.DTAG = os.getenv("TASK_TRANSLATION_NIP89_DTAG6")
nip89config.CONTENT = json.dumps(nip89info)
return TranslationLibre(name=name, dvm_config=dvm_config, nip89config=nip89config,
admin_config=admin_config, options=options)
def build_unstable_diffusion(name):
dvm_config = DVMConfig()
dvm_config.PRIVATE_KEY = os.getenv("NOSTR_PRIVATE_KEY")
dvm_config.LNBITS_INVOICE_KEY = "" #This one will not use Lnbits to create invoices, but rely on zaps
dvm_config.LNBITS_INVOICE_KEY = "" # This one will not use Lnbits to create invoices, but rely on zaps
dvm_config.LNBITS_URL = ""
# A module might have options it can be initialized with, here we set a default model, and the nova-server
@@ -126,6 +163,7 @@ def build_unstable_diffusion(name):
return ImageGenerationSDXL(name=name, dvm_config=dvm_config, nip89config=nip89config,
admin_config=admin_config, options=options)
def build_whisperx(name):
dvm_config = DVMConfig()
dvm_config.PRIVATE_KEY = os.getenv("NOSTR_PRIVATE_KEY4")
@@ -139,11 +177,12 @@ def build_whisperx(name):
nip90params = {
"model": {
"required": False,
"values": ["base","tiny","small","medium","large-v1","large-v2","tiny.en","base.en","small.en","medium.en"]
"values": ["base", "tiny", "small", "medium", "large-v1", "large-v2", "tiny.en", "base.en", "small.en",
"medium.en"]
},
"alignment": {
"required": False,
"values": ["raw", "segment","word"]
"values": ["raw", "segment", "word"]
}
}
nip89info = {
@@ -156,7 +195,7 @@ def build_whisperx(name):
nip89config.DTAG = os.getenv("TASK_SPEECH_TO_TEXT_NIP89")
nip89config.CONTENT = json.dumps(nip89info)
return SpeechToTextWhisperX(name=name, dvm_config=dvm_config, nip89config=nip89config,
admin_config=admin_config, options=options)
admin_config=admin_config, options=options)
def build_sketcher(name):
@@ -237,6 +276,7 @@ def external_dvm(name, pubkey):
return DVMTaskInterface(name=name, dvm_config=dvm_config, nip89config=nip89config)
# Little Gimmick:
# For Dalle where we have to pay 4cent per image, we fetch current sat price in fiat
# and update cost at each start

6
tasks/translation.py → tasks/translation_google.py
View File

@@ -12,13 +12,13 @@ from utils.nostr_utils import get_referenced_event_by_id, get_event_by_id
"""
This File contains a Module to call Google Translate Services locally on the DVM Machine
Accepted Inputs: Text, Events, Jobs (Text Extraction, Summary, Translation)
Outputs: Text containing the Translation in the desired language.
Accepted Inputs: Text, Events, Jobs (Text Extraction, Summary, TranslationGoogle)
Outputs: Text containing the TranslationGoogle in the desired language.
Params: -language The target language
"""
class Translation(DVMTaskInterface):
class TranslationGoogle(DVMTaskInterface):
KIND: int = EventDefinitions.KIND_NIP90_TRANSLATE_TEXT
TASK: str = "translation"
FIX_COST: float = 0

100
tasks/translation_libretranslate.py Normal file
View File

@@ -0,0 +1,100 @@
import json
from threading import Thread
import requests
from dvm import DVM
from interfaces.dvmtaskinterface import DVMTaskInterface
from utils.admin_utils import AdminConfig
from utils.definitions import EventDefinitions
from utils.dvmconfig import DVMConfig
from utils.nip89_utils import NIP89Config
from utils.nostr_utils import get_referenced_event_by_id, get_event_by_id
"""
This File contains a Module to call Google Translate Services locally on the DVM Machine
Accepted Inputs: Text, Events, Jobs (Text Extraction, Summary, TranslationGoogle)
Outputs: Text containing the TranslationGoogle in the desired language.
Params: -language The target language
"""
class TranslationLibre(DVMTaskInterface):
KIND: int = EventDefinitions.KIND_NIP90_TRANSLATE_TEXT
TASK: str = "translation"
FIX_COST: float = 0
def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config,
admin_config: AdminConfig = None, options=None):
super().__init__(name, dvm_config, nip89config, admin_config, options)
def is_input_supported(self, tags):
for tag in tags:
if tag.as_vec()[0] == 'i':
input_value = tag.as_vec()[1]
input_type = tag.as_vec()[2]
if input_type != "event" and input_type != "job" and input_type != "text":
return False
if input_type != "text" and len(input_value) > 4999:
return False
return True
def create_request_form_from_nostr_event(self, event, client=None, dvm_config=None):
request_form = {"jobID": event.id().to_hex()}
text = ""
translation_lang = "en"
for tag in event.tags():
if tag.as_vec()[0] == 'i':
input_type = tag.as_vec()[2]
if input_type == "event":
evt = get_event_by_id(tag.as_vec()[1], client=client, config=dvm_config)
text = evt.content()
elif input_type == "text":
text = tag.as_vec()[1]
elif input_type == "job":
evt = get_referenced_event_by_id(event_id=tag.as_vec()[1], client=client,
kinds=[EventDefinitions.KIND_NIP90_RESULT_EXTRACT_TEXT,
EventDefinitions.KIND_NIP90_RESULT_SUMMARIZE_TEXT,
EventDefinitions.KIND_NIP90_RESULT_TRANSLATE_TEXT],
dvm_config=dvm_config)
text = evt.content()
elif tag.as_vec()[0] == 'param':
param = tag.as_vec()[1]
if param == "language": # check for param type
translation_lang = str(tag.as_vec()[2]).split('-')[0]
options = {
"text": text,
"language": translation_lang
}
request_form['options'] = json.dumps(options)
return request_form
def process(self, request_form):
options = DVMTaskInterface.set_options(request_form)
request = {
"q": options["text"],
"source": "auto",
"target": options["language"]
}
if options["libre_api_key"] != "":
request["api_key"] = options["libre_api_key"]
data = json.dumps(request)
headers = {'Content-type': 'application/json'}
response = requests.post(options["libre_end_point"] + "/translate", headers=headers, data=data)
reply = json.loads(response.text)
if reply.get("translatedText"):
translated_text = reply['translatedText']
confidence = reply["detectedLanguage"]['confidence']
language = reply["detectedLanguage"]['language']
print(translated_text + "language: " + language + "conf: " + confidence)
else:
return response.text
return translated_text