From 608c5fa651da51ce8bfa1ecb59480099d47d0d99 Mon Sep 17 00:00:00 2001 From: Believethehype <1097224+believethehype@users.noreply.github.com> Date: Mon, 26 Aug 2024 07:39:18 +0200 Subject: [PATCH] error handling for db reconciliation for content-discovery --- ...ent_discovery_currently_latest_longform.py | 56 +++++------ ...content_discovery_currently_latest_wiki.py | 56 +++++------ .../content_discovery_currently_popular.py | 54 ++++++----- ...discovery_currently_popular_by_top_zaps.py | 54 ++++++----- ...t_discovery_currently_popular_followers.py | 55 ++++++----- ...ent_discovery_currently_popular_gallery.py | 52 +++++----- ...ntent_discovery_currently_popular_mostr.py | 95 ++++++++++--------- ...iscovery_currently_popular_nonfollowers.py | 54 ++++++----- ...ntent_discovery_currently_popular_topic.py | 52 +++++----- setup.py | 2 +- 10 files changed, 275 insertions(+), 255 deletions(-) diff --git a/nostr_dvm/tasks/content_discovery_currently_latest_longform.py b/nostr_dvm/tasks/content_discovery_currently_latest_longform.py index 2479884..cf125a3 100644 --- a/nostr_dvm/tasks/content_discovery_currently_latest_longform.py +++ b/nostr_dvm/tasks/content_discovery_currently_latest_longform.py @@ -171,38 +171,40 @@ class DicoverContentLatestLongForm(DVMTaskInterface): return 1 async def sync_db(self): - relaylimits = RelayLimits.disable() - opts = (Options().wait_for_send(False).send_timeout( - timedelta(seconds=self.dvm_config.RELAY_LONG_TIMEOUT))).relay_limits(relaylimits) - sk = SecretKey.from_hex(self.dvm_config.PRIVATE_KEY) - keys = Keys.parse(sk.to_hex()) - signer = NostrSigner.keys(keys) - database = await NostrDatabase.sqlite(self.db_name) - cli = ClientBuilder().signer(signer).database(database).opts(opts).build() + try: + relaylimits = RelayLimits.disable() + opts = (Options().wait_for_send(False).send_timeout( + timedelta(seconds=self.dvm_config.RELAY_LONG_TIMEOUT))).relay_limits(relaylimits) + sk = SecretKey.from_hex(self.dvm_config.PRIVATE_KEY) + keys = Keys.parse(sk.to_hex()) + signer = NostrSigner.keys(keys) + database = await NostrDatabase.sqlite(self.db_name) + cli = ClientBuilder().signer(signer).database(database).opts(opts).build() - for relay in self.dvm_config.RECONCILE_DB_RELAY_LIST: - await cli.add_relay(relay) + for relay in self.dvm_config.RECONCILE_DB_RELAY_LIST: + await cli.add_relay(relay) - await cli.connect() + await cli.connect() - timestamp_since = Timestamp.now().as_secs() - self.db_since - since = Timestamp.from_secs(timestamp_since) + timestamp_since = Timestamp.now().as_secs() - self.db_since + since = Timestamp.from_secs(timestamp_since) - filter1 = Filter().kinds([definitions.EventDefinitions.KIND_LONGFORM]).since(since) # Notes, reactions, zaps - - # filter = Filter().author(keys.public_key()) - if self.dvm_config.LOGLEVEL.value >= LogLevel.DEBUG.value: - print("[" + self.dvm_config.NIP89.NAME + "] Syncing notes of the last " + str( - self.db_since) + " seconds.. this might take a while..") - dbopts = NegentropyOptions().direction(NegentropyDirection.DOWN) - await cli.reconcile(filter1, dbopts) - await cli.database().delete(Filter().until(Timestamp.from_secs( - Timestamp.now().as_secs() - self.db_since))) # Clear old events so db doesn't get too full. - await cli.shutdown() - if self.dvm_config.LOGLEVEL.value >= LogLevel.DEBUG.value: - print( - "[" + self.dvm_config.NIP89.NAME + "] Done Syncing Notes of the last " + str(self.db_since) + " seconds..") + filter1 = Filter().kinds([definitions.EventDefinitions.KIND_LONGFORM]).since(since) # Notes, reactions, zaps + # filter = Filter().author(keys.public_key()) + if self.dvm_config.LOGLEVEL.value >= LogLevel.DEBUG.value: + print("[" + self.dvm_config.NIP89.NAME + "] Syncing notes of the last " + str( + self.db_since) + " seconds.. this might take a while..") + dbopts = NegentropyOptions().direction(NegentropyDirection.DOWN) + await cli.reconcile(filter1, dbopts) + await cli.database().delete(Filter().until(Timestamp.from_secs( + Timestamp.now().as_secs() - self.db_since))) # Clear old events so db doesn't get too full. + await cli.shutdown() + if self.dvm_config.LOGLEVEL.value >= LogLevel.DEBUG.value: + print( + "[" + self.dvm_config.NIP89.NAME + "] Done Syncing Notes of the last " + str(self.db_since) + " seconds..") + except Exception as e: + print(e) # We build an example here that we can call by either calling this file directly from the main directory, # or by adding it to our playground. You can call the example and adjust it to your needs or redefine it in the diff --git a/nostr_dvm/tasks/content_discovery_currently_latest_wiki.py b/nostr_dvm/tasks/content_discovery_currently_latest_wiki.py index 4b94b7b..91430a6 100644 --- a/nostr_dvm/tasks/content_discovery_currently_latest_wiki.py +++ b/nostr_dvm/tasks/content_discovery_currently_latest_wiki.py @@ -171,38 +171,40 @@ class DicoverContentLatestWiki(DVMTaskInterface): return 1 async def sync_db(self): - relaylimits = RelayLimits.disable() - opts = (Options().wait_for_send(False).send_timeout( - timedelta(seconds=self.dvm_config.RELAY_LONG_TIMEOUT))).relay_limits(relaylimits) - sk = SecretKey.from_hex(self.dvm_config.PRIVATE_KEY) - keys = Keys.parse(sk.to_hex()) - signer = NostrSigner.keys(keys) - database = await NostrDatabase.sqlite(self.db_name) - cli = ClientBuilder().signer(signer).database(database).opts(opts).build() + try: + relaylimits = RelayLimits.disable() + opts = (Options().wait_for_send(False).send_timeout( + timedelta(seconds=self.dvm_config.RELAY_LONG_TIMEOUT))).relay_limits(relaylimits) + sk = SecretKey.from_hex(self.dvm_config.PRIVATE_KEY) + keys = Keys.parse(sk.to_hex()) + signer = NostrSigner.keys(keys) + database = await NostrDatabase.sqlite(self.db_name) + cli = ClientBuilder().signer(signer).database(database).opts(opts).build() - for relay in self.dvm_config.RECONCILE_DB_RELAY_LIST: - await cli.add_relay(relay) + for relay in self.dvm_config.RECONCILE_DB_RELAY_LIST: + await cli.add_relay(relay) - await cli.connect() + await cli.connect() - timestamp_since = Timestamp.now().as_secs() - self.db_since - since = Timestamp.from_secs(timestamp_since) + timestamp_since = Timestamp.now().as_secs() - self.db_since + since = Timestamp.from_secs(timestamp_since) - filter1 = Filter().kinds([definitions.EventDefinitions.KIND_WIKI]).since(since) # Notes, reactions, zaps - - # filter = Filter().author(keys.public_key()) - if self.dvm_config.LOGLEVEL.value >= LogLevel.DEBUG.value: - print("[" + self.dvm_config.NIP89.NAME + "] Syncing notes of the last " + str( - self.db_since) + " seconds.. this might take a while..") - dbopts = NegentropyOptions().direction(NegentropyDirection.DOWN) - await cli.reconcile(filter1, dbopts) - await cli.database().delete(Filter().until(Timestamp.from_secs( - Timestamp.now().as_secs() - self.db_since))) # Clear old events so db doesn't get too full. - await cli.shutdown() - if self.dvm_config.LOGLEVEL.value >= LogLevel.DEBUG.value: - print( - "[" + self.dvm_config.NIP89.NAME + "] Done Syncing Notes of the last " + str(self.db_since) + " seconds..") + filter1 = Filter().kinds([definitions.EventDefinitions.KIND_WIKI]).since(since) # Notes, reactions, zaps + # filter = Filter().author(keys.public_key()) + if self.dvm_config.LOGLEVEL.value >= LogLevel.DEBUG.value: + print("[" + self.dvm_config.NIP89.NAME + "] Syncing notes of the last " + str( + self.db_since) + " seconds.. this might take a while..") + dbopts = NegentropyOptions().direction(NegentropyDirection.DOWN) + await cli.reconcile(filter1, dbopts) + await cli.database().delete(Filter().until(Timestamp.from_secs( + Timestamp.now().as_secs() - self.db_since))) # Clear old events so db doesn't get too full. + await cli.shutdown() + if self.dvm_config.LOGLEVEL.value >= LogLevel.DEBUG.value: + print( + "[" + self.dvm_config.NIP89.NAME + "] Done Syncing Notes of the last " + str(self.db_since) + " seconds..") + except Exception as e: + print(e) # We build an example here that we can call by either calling this file directly from the main directory, # or by adding it to our playground. You can call the example and adjust it to your needs or redefine it in the diff --git a/nostr_dvm/tasks/content_discovery_currently_popular.py b/nostr_dvm/tasks/content_discovery_currently_popular.py index b16fc98..3d05204 100644 --- a/nostr_dvm/tasks/content_discovery_currently_popular.py +++ b/nostr_dvm/tasks/content_discovery_currently_popular.py @@ -170,37 +170,39 @@ class DicoverContentCurrentlyPopular(DVMTaskInterface): return 0 async def sync_db(self): - opts = (Options().wait_for_send(False).send_timeout(timedelta(seconds=self.dvm_config.RELAY_LONG_TIMEOUT))) - sk = SecretKey.from_hex(self.dvm_config.PRIVATE_KEY) - keys = Keys.parse(sk.to_hex()) - signer = NostrSigner.keys(keys) - database = await NostrDatabase.sqlite(self.db_name) - cli = ClientBuilder().signer(signer).database(database).opts(opts).build() + try: + opts = (Options().wait_for_send(False).send_timeout(timedelta(seconds=self.dvm_config.RELAY_LONG_TIMEOUT))) + sk = SecretKey.from_hex(self.dvm_config.PRIVATE_KEY) + keys = Keys.parse(sk.to_hex()) + signer = NostrSigner.keys(keys) + database = await NostrDatabase.sqlite(self.db_name) + cli = ClientBuilder().signer(signer).database(database).opts(opts).build() - for relay in self.dvm_config.RECONCILE_DB_RELAY_LIST: - await cli.add_relay(relay) + for relay in self.dvm_config.RECONCILE_DB_RELAY_LIST: + await cli.add_relay(relay) - await cli.connect() + await cli.connect() - timestamp_since = Timestamp.now().as_secs() - self.db_since - since = Timestamp.from_secs(timestamp_since) + timestamp_since = Timestamp.now().as_secs() - self.db_since + since = Timestamp.from_secs(timestamp_since) - filter1 = Filter().kinds([definitions.EventDefinitions.KIND_NOTE, definitions.EventDefinitions.KIND_REACTION, - definitions.EventDefinitions.KIND_ZAP]).since(since) # Notes, reactions, zaps - - # filter = Filter().author(keys.public_key()) - if self.dvm_config.LOGLEVEL.value >= LogLevel.DEBUG.value: - print("[" + self.dvm_config.NIP89.NAME + "] Syncing notes of the last " + str( - self.db_since) + " seconds.. this might take a while..") - dbopts = NegentropyOptions().direction(NegentropyDirection.DOWN) - await cli.reconcile(filter1, dbopts) - await cli.database().delete(Filter().until(Timestamp.from_secs( - Timestamp.now().as_secs() - self.db_since))) # Clear old events so db doesn't get too full. - await cli.shutdown() - if self.dvm_config.LOGLEVEL.value >= LogLevel.DEBUG.value: - print( - "[" + self.dvm_config.NIP89.NAME + "] Done Syncing Notes of the last " + str(self.db_since) + " seconds..") + filter1 = Filter().kinds([definitions.EventDefinitions.KIND_NOTE, definitions.EventDefinitions.KIND_REACTION, + definitions.EventDefinitions.KIND_ZAP]).since(since) # Notes, reactions, zaps + # filter = Filter().author(keys.public_key()) + if self.dvm_config.LOGLEVEL.value >= LogLevel.DEBUG.value: + print("[" + self.dvm_config.NIP89.NAME + "] Syncing notes of the last " + str( + self.db_since) + " seconds.. this might take a while..") + dbopts = NegentropyOptions().direction(NegentropyDirection.DOWN) + await cli.reconcile(filter1, dbopts) + await cli.database().delete(Filter().until(Timestamp.from_secs( + Timestamp.now().as_secs() - self.db_since))) # Clear old events so db doesn't get too full. + await cli.shutdown() + if self.dvm_config.LOGLEVEL.value >= LogLevel.DEBUG.value: + print( + "[" + self.dvm_config.NIP89.NAME + "] Done Syncing Notes of the last " + str(self.db_since) + " seconds..") + except Exception as e: + print(e) # We build an example here that we can call by either calling this file directly from the main directory, # or by adding it to our playground. You can call the example and adjust it to your needs or redefine it in the diff --git a/nostr_dvm/tasks/content_discovery_currently_popular_by_top_zaps.py b/nostr_dvm/tasks/content_discovery_currently_popular_by_top_zaps.py index d215063..7d882f8 100644 --- a/nostr_dvm/tasks/content_discovery_currently_popular_by_top_zaps.py +++ b/nostr_dvm/tasks/content_discovery_currently_popular_by_top_zaps.py @@ -225,37 +225,39 @@ class DicoverContentCurrentlyPopularZaps(DVMTaskInterface): return 1 async def sync_db(self): - opts = (Options().wait_for_send(False).send_timeout(timedelta(seconds=self.dvm_config.RELAY_LONG_TIMEOUT))) - sk = SecretKey.from_hex(self.dvm_config.PRIVATE_KEY) - keys = Keys.parse(sk.to_hex()) - signer = NostrSigner.keys(keys) - database = await NostrDatabase.sqlite(self.db_name) - cli = ClientBuilder().signer(signer).database(database).opts(opts).build() + try: + opts = (Options().wait_for_send(False).send_timeout(timedelta(seconds=self.dvm_config.RELAY_LONG_TIMEOUT))) + sk = SecretKey.from_hex(self.dvm_config.PRIVATE_KEY) + keys = Keys.parse(sk.to_hex()) + signer = NostrSigner.keys(keys) + database = await NostrDatabase.sqlite(self.db_name) + cli = ClientBuilder().signer(signer).database(database).opts(opts).build() - for relay in self.dvm_config.RECONCILE_DB_RELAY_LIST: - await cli.add_relay(relay) + for relay in self.dvm_config.RECONCILE_DB_RELAY_LIST: + await cli.add_relay(relay) - await cli.connect() + await cli.connect() - timestamp_since = Timestamp.now().as_secs() - self.db_since - since = Timestamp.from_secs(timestamp_since) + timestamp_since = Timestamp.now().as_secs() - self.db_since + since = Timestamp.from_secs(timestamp_since) - filter1 = Filter().kinds([definitions.EventDefinitions.KIND_NOTE, definitions.EventDefinitions.KIND_REACTION, - definitions.EventDefinitions.KIND_ZAP]).since(since) # Notes, reactions, zaps - - # filter = Filter().author(keys.public_key()) - if self.dvm_config.LOGLEVEL.value >= LogLevel.DEBUG.value: - print("[" + self.dvm_config.NIP89.NAME + "] Syncing notes of the last " + str( - self.db_since) + " seconds.. this might take a while..") - dbopts = NegentropyOptions().direction(NegentropyDirection.DOWN) - await cli.reconcile(filter1, dbopts) - await cli.database().delete(Filter().until(Timestamp.from_secs( - Timestamp.now().as_secs() - self.db_since))) # Clear old events so db doesn't get too full. - await cli.shutdown() - if self.dvm_config.LOGLEVEL.value >= LogLevel.DEBUG.value: - print( - "[" + self.dvm_config.NIP89.NAME + "] Done Syncing Notes of the last " + str(self.db_since) + " seconds..") + filter1 = Filter().kinds([definitions.EventDefinitions.KIND_NOTE, definitions.EventDefinitions.KIND_REACTION, + definitions.EventDefinitions.KIND_ZAP]).since(since) # Notes, reactions, zaps + # filter = Filter().author(keys.public_key()) + if self.dvm_config.LOGLEVEL.value >= LogLevel.DEBUG.value: + print("[" + self.dvm_config.NIP89.NAME + "] Syncing notes of the last " + str( + self.db_since) + " seconds.. this might take a while..") + dbopts = NegentropyOptions().direction(NegentropyDirection.DOWN) + await cli.reconcile(filter1, dbopts) + await cli.database().delete(Filter().until(Timestamp.from_secs( + Timestamp.now().as_secs() - self.db_since))) # Clear old events so db doesn't get too full. + await cli.shutdown() + if self.dvm_config.LOGLEVEL.value >= LogLevel.DEBUG.value: + print( + "[" + self.dvm_config.NIP89.NAME + "] Done Syncing Notes of the last " + str(self.db_since) + " seconds..") + except Exception as e: + print(e) # We build an example here that we can call by either calling this file directly from the main directory, # or by adding it to our playground. You can call the example and adjust it to your needs or redefine it in the diff --git a/nostr_dvm/tasks/content_discovery_currently_popular_followers.py b/nostr_dvm/tasks/content_discovery_currently_popular_followers.py index b3a7fc1..aec9da3 100644 --- a/nostr_dvm/tasks/content_discovery_currently_popular_followers.py +++ b/nostr_dvm/tasks/content_discovery_currently_popular_followers.py @@ -194,37 +194,40 @@ class DicoverContentCurrentlyPopularFollowers(DVMTaskInterface): async def sync_db(self): - opts = (Options().wait_for_send(False).send_timeout(timedelta(seconds=self.dvm_config.RELAY_LONG_TIMEOUT))) - sk = SecretKey.from_hex(self.dvm_config.PRIVATE_KEY) - keys = Keys.parse(sk.to_hex()) - signer = NostrSigner.keys(keys) - database = await NostrDatabase.sqlite(self.db_name) - cli = ClientBuilder().signer(signer).database(database).opts(opts).build() + try: + opts = (Options().wait_for_send(False).send_timeout(timedelta(seconds=self.dvm_config.RELAY_LONG_TIMEOUT))) + sk = SecretKey.from_hex(self.dvm_config.PRIVATE_KEY) + keys = Keys.parse(sk.to_hex()) + signer = NostrSigner.keys(keys) + database = await NostrDatabase.sqlite(self.db_name) + cli = ClientBuilder().signer(signer).database(database).opts(opts).build() - for relay in self.dvm_config.RECONCILE_DB_RELAY_LIST: - await cli.add_relay(relay) + for relay in self.dvm_config.RECONCILE_DB_RELAY_LIST: + await cli.add_relay(relay) - await cli.connect() + await cli.connect() - timestamp_since = Timestamp.now().as_secs() - self.db_since - since = Timestamp.from_secs(timestamp_since) + timestamp_since = Timestamp.now().as_secs() - self.db_since + since = Timestamp.from_secs(timestamp_since) - filter1 = Filter().kinds( - [definitions.EventDefinitions.KIND_NOTE, definitions.EventDefinitions.KIND_REACTION, - definitions.EventDefinitions.KIND_ZAP]).since(since) # Notes, reactions, zaps + filter1 = Filter().kinds( + [definitions.EventDefinitions.KIND_NOTE, definitions.EventDefinitions.KIND_REACTION, + definitions.EventDefinitions.KIND_ZAP]).since(since) # Notes, reactions, zaps - # filter = Filter().author(keys.public_key()) - if self.dvm_config.LOGLEVEL.value >= LogLevel.DEBUG.value: - print("[" + self.dvm_config.NIP89.NAME + "] Syncing notes of the last " + str( - self.db_since) + " seconds.. this might take a while..") - dbopts = NegentropyOptions().direction(NegentropyDirection.DOWN) - await cli.reconcile(filter1, dbopts) - await cli.database().delete(Filter().until(Timestamp.from_secs( - Timestamp.now().as_secs() - self.db_since))) # Clear old events so db doesn't get too full. - await cli.shutdown() - if self.dvm_config.LOGLEVEL.value >= LogLevel.DEBUG.value: - print("[" + self.dvm_config.NIP89.NAME + "] Done Syncing Notes of the last " + str( - self.db_since) + " seconds..") + # filter = Filter().author(keys.public_key()) + if self.dvm_config.LOGLEVEL.value >= LogLevel.DEBUG.value: + print("[" + self.dvm_config.NIP89.NAME + "] Syncing notes of the last " + str( + self.db_since) + " seconds.. this might take a while..") + dbopts = NegentropyOptions().direction(NegentropyDirection.DOWN) + await cli.reconcile(filter1, dbopts) + await cli.database().delete(Filter().until(Timestamp.from_secs( + Timestamp.now().as_secs() - self.db_since))) # Clear old events so db doesn't get too full. + await cli.shutdown() + if self.dvm_config.LOGLEVEL.value >= LogLevel.DEBUG.value: + print("[" + self.dvm_config.NIP89.NAME + "] Done Syncing Notes of the last " + str( + self.db_since) + " seconds..") + except Exception as e: + print(e) # We build an example here that we can call by either calling this file directly from the main directory, diff --git a/nostr_dvm/tasks/content_discovery_currently_popular_gallery.py b/nostr_dvm/tasks/content_discovery_currently_popular_gallery.py index 8aad1f7..858b688 100644 --- a/nostr_dvm/tasks/content_discovery_currently_popular_gallery.py +++ b/nostr_dvm/tasks/content_discovery_currently_popular_gallery.py @@ -245,36 +245,38 @@ class DicoverContentCurrentlyPopularGallery(DVMTaskInterface): return 0 async def sync_db(self): - opts = (Options().wait_for_send(False).send_timeout(timedelta(seconds=self.dvm_config.RELAY_LONG_TIMEOUT))) - sk = SecretKey.from_hex(self.dvm_config.PRIVATE_KEY) - keys = Keys.parse(sk.to_hex()) - signer = NostrSigner.keys(keys) - database = await NostrDatabase.sqlite(self.db_name) - cli = ClientBuilder().signer(signer).database(database).opts(opts).build() + try: + opts = (Options().wait_for_send(False).send_timeout(timedelta(seconds=self.dvm_config.RELAY_LONG_TIMEOUT))) + sk = SecretKey.from_hex(self.dvm_config.PRIVATE_KEY) + keys = Keys.parse(sk.to_hex()) + signer = NostrSigner.keys(keys) + database = await NostrDatabase.sqlite(self.db_name) + cli = ClientBuilder().signer(signer).database(database).opts(opts).build() - for relay in self.dvm_config.RECONCILE_DB_RELAY_LIST: - await cli.add_relay(relay) + for relay in self.dvm_config.RECONCILE_DB_RELAY_LIST: + await cli.add_relay(relay) - await cli.connect() + await cli.connect() - timestamp_since = Timestamp.now().as_secs() - self.db_since - since = Timestamp.from_secs(timestamp_since) + timestamp_since = Timestamp.now().as_secs() - self.db_since + since = Timestamp.from_secs(timestamp_since) - filter1 = Filter().kinds([definitions.EventDefinitions.KIND_NIP93_GALLERYENTRY]).since(since) # Notes, reactions, zaps - - # filter = Filter().author(keys.public_key()) - if self.dvm_config.LOGLEVEL.value >= LogLevel.DEBUG.value: - print("[" + self.dvm_config.NIP89.NAME + "] Syncing notes of the last " + str( - self.db_since) + " seconds.. this might take a while..") - dbopts = NegentropyOptions().direction(NegentropyDirection.DOWN) - await cli.reconcile(filter1, dbopts) - await cli.database().delete(Filter().until(Timestamp.from_secs( - Timestamp.now().as_secs() - self.db_since))) # Clear old events so db doesn't get too full. - await cli.shutdown() - if self.dvm_config.LOGLEVEL.value >= LogLevel.DEBUG.value: - print( - "[" + self.dvm_config.NIP89.NAME + "] Done Syncing Notes of the last " + str(self.db_since) + " seconds..") + filter1 = Filter().kinds([definitions.EventDefinitions.KIND_NIP93_GALLERYENTRY]).since(since) # Notes, reactions, zaps + # filter = Filter().author(keys.public_key()) + if self.dvm_config.LOGLEVEL.value >= LogLevel.DEBUG.value: + print("[" + self.dvm_config.NIP89.NAME + "] Syncing notes of the last " + str( + self.db_since) + " seconds.. this might take a while..") + dbopts = NegentropyOptions().direction(NegentropyDirection.DOWN) + await cli.reconcile(filter1, dbopts) + await cli.database().delete(Filter().until(Timestamp.from_secs( + Timestamp.now().as_secs() - self.db_since))) # Clear old events so db doesn't get too full. + await cli.shutdown() + if self.dvm_config.LOGLEVEL.value >= LogLevel.DEBUG.value: + print( + "[" + self.dvm_config.NIP89.NAME + "] Done Syncing Notes of the last " + str(self.db_since) + " seconds..") + except Exception as e: + print(e) # We build an example here that we can call by either calling this file directly from the main directory, # or by adding it to our playground. You can call the example and adjust it to your needs or redefine it in the diff --git a/nostr_dvm/tasks/content_discovery_currently_popular_mostr.py b/nostr_dvm/tasks/content_discovery_currently_popular_mostr.py index 50bbf6b..1e991d8 100644 --- a/nostr_dvm/tasks/content_discovery_currently_popular_mostr.py +++ b/nostr_dvm/tasks/content_discovery_currently_popular_mostr.py @@ -219,55 +219,58 @@ class DicoverContentCurrentlyPopularMostr(DVMTaskInterface): return 0 async def sync_db(self): - opts = (Options().wait_for_send(False).send_timeout(timedelta(seconds=self.dvm_config.RELAY_LONG_TIMEOUT))) - sk = SecretKey.from_hex(self.dvm_config.PRIVATE_KEY) - keys = Keys.parse(sk.to_hex()) - signer = NostrSigner.keys(keys) - database = await NostrDatabase.sqlite(self.db_name) - cli = ClientBuilder().signer(signer).database(database).opts(opts).build() - - for relay in self.dvm_config.RECONCILE_DB_RELAY_LIST: - await cli.add_relay(relay) - - await cli.connect() - - timestamp_since = Timestamp.now().as_secs() - self.db_since - since = Timestamp.from_secs(timestamp_since) - - filter1 = Filter().kinds( - [definitions.EventDefinitions.KIND_NOTE, EventDefinitions.KIND_PROFILE, EventDefinitions.KIND_ZAP, - EventDefinitions.KIND_REPOST, - EventDefinitions.KIND_REACTION]).since(since) # Notes, reactions, zaps - filter2 = Filter().kinds( - [EventDefinitions.KIND_PROFILE]).limit(10000) # Notes, reactions, zaps - filter3 = Filter().kinds( - [EventDefinitions.KIND_ZAP, - EventDefinitions.KIND_REPOST, - EventDefinitions.KIND_REACTION]).since(since) # Notes, reactions, zaps - # Notes, reactions, zaps - - # filter = Filter().author(keys.public_key()) - if self.dvm_config.LOGLEVEL.value >= LogLevel.DEBUG.value: - print("[" + self.dvm_config.NIP89.NAME + "] Syncing notes of the last " + str( - self.db_since) + " seconds.. this might take a while..") - #dbopts = NegentropyOptions().direction(NegentropyDirection.DOWN) - #await cli.reconcile(filter1, dbopts) - #await cli.reconcile(filter2, dbopts) - #await cli.reconcile(filter3, dbopts) - - # RECONCOILE NOT POSSIBLE ON THESE RELAYS SO WE FETCH AB BUNCH (will be stored in db) try: - events = await cli.get_events_of([filter1, filter2, filter3], timedelta(20)) + opts = (Options().wait_for_send(False).send_timeout(timedelta(seconds=self.dvm_config.RELAY_LONG_TIMEOUT))) + sk = SecretKey.from_hex(self.dvm_config.PRIVATE_KEY) + keys = Keys.parse(sk.to_hex()) + signer = NostrSigner.keys(keys) + database = await NostrDatabase.sqlite(self.db_name) + cli = ClientBuilder().signer(signer).database(database).opts(opts).build() + + for relay in self.dvm_config.RECONCILE_DB_RELAY_LIST: + await cli.add_relay(relay) + + await cli.connect() + + timestamp_since = Timestamp.now().as_secs() - self.db_since + since = Timestamp.from_secs(timestamp_since) + + filter1 = Filter().kinds( + [definitions.EventDefinitions.KIND_NOTE, EventDefinitions.KIND_PROFILE, EventDefinitions.KIND_ZAP, + EventDefinitions.KIND_REPOST, + EventDefinitions.KIND_REACTION]).since(since) # Notes, reactions, zaps + filter2 = Filter().kinds( + [EventDefinitions.KIND_PROFILE]).limit(10000) # Notes, reactions, zaps + filter3 = Filter().kinds( + [EventDefinitions.KIND_ZAP, + EventDefinitions.KIND_REPOST, + EventDefinitions.KIND_REACTION]).since(since) # Notes, reactions, zaps + # Notes, reactions, zaps + + # filter = Filter().author(keys.public_key()) + if self.dvm_config.LOGLEVEL.value >= LogLevel.DEBUG.value: + print("[" + self.dvm_config.NIP89.NAME + "] Syncing notes of the last " + str( + self.db_since) + " seconds.. this might take a while..") + #dbopts = NegentropyOptions().direction(NegentropyDirection.DOWN) + #await cli.reconcile(filter1, dbopts) + #await cli.reconcile(filter2, dbopts) + #await cli.reconcile(filter3, dbopts) + + # RECONCOILE NOT POSSIBLE ON THESE RELAYS SO WE FETCH AB BUNCH (will be stored in db) + try: + events = await cli.get_events_of([filter1, filter2, filter3], timedelta(20)) + except Exception as e: + print(e) + # Do not delete profiles + await cli.database().delete(Filter().kinds([EventDefinitions.KIND_NOTE, EventDefinitions.KIND_ZAP, EventDefinitions.KIND_REPOST, EventDefinitions.KIND_REACTION]).until(Timestamp.from_secs( + Timestamp.now().as_secs() - self.db_since))) # Clear old events so db doesn't get too full. + await cli.shutdown() + if self.dvm_config.LOGLEVEL.value >= LogLevel.DEBUG.value: + print( + "[" + self.dvm_config.NIP89.NAME + "] Done Syncing Notes of the last " + str( + self.db_since) + " seconds..") except Exception as e: print(e) - # Do not delete profiles - await cli.database().delete(Filter().kinds([EventDefinitions.KIND_NOTE, EventDefinitions.KIND_ZAP, EventDefinitions.KIND_REPOST, EventDefinitions.KIND_REACTION]).until(Timestamp.from_secs( - Timestamp.now().as_secs() - self.db_since))) # Clear old events so db doesn't get too full. - await cli.shutdown() - if self.dvm_config.LOGLEVEL.value >= LogLevel.DEBUG.value: - print( - "[" + self.dvm_config.NIP89.NAME + "] Done Syncing Notes of the last " + str( - self.db_since) + " seconds..") # We build an example here that we can call by either calling this file directly from the main directory, diff --git a/nostr_dvm/tasks/content_discovery_currently_popular_nonfollowers.py b/nostr_dvm/tasks/content_discovery_currently_popular_nonfollowers.py index 6381cac..8f4d92f 100644 --- a/nostr_dvm/tasks/content_discovery_currently_popular_nonfollowers.py +++ b/nostr_dvm/tasks/content_discovery_currently_popular_nonfollowers.py @@ -221,37 +221,39 @@ class DicoverContentCurrentlyPopularNonFollowers(DVMTaskInterface): return 1 async def sync_db(self): - opts = (Options().wait_for_send(False).send_timeout(timedelta(seconds=self.dvm_config.RELAY_LONG_TIMEOUT))) - sk = SecretKey.from_hex(self.dvm_config.PRIVATE_KEY) - keys = Keys.parse(sk.to_hex()) - signer = NostrSigner.keys(keys) - database = await NostrDatabase.sqlite(self.db_name) - cli = ClientBuilder().signer(signer).database(database).opts(opts).build() + try: + opts = (Options().wait_for_send(False).send_timeout(timedelta(seconds=self.dvm_config.RELAY_LONG_TIMEOUT))) + sk = SecretKey.from_hex(self.dvm_config.PRIVATE_KEY) + keys = Keys.parse(sk.to_hex()) + signer = NostrSigner.keys(keys) + database = await NostrDatabase.sqlite(self.db_name) + cli = ClientBuilder().signer(signer).database(database).opts(opts).build() - for relay in self.dvm_config.RECONCILE_DB_RELAY_LIST: - await cli.add_relay(relay) + for relay in self.dvm_config.RECONCILE_DB_RELAY_LIST: + await cli.add_relay(relay) - await cli.connect() + await cli.connect() - timestamp_since = Timestamp.now().as_secs() - self.db_since - since = Timestamp.from_secs(timestamp_since) + timestamp_since = Timestamp.now().as_secs() - self.db_since + since = Timestamp.from_secs(timestamp_since) - filter1 = Filter().kinds([definitions.EventDefinitions.KIND_NOTE, definitions.EventDefinitions.KIND_REACTION, - definitions.EventDefinitions.KIND_ZAP]).since(since) # Notes, reactions, zaps - - # filter = Filter().author(keys.public_key()) - if self.dvm_config.LOGLEVEL.value >= LogLevel.DEBUG.value: - print("[" + self.dvm_config.NIP89.NAME + "] Syncing notes of the last " + str( - self.db_since) + " seconds.. this might take a while..") - dbopts = NegentropyOptions().direction(NegentropyDirection.DOWN) - await cli.reconcile(filter1, dbopts) - await cli.database().delete(Filter().until(Timestamp.from_secs( - Timestamp.now().as_secs() - self.db_since))) # Clear old events so db doesn't get too full. - await cli.shutdown() - if self.dvm_config.LOGLEVEL.value >= LogLevel.DEBUG.value: - print( - "[" + self.dvm_config.NIP89.NAME + "] Done Syncing Notes of the last " + str(self.db_since) + " seconds..") + filter1 = Filter().kinds([definitions.EventDefinitions.KIND_NOTE, definitions.EventDefinitions.KIND_REACTION, + definitions.EventDefinitions.KIND_ZAP]).since(since) # Notes, reactions, zaps + # filter = Filter().author(keys.public_key()) + if self.dvm_config.LOGLEVEL.value >= LogLevel.DEBUG.value: + print("[" + self.dvm_config.NIP89.NAME + "] Syncing notes of the last " + str( + self.db_since) + " seconds.. this might take a while..") + dbopts = NegentropyOptions().direction(NegentropyDirection.DOWN) + await cli.reconcile(filter1, dbopts) + await cli.database().delete(Filter().until(Timestamp.from_secs( + Timestamp.now().as_secs() - self.db_since))) # Clear old events so db doesn't get too full. + await cli.shutdown() + if self.dvm_config.LOGLEVEL.value >= LogLevel.DEBUG.value: + print( + "[" + self.dvm_config.NIP89.NAME + "] Done Syncing Notes of the last " + str(self.db_since) + " seconds..") + except Exception as e: + print(e) # We build an example here that we can call by either calling this file directly from the main directory, # or by adding it to our playground. You can call the example and adjust it to your needs or redefine it in the diff --git a/nostr_dvm/tasks/content_discovery_currently_popular_topic.py b/nostr_dvm/tasks/content_discovery_currently_popular_topic.py index f61749c..c59c924 100644 --- a/nostr_dvm/tasks/content_discovery_currently_popular_topic.py +++ b/nostr_dvm/tasks/content_discovery_currently_popular_topic.py @@ -188,37 +188,39 @@ class DicoverContentCurrentlyPopularbyTopic(DVMTaskInterface): return 1 async def sync_db(self): - opts = (Options().wait_for_send(False).send_timeout(timedelta(seconds=self.dvm_config.RELAY_LONG_TIMEOUT))) - sk = SecretKey.from_hex(self.dvm_config.PRIVATE_KEY) - keys = Keys.parse(sk.to_hex()) - signer = NostrSigner.keys(keys) - database = await NostrDatabase.sqlite(self.db_name) - cli = ClientBuilder().signer(signer).database(database).opts(opts).build() + try: + opts = (Options().wait_for_send(False).send_timeout(timedelta(seconds=self.dvm_config.RELAY_LONG_TIMEOUT))) + sk = SecretKey.from_hex(self.dvm_config.PRIVATE_KEY) + keys = Keys.parse(sk.to_hex()) + signer = NostrSigner.keys(keys) + database = await NostrDatabase.sqlite(self.db_name) + cli = ClientBuilder().signer(signer).database(database).opts(opts).build() - for relay in self.dvm_config.RECONCILE_DB_RELAY_LIST: - await cli.add_relay(relay) + for relay in self.dvm_config.RECONCILE_DB_RELAY_LIST: + await cli.add_relay(relay) - await cli.connect() + await cli.connect() - timestamp_since = Timestamp.now().as_secs() - self.db_since - since = Timestamp.from_secs(timestamp_since) + timestamp_since = Timestamp.now().as_secs() - self.db_since + since = Timestamp.from_secs(timestamp_since) - filter1 = Filter().kinds([definitions.EventDefinitions.KIND_NOTE, definitions.EventDefinitions.KIND_REACTION, - definitions.EventDefinitions.KIND_ZAP]).since(since) # Notes, reactions, zaps + filter1 = Filter().kinds([definitions.EventDefinitions.KIND_NOTE, definitions.EventDefinitions.KIND_REACTION, + definitions.EventDefinitions.KIND_ZAP]).since(since) # Notes, reactions, zaps - # filter = Filter().author(keys.public_key()) - if self.dvm_config.LOGLEVEL.value >= LogLevel.DEBUG.value: - print("[" + self.dvm_config.NIP89.NAME + "] Syncing notes of the last " + str( - self.db_since) + " seconds.. this might take a while..") - dbopts = NegentropyOptions().direction(NegentropyDirection.DOWN) - await cli.reconcile(filter1, dbopts) - await cli.database().delete(Filter().until(Timestamp.from_secs( - Timestamp.now().as_secs() - self.db_since))) # Clear old events so db doesn't get too full. - await cli.shutdown() - if self.dvm_config.LOGLEVEL.value >= LogLevel.DEBUG.value: - print( - "[" + self.dvm_config.NIP89.NAME + "] Done Syncing Notes of the last " + str(self.db_since) + " seconds..") + # filter = Filter().author(keys.public_key()) + if self.dvm_config.LOGLEVEL.value >= LogLevel.DEBUG.value: + print("[" + self.dvm_config.NIP89.NAME + "] Syncing notes of the last " + str( + self.db_since) + " seconds.. this might take a while..") + dbopts = NegentropyOptions().direction(NegentropyDirection.DOWN) + await cli.reconcile(filter1, dbopts) + await cli.database().delete(Filter().until(Timestamp.from_secs( + Timestamp.now().as_secs() - self.db_since))) # Clear old events so db doesn't get too full. + await cli.shutdown() + if self.dvm_config.LOGLEVEL.value >= LogLevel.DEBUG.value: + print("[" + self.dvm_config.NIP89.NAME + "] Done Syncing Notes of the last " + str(self.db_since) + " seconds..") + except Exception as e: + print(e) # We build an example here that we can call by either calling this file directly from the main directory, # or by adding it to our playground. You can call the example and adjust it to your needs or redefine it in the diff --git a/setup.py b/setup.py index be3e9f7..f269373 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -VERSION = '0.8.6' +VERSION = '0.8.7' DESCRIPTION = 'A framework to build and run Nostr NIP90 Data Vending Machines' LONG_DESCRIPTION = ('A framework to build and run Nostr NIP90 Data Vending Machines. See the github repository for more information')