From 9efa5a60f2a455445f8bff88e58ca3c6442d0274 Mon Sep 17 00:00:00 2001 From: Vishal <64505169+vishalxl@users.noreply.github.com> Date: Sat, 24 Dec 2022 11:22:57 +0530 Subject: [PATCH] improved fetching to get better response from relays reduced days fetched. combined calls. --- bin/nostr_console.dart | 44 ++++++++++++++++++++++++------------------ lib/event_ds.dart | 2 +- lib/relays.dart | 10 +++++----- lib/settings.dart | 23 +++++++++++++++++----- lib/user.dart | 24 +++++++++++++++++++++++ lib/utils.dart | 18 ++++++++++------- 6 files changed, 84 insertions(+), 37 deletions(-) diff --git a/bin/nostr_console.dart b/bin/nostr_console.dart index 5a15c7c..b81c0bd 100644 --- a/bin/nostr_console.dart +++ b/bin/nostr_console.dart @@ -254,13 +254,13 @@ Future main(List arguments) async { } int limitSelfEvents = 200; - int limitOthersEvents = 4; + int limitOthersEvents = 3; int limitPerSubscription = gLimitPerSubscription; // if more than 1000 posts have already been read from the file, then don't get too many day's events. Only for last 3 days. if(numFileEvents > 1000) { - limitSelfEvents = 10; - limitOthersEvents = 3; + limitSelfEvents = 4; + limitOthersEvents = 2; gDefaultNumWaitSeconds = gDefaultNumWaitSeconds ~/5; } else { printInfoForNewUser(); @@ -279,7 +279,7 @@ Future main(List arguments) async { } if( userPublicKey!= "") { - getIdAndMentionEvents(gListRelayUrls1, {userPublicKey}, limitPerSubscription, getSecondsDaysAgo(limitSelfEvents), "#p"); + getIdAndMentionEvents(gListRelayUrls1, {userPublicKey}, limitPerSubscription, getSecondsDaysAgo(limitSelfEvents), getSecondsDaysAgo(limitSelfEvents), "#p", "authors"); } Future.delayed(Duration(milliseconds: numWaitSeconds), () { @@ -308,15 +308,15 @@ Future main(List arguments) async { // get event for user if( userPublicKey!= "") { - getIdAndMentionEvents(gListRelayUrls1, {userPublicKey}, limitPerSubscription, getSecondsDaysAgo(limitSelfEvents), "#p"); + //getIdAndMentionEvents(gListRelayUrls1, {userPublicKey}, limitPerSubscription, 0, getSecondsDaysAgo(limitSelfEvents), "#p", "authors"); + //getIdAndMentionEvents(gListRelayUrls2, {userPublicKey}, limitPerSubscription, getSecondsDaysAgo(limitSelfEvents), getSecondsDaysAgo(limitSelfEvents), "#p", "authors"); + getUserEvents(gListRelayUrls1, userPublicKey, limitPerSubscription, getSecondsDaysAgo(limitSelfEvents)); + getMentionEvents(gListRelayUrls1, {userPublicKey}, limitPerSubscription, getSecondsDaysAgo(limitSelfEvents), "#p"); + } - // get group and meta info events - getKindEvents([40, 41], gListRelayUrls1, limitPerSubscription, getSecondsDaysAgo(limitSelfEvents)); - getKindEvents([42], gListRelayUrls1, 3 * limitPerSubscription, getSecondsDaysAgo(limitOthersEvents)); - // get default users; remove user from default list if user exists in it. because theyv'e already been fetched. - getMultiUserEvents(gListRelayUrls1, gDefaultFollows.difference({userPublicKey}), 4 * limitPerSubscription, getSecondsDaysAgo(limitOthersEvents)); + getMultiUserEvents(gListRelayUrls2, gDefaultFollows.difference({userPublicKey}), 4 * limitPerSubscription, getSecondsDaysAgo(limitOthersEvents)); Set usersFetched = gDefaultFollows.union({userPublicKey}); stdout.write('Waiting for user posts to come in.....'); @@ -324,12 +324,21 @@ Future main(List arguments) async { initialEvents.addAll(getRecievedEvents()); clearEvents(); + initialEvents.forEach((element) { element.eventData.kind == 1? numUserPosts++: numUserPosts;}); numUserPosts -= numFilePosts; stdout.write("...done\n");//received $numUserPosts new posts made by the user\n"); - initialEvents.forEach((e) => processKind3Event(e)); // first process the kind 3 event - + Set userEvents = getOnlyUserEvents(initialEvents, userPublicKey); + print('Total events fetched till now: ${initialEvents.length}. Total user events fetched: ${userEvents.length}'); + + // get events from channels of user + Set userChannels = getUserChannels(initialEvents, userPublicKey); + //printSet(userChannels, "user channels: \n", "\n"); + getIdAndMentionEvents(gListRelayUrls2, userChannels, limitPerSubscription, 0, getSecondsDaysAgo(limitOthersEvents), "#e", "ids"); + + initialEvents.forEach((e) => processKind3Event(e)); // first process the kind 3 event ; basically populate the global structure that holds this info + Set contacts = {}; Set pTags = {}; @@ -354,23 +363,20 @@ Future main(List arguments) async { pTags = getpTags(initialEvents, gMaxPtagsToGet); } - // get only 200 contacts maximum - int maxContactsFetched = 200; + // get only limited number of contacts otherwise relays get less responsive + int maxContactsFetched = 100; if( contacts.length > maxContactsFetched) { int i = 0; - contacts.retainWhere((element) => i++ > 200); // retain only first 200, whichever they may be + contacts.retainWhere((element) => i++ > maxContactsFetched); // retain only first 200, whichever they may be } getMultiUserEvents(gListRelayUrls1, contacts.union(pTags).difference(usersFetched), 4 * limitPerSubscription, getSecondsDaysAgo(limitOthersEvents)); usersFetched = usersFetched.union(contacts).union(pTags); // get meta events of all users fetched - //print("getting meta for # users : ${usersFetched.length} #contacts = ${contacts.length}"); getMultiUserEvents(gListRelayUrls1, usersFetched, 4 * limitPerSubscription, getSecondsDaysAgo(limitSelfEvents*2), {0,3}); + //print("fetched meta of ${usersFetched.length}"); - // get events from channels of user - Set userChannels = getUserChannels(initialEvents, userPublicKey); - //getMentionEvents(gListRelayUrls1, userChannels, limitPerSubscription, getSecondsDaysAgo(limitSelfEvents), "#e"); stdout.write('Waiting for feed to come in..............'); Future.delayed(Duration(milliseconds: gDefaultNumWaitSeconds * 1), () { diff --git a/lib/event_ds.dart b/lib/event_ds.dart index dd8828d..fddf5c3 100644 --- a/lib/event_ds.dart +++ b/lib/event_ds.dart @@ -368,7 +368,7 @@ class EventData { } // is called only once for each event received ( or read from file) - void translateAndExpandMentions(Map tempChildEventsMap) { + void translateAndExpandMentions(Map tempChildEventsMap) { if( id == gCheckEventId) { //printInColor("in translateAndExpandMentions: decoding $gCheckEventId\n", redColor); } diff --git a/lib/relays.dart b/lib/relays.dart index 7c4ccf9..d031b7f 100644 --- a/lib/relays.dart +++ b/lib/relays.dart @@ -102,10 +102,10 @@ class Relays { sendRequest(relayUrl, request); } - void getIdAndMentionEvents(String relayUrl, Set ids, int limit, int sinceWhen, String tagToGet) { + void getIdAndMentionEvents(String relayUrl, Set ids, int limit, int idSinceWhen, int mentionSinceWhen, String tagToGet, String idType) { - String subscriptionId = "id_mention_${tagToGet}" + (relays[relayUrl]?.numRequestsSent??"").toString() + "_" + relayUrl.substring(6); - String request = getIdAndMentionRequest(subscriptionId, ids, limit, sinceWhen, tagToGet); + String subscriptionId = "id_mention_tag" + (relays[relayUrl]?.numRequestsSent??"").toString() + "_" + relayUrl.substring(6); + String request = getIdAndMentionRequest(subscriptionId, ids, limit, idSinceWhen, mentionSinceWhen, tagToGet, idType); sendRequest(relayUrl, request); } @@ -263,9 +263,9 @@ void getMentionEvents(Set serverUrls, Set ids, int numUserEvents }); } -void getIdAndMentionEvents(Set serverUrls, Set ids, int numUserEvents, int sinceWhen, String tagToGet) { +void getIdAndMentionEvents(Set serverUrls, Set ids, int numUserEvents, int idSinceWhen, int mentionSinceWhen, String tagToGet, String idType) { serverUrls.forEach((serverUrl) { - relays.getIdAndMentionEvents(serverUrl, ids, numUserEvents, sinceWhen, tagToGet); + relays.getIdAndMentionEvents(serverUrl, ids, numUserEvents, idSinceWhen, mentionSinceWhen, tagToGet, idType); }); } diff --git a/lib/settings.dart b/lib/settings.dart index ea48a4b..3c20680 100644 --- a/lib/settings.dart +++ b/lib/settings.dart @@ -40,7 +40,7 @@ const int gLimitPerSubscription = 20000; // applicable only for notifications and not for search results. Search results set a flag in EventData and don't use this variable const int gDontHighlightEventsOlderThan = 4; -int gDefaultNumWaitSeconds = 10000; // is used in main() +int gDefaultNumWaitSeconds = 12000; // is used in main() const int gMaxAuthorsInOneRequest = 300; // number of author requests to send in one request const int gMaxPtagsToGet = 100; // maximum number of p tags that are taken from the comments of feed ( the top most, most frequent) @@ -50,20 +50,33 @@ int numFileEvents = 0, numFilePosts = 0, numUserPosts = 0, numFeedPosts = 0, num String defaultServerUrl = "wss://relay.damus.io"; const String relayNostrInfo = 'wss://relay.nostr.info'; + + Set gListRelayUrls1 = { defaultServerUrl, relayNostrInfo, "wss://nostr-2.zebedee.cloud", "wss://nostr.semisol.dev", - "wss://nostr.onsats.org", - "wss://nostr.coinos.io" + "wss://nostr.coinos.io", + "wss://nostr-relay.digitalmob.ro", + "wss://nostr.drss.io" }; Set gListRelayUrls2 = { - "wss://nostr.oxtr.dev", - "wss://nostr.bitcoiner.social" + // "wss://nostr.oxtr.dev", + "wss://nostr.bitcoiner.social", + "wss://nostr.zerofeerouting.com", + "wss://nostr-relay.trustbtc.org", + "wss://relay.stoner.com" }; +Set gListRelayUrls3 = { + "wss://nostr.onsats.org", + "wss://relay.stoner.com", + "wss://nostr.openchain.fr" + }; + + // well known disposable test private key const String gDefaultPublicKey = ""; String userPrivateKey = ""; diff --git a/lib/user.dart b/lib/user.dart index 98412fa..4e6db58 100644 --- a/lib/user.dart +++ b/lib/user.dart @@ -1,5 +1,6 @@ import 'package:nostr_console/event_ds.dart'; +import 'package:nostr_console/settings.dart'; import 'package:nostr_console/utils.dart'; // From the list of events provided, lookup the lastst contact information for the given user/pubkey @@ -17,6 +18,19 @@ Event? getContactEvent(String pubkey) { Set getUserChannels(Set userEvents, String userPublicKey) { Set userChannels = {}; + userEvents.forEach((event) { + if( event.eventData.pubkey == userPublicKey) { + if( event.eventData.kind == 42) { + String channelId = event.eventData.getChannelIdForKind4x(); + if( channelId.length == 64) { + userChannels.add(channelId); + } + } else if([40,41].contains(event.eventData.kind)) { + userChannels.add(event.eventData.id); + } + } + }); + return userChannels; } @@ -55,4 +69,14 @@ Set getpTags(Set events, int numMostFrequent) { } return ptags.toSet(); +} + +Set getOnlyUserEvents(Set initialEvents, String userPubkey) { + Set userEvents = {}; + initialEvents.forEach((event) { + if( event.eventData.pubkey == userPubkey) { + userEvents.add(event.eventData.id); + } + }); + return userEvents; } \ No newline at end of file diff --git a/lib/utils.dart b/lib/utils.dart index aa13e6b..b867f2f 100644 --- a/lib/utils.dart +++ b/lib/utils.dart @@ -451,15 +451,19 @@ String getMentionRequest(String subscriptionId, Set ids, int numUserEven return strSubscription1 + getCommaSeparatedQuotedStrs(ids) + strSubscription2; } -String getIdAndMentionRequest(String subscriptionId, Set ids, int numUserEvents, int sinceWhen, String tagToGet) { - String strTime = ""; - if( sinceWhen != 0) { - strTime = ', "since": ${sinceWhen.toString()}'; +String getIdAndMentionRequest(String subscriptionId, Set ids, int numUserEvents, int idSinceWhen, int mentionSinceWhen, String tagToGet, String idString) { + String idStrTime = "", mentionStrTime = ""; + if( idSinceWhen != 0) { + idStrTime = ', "since": ${idSinceWhen.toString()}'; } - + + if( mentionSinceWhen != 0) { + mentionStrTime = ', "since": ${mentionSinceWhen.toString()}'; + } + var strSubscription1 = '["REQ","$subscriptionId",{ "$tagToGet": ['; - var strSubscription2 ='], "limit": $numUserEvents $strTime } ]'; - String req = '["REQ","$subscriptionId",{ "$tagToGet": [' + getCommaSeparatedQuotedStrs(ids) + '], "limit": $numUserEvents $strTime},{"authors":[' + getCommaSeparatedQuotedStrs(ids) + ']} ]'; + var strSubscription2 ='], "limit": $numUserEvents $idStrTime } ]'; + String req = '["REQ","$subscriptionId",{ "$tagToGet": [' + getCommaSeparatedQuotedStrs(ids) + '], "limit": $numUserEvents $mentionStrTime},{"$idString":[' + getCommaSeparatedQuotedStrs(ids) + ']$idStrTime}]'; //print("Created id and mention request: $req"); return req; }