highperfocused/nostr_console
1
0
Fork 0
mirror of https://github.com/vishalxl/nostr_console.git synced 2025-03-27 02:01:51 +01:00
Code Issues Packages Projects Releases Wiki Activity

improved fetching to get better response from relays

Browse Source 
reduced days fetched.

combined calls.
This commit is contained in:
Vishal 2022-12-24 11:22:57 +05:30
parent d393acdef8
commit 9efa5a60f2
6 changed files with 84 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
bin/nostr_console.dart
View File

@ -254,13 +254,13 @@ Future<void> main(List<String> arguments) async {
}
int limitSelfEvents = 200;
int limitOthersEvents = 4;
int limitOthersEvents = 3;
int limitPerSubscription = gLimitPerSubscription;
// if more than 1000 posts have already been read from the file, then don't get too many day's events. Only for last 3 days.
if(numFileEvents > 1000) {
limitSelfEvents = 10;
limitOthersEvents = 3;
limitSelfEvents = 4;
limitOthersEvents = 2;
gDefaultNumWaitSeconds = gDefaultNumWaitSeconds ~/5;
} else {
printInfoForNewUser();
@ -279,7 +279,7 @@ Future<void> main(List<String> arguments) async {
}
if( userPublicKey!= "") {
getIdAndMentionEvents(gListRelayUrls1, {userPublicKey}, limitPerSubscription, getSecondsDaysAgo(limitSelfEvents), "#p");
getIdAndMentionEvents(gListRelayUrls1, {userPublicKey}, limitPerSubscription, getSecondsDaysAgo(limitSelfEvents), getSecondsDaysAgo(limitSelfEvents), "#p", "authors");
}
Future.delayed(Duration(milliseconds: numWaitSeconds), () {
@ -308,15 +308,15 @@ Future<void> main(List<String> arguments) async {
// get event for user
if( userPublicKey!= "") {
getIdAndMentionEvents(gListRelayUrls1, {userPublicKey}, limitPerSubscription, getSecondsDaysAgo(limitSelfEvents), "#p");
//getIdAndMentionEvents(gListRelayUrls1, {userPublicKey}, limitPerSubscription, 0, getSecondsDaysAgo(limitSelfEvents), "#p", "authors");
//getIdAndMentionEvents(gListRelayUrls2, {userPublicKey}, limitPerSubscription, getSecondsDaysAgo(limitSelfEvents), getSecondsDaysAgo(limitSelfEvents), "#p", "authors");
getUserEvents(gListRelayUrls1, userPublicKey, limitPerSubscription, getSecondsDaysAgo(limitSelfEvents));
getMentionEvents(gListRelayUrls1, {userPublicKey}, limitPerSubscription, getSecondsDaysAgo(limitSelfEvents), "#p");
}
// get group and meta info events
getKindEvents([40, 41], gListRelayUrls1, limitPerSubscription, getSecondsDaysAgo(limitSelfEvents));
getKindEvents([42], gListRelayUrls1, 3 * limitPerSubscription, getSecondsDaysAgo(limitOthersEvents));
// get default users; remove user from default list if user exists in it. because theyv'e already been fetched.
getMultiUserEvents(gListRelayUrls1, gDefaultFollows.difference({userPublicKey}), 4 * limitPerSubscription, getSecondsDaysAgo(limitOthersEvents));
getMultiUserEvents(gListRelayUrls2, gDefaultFollows.difference({userPublicKey}), 4 * limitPerSubscription, getSecondsDaysAgo(limitOthersEvents));
Set<String> usersFetched = gDefaultFollows.union({userPublicKey});
stdout.write('Waiting for user posts to come in.....');
@ -324,12 +324,21 @@ Future<void> main(List<String> arguments) async {
initialEvents.addAll(getRecievedEvents());
clearEvents();
initialEvents.forEach((element) { element.eventData.kind == 1? numUserPosts++: numUserPosts;});
numUserPosts -= numFilePosts;
stdout.write("...done\n");//received $numUserPosts new posts made by the user\n");
initialEvents.forEach((e) => processKind3Event(e)); // first process the kind 3 event
Set<String> userEvents = getOnlyUserEvents(initialEvents, userPublicKey);
print('Total events fetched till now: ${initialEvents.length}. Total user events fetched: ${userEvents.length}');
// get events from channels of user
Set<String> userChannels = getUserChannels(initialEvents, userPublicKey);
//printSet(userChannels, "user channels: \n", "\n");
getIdAndMentionEvents(gListRelayUrls2, userChannels, limitPerSubscription, 0, getSecondsDaysAgo(limitOthersEvents), "#e", "ids");
initialEvents.forEach((e) => processKind3Event(e)); // first process the kind 3 event ; basically populate the global structure that holds this info
Set<String> contacts = {};
Set<String> pTags = {};
@ -354,23 +363,20 @@ Future<void> main(List<String> arguments) async {
pTags = getpTags(initialEvents, gMaxPtagsToGet);
}
// get only 200 contacts maximum
int maxContactsFetched = 200;
// get only limited number of contacts otherwise relays get less responsive
int maxContactsFetched = 100;
if( contacts.length > maxContactsFetched) {
int i = 0;
contacts.retainWhere((element) => i++ > 200); // retain only first 200, whichever they may be
contacts.retainWhere((element) => i++ > maxContactsFetched); // retain only first 200, whichever they may be
}
getMultiUserEvents(gListRelayUrls1, contacts.union(pTags).difference(usersFetched), 4 * limitPerSubscription, getSecondsDaysAgo(limitOthersEvents));
usersFetched = usersFetched.union(contacts).union(pTags);
// get meta events of all users fetched
//print("getting meta for # users : ${usersFetched.length} #contacts = ${contacts.length}");
getMultiUserEvents(gListRelayUrls1, usersFetched, 4 * limitPerSubscription, getSecondsDaysAgo(limitSelfEvents*2), {0,3});
//print("fetched meta of ${usersFetched.length}");
// get events from channels of user
Set<String> userChannels = getUserChannels(initialEvents, userPublicKey);
//getMentionEvents(gListRelayUrls1, userChannels, limitPerSubscription, getSecondsDaysAgo(limitSelfEvents), "#e");
stdout.write('Waiting for feed to come in..............');
Future.delayed(Duration(milliseconds: gDefaultNumWaitSeconds * 1), () {

2
lib/event_ds.dart
View File

@ -368,7 +368,7 @@ class EventData {
}
// is called only once for each event received ( or read from file)
void translateAndExpandMentions(Map<String, Tree> tempChildEventsMap) {
void translateAndExpandMentions(Map<String, Tree> tempChildEventsMap) {
if( id == gCheckEventId) {
//printInColor("in translateAndExpandMentions: decoding $gCheckEventId\n", redColor);
}

10
lib/relays.dart
View File

@ -102,10 +102,10 @@ class Relays {
sendRequest(relayUrl, request);
}
void getIdAndMentionEvents(String relayUrl, Set<String> ids, int limit, int sinceWhen, String tagToGet) {
void getIdAndMentionEvents(String relayUrl, Set<String> ids, int limit, int idSinceWhen, int mentionSinceWhen, String tagToGet, String idType) {
String subscriptionId = "id_mention_${tagToGet}" + (relays[relayUrl]?.numRequestsSent??"").toString() + "_" + relayUrl.substring(6);
String request = getIdAndMentionRequest(subscriptionId, ids, limit, sinceWhen, tagToGet);
String subscriptionId = "id_mention_tag" + (relays[relayUrl]?.numRequestsSent??"").toString() + "_" + relayUrl.substring(6);
String request = getIdAndMentionRequest(subscriptionId, ids, limit, idSinceWhen, mentionSinceWhen, tagToGet, idType);
sendRequest(relayUrl, request);
}
@ -263,9 +263,9 @@ void getMentionEvents(Set<String> serverUrls, Set<String> ids, int numUserEvents
});
}
void getIdAndMentionEvents(Set<String> serverUrls, Set<String> ids, int numUserEvents, int sinceWhen, String tagToGet) {
void getIdAndMentionEvents(Set<String> serverUrls, Set<String> ids, int numUserEvents, int idSinceWhen, int mentionSinceWhen, String tagToGet, String idType) {
serverUrls.forEach((serverUrl) {
relays.getIdAndMentionEvents(serverUrl, ids, numUserEvents, sinceWhen, tagToGet);
relays.getIdAndMentionEvents(serverUrl, ids, numUserEvents, idSinceWhen, mentionSinceWhen, tagToGet, idType);
});
}

23
lib/settings.dart
View File

@ -40,7 +40,7 @@ const int gLimitPerSubscription = 20000;
// applicable only for notifications and not for search results. Search results set a flag in EventData and don't use this variable
const int gDontHighlightEventsOlderThan = 4;
int gDefaultNumWaitSeconds = 10000; // is used in main()
int gDefaultNumWaitSeconds = 12000; // is used in main()
const int gMaxAuthorsInOneRequest = 300; // number of author requests to send in one request
const int gMaxPtagsToGet = 100; // maximum number of p tags that are taken from the comments of feed ( the top most, most frequent)
@ -50,20 +50,33 @@ int numFileEvents = 0, numFilePosts = 0, numUserPosts = 0, numFeedPosts = 0, num
String defaultServerUrl = "wss://relay.damus.io";
const String relayNostrInfo = 'wss://relay.nostr.info';
Set<String> gListRelayUrls1 = { defaultServerUrl,
relayNostrInfo,
"wss://nostr-2.zebedee.cloud",
"wss://nostr.semisol.dev",
"wss://nostr.onsats.org",
"wss://nostr.coinos.io"
"wss://nostr.coinos.io",
"wss://nostr-relay.digitalmob.ro",
"wss://nostr.drss.io"
};
Set<String> gListRelayUrls2 = {
"wss://nostr.oxtr.dev",
"wss://nostr.bitcoiner.social"
// "wss://nostr.oxtr.dev",
"wss://nostr.bitcoiner.social",
"wss://nostr.zerofeerouting.com",
"wss://nostr-relay.trustbtc.org",
"wss://relay.stoner.com"
};
Set<String> gListRelayUrls3 = {
"wss://nostr.onsats.org",
"wss://relay.stoner.com",
"wss://nostr.openchain.fr"
};
// well known disposable test private key
const String gDefaultPublicKey = "";
String userPrivateKey = "";

24
lib/user.dart
View File

@ -1,5 +1,6 @@
import 'package:nostr_console/event_ds.dart';
import 'package:nostr_console/settings.dart';
import 'package:nostr_console/utils.dart';
// From the list of events provided, lookup the lastst contact information for the given user/pubkey
@ -17,6 +18,19 @@ Event? getContactEvent(String pubkey) {
Set<String> getUserChannels(Set<Event> userEvents, String userPublicKey) {
Set<String> userChannels = {};
userEvents.forEach((event) {
if( event.eventData.pubkey == userPublicKey) {
if( event.eventData.kind == 42) {
String channelId = event.eventData.getChannelIdForKind4x();
if( channelId.length == 64) {
userChannels.add(channelId);
}
} else if([40,41].contains(event.eventData.kind)) {
userChannels.add(event.eventData.id);
}
}
});
return userChannels;
}
@ -55,4 +69,14 @@ Set<String> getpTags(Set<Event> events, int numMostFrequent) {
}
return ptags.toSet();
}
Set<String> getOnlyUserEvents(Set<Event> initialEvents, String userPubkey) {
Set<String> userEvents = {};
initialEvents.forEach((event) {
if( event.eventData.pubkey == userPubkey) {
userEvents.add(event.eventData.id);
}
});
return userEvents;
}

18
lib/utils.dart
View File

@ -451,15 +451,19 @@ String getMentionRequest(String subscriptionId, Set<String> ids, int numUserEven
return strSubscription1 + getCommaSeparatedQuotedStrs(ids) + strSubscription2;
}
String getIdAndMentionRequest(String subscriptionId, Set<String> ids, int numUserEvents, int sinceWhen, String tagToGet) {
String strTime = "";
if( sinceWhen != 0) {
strTime = ', "since": ${sinceWhen.toString()}';
String getIdAndMentionRequest(String subscriptionId, Set<String> ids, int numUserEvents, int idSinceWhen, int mentionSinceWhen, String tagToGet, String idString) {
String idStrTime = "", mentionStrTime = "";
if( idSinceWhen != 0) {
idStrTime = ', "since": ${idSinceWhen.toString()}';
}
if( mentionSinceWhen != 0) {
mentionStrTime = ', "since": ${mentionSinceWhen.toString()}';
}
var strSubscription1 = '["REQ","$subscriptionId",{ "$tagToGet": [';
var strSubscription2 ='], "limit": $numUserEvents $strTime } ]';
String req = '["REQ","$subscriptionId",{ "$tagToGet": [' + getCommaSeparatedQuotedStrs(ids) + '], "limit": $numUserEvents $strTime},{"authors":[' + getCommaSeparatedQuotedStrs(ids) + ']} ]';
var strSubscription2 ='], "limit": $numUserEvents $idStrTime } ]';
String req = '["REQ","$subscriptionId",{ "$tagToGet": [' + getCommaSeparatedQuotedStrs(ids) + '], "limit": $numUserEvents $mentionStrTime},{"$idString":[' + getCommaSeparatedQuotedStrs(ids) + ']$idStrTime}]';
//print("Created id and mention request: $req");
return req;
}