highperfocused/nostr_console
1
0
Fork 0
mirror of https://github.com/vishalxl/nostr_console.git synced 2025-04-25 16:00:43 +02:00
Code Issues Packages Projects Releases Wiki Activity

getting only 200 maximum contacts for any user

Browse Source 
reduced calls made. combined user fetch and mention fetch to one request.
This commit is contained in:
Vishal 2022-12-24 01:35:33 +05:30
parent 91fa45a8c6
commit 0b1bd5d9be
7 changed files with 83 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
bin/nostr_console.dart
View File

@ -259,7 +259,6 @@ Future<void> main(List<String> arguments) async {
// if more than 1000 posts have already been read from the file, then don't get too many day's events. Only for last 3 days.
if(numFileEvents > 1000) {
limitPerSubscription = 5000;
limitSelfEvents = 10;
limitOthersEvents = 3;
gDefaultNumWaitSeconds = gDefaultNumWaitSeconds ~/5;
@ -280,8 +279,7 @@ Future<void> main(List<String> arguments) async {
}
if( userPublicKey!= "") {
getUserEvents(gListRelayUrls1, userPublicKey, limitPerSubscription, getSecondsDaysAgo(limitSelfEvents));
getMentionEvents(gListRelayUrls1, {userPublicKey}, limitPerSubscription, getSecondsDaysAgo(limitSelfEvents), "#p");
getIdAndMentionEvents(gListRelayUrls1, {userPublicKey}, limitPerSubscription, getSecondsDaysAgo(limitSelfEvents), "#p");
}
Future.delayed(Duration(milliseconds: numWaitSeconds), () {
@ -310,29 +308,19 @@ Future<void> main(List<String> arguments) async {
// get event for user
if( userPublicKey!= "") {
getUserEvents(gListRelayUrls1, userPublicKey, limitPerSubscription, getSecondsDaysAgo(limitSelfEvents));
getMentionEvents(gListRelayUrls1, {userPublicKey}, limitPerSubscription, getSecondsDaysAgo(limitSelfEvents), "#p");
getIdAndMentionEvents(gListRelayUrls1, {userPublicKey}, limitPerSubscription, getSecondsDaysAgo(limitSelfEvents), "#p");
}
//getKindEvents([gSecretMessageKind], gListRelayUrls1, limitPerSubscription, getSecondsDaysAgo( limitSelfEvents));
Set<String> usersFetched = {userPublicKey};
// remove user from default list if he exists in it. because theyv'e already been fetched.
gDefaultFollows = gDefaultFollows.difference(usersFetched);
// get other user events
getMultiUserEvents(gListRelayUrls1, gDefaultFollows, 4 * limitPerSubscription, getSecondsDaysAgo(limitOthersEvents));
usersFetched = usersFetched.union(gDefaultFollows);
// get group and meta info events
getKindEvents([40, 41], gListRelayUrls1, limitPerSubscription, getSecondsDaysAgo(limitSelfEvents));
getKindEvents([42], gListRelayUrls1, 3 * limitPerSubscription, getSecondsDaysAgo(4));
getKindEvents([42], gListRelayUrls1, 3 * limitPerSubscription, getSecondsDaysAgo(limitOthersEvents));
getMultiUserEvents(gListRelayUrls1, usersFetched, 4 * limitPerSubscription, getSecondsDaysAgo(limitSelfEvents), {0,3});
// get default users; remove user from default list if user exists in it. because theyv'e already been fetched.
getMultiUserEvents(gListRelayUrls1, gDefaultFollows.difference({userPublicKey}), 4 * limitPerSubscription, getSecondsDaysAgo(limitOthersEvents));
Set<String> usersFetched = gDefaultFollows.union({userPublicKey});
stdout.write('Waiting for user posts to come in.....');
Future.delayed( Duration(milliseconds: gDefaultNumWaitSeconds), () {
//print("total users fetched: ${usersFetched.length}");
initialEvents.addAll(getRecievedEvents());
clearEvents();
@ -343,35 +331,43 @@ Future<void> main(List<String> arguments) async {
initialEvents.forEach((e) => processKind3Event(e)); // first process the kind 3 event
Set<String> contacts = {};
Set<String> pTags = {};
if( userPublicKey != "") {
// get the latest kind 3 event for the user, which lists his 'follows' list
// get the latest kind 3 event for the user, which has the 'follows' list
Event? contactEvent = getContactEvent(userPublicKey);
// if contact list was found, get user's feed; also get some default contacts
if (contactEvent != null ) {
if(gDebug > 0) print("In main: found contact list: \n ${contactEvent.originalJson}");
contactEvent.eventData.contactList.forEach((contact) {
contactEvent.eventData.contactList.forEach((contact) {
contacts.add(contact.id);
});
contacts = contacts.difference(usersFetched); // remove already fetched users from this list
getContactFeed(gListRelayUrls1, contacts, 3 * gLimitPerSubscription, getSecondsDaysAgo( limitOthersEvents));
usersFetched = usersFetched.union(contacts);
} else {
print("Could not find your contact list.");
}
}
// fetch extra events for people who don't have too large a follow list
if( usersFetched.length < gMaxPtagsToGet * 2 ) {
if( contacts.union(gDefaultFollows).length < gMaxPtagsToGet ) {
// calculate top mentioned ptags, and then get the events for those users
Set<String> pTags = getpTags(initialEvents, gMaxPtagsToGet);
pTags = pTags.difference(usersFetched);
getMultiUserEvents(gListRelayUrls1, pTags, 4 * gLimitPerSubscription, getSecondsDaysAgo(limitOthersEvents));
usersFetched = usersFetched.union(pTags);
pTags = getpTags(initialEvents, gMaxPtagsToGet);
}
// get only 200 contacts maximum
int maxContactsFetched = 200;
if( contacts.length > maxContactsFetched) {
int i = 0;
contacts.retainWhere((element) => i++ > 200); // retain only first 200, whichever they may be
}
getMultiUserEvents(gListRelayUrls1, contacts.union(pTags).difference(usersFetched), 4 * limitPerSubscription, getSecondsDaysAgo(limitOthersEvents));
usersFetched = usersFetched.union(contacts).union(pTags);
// get meta events of all users fetched
print("getting meta for # users : ${usersFetched.length} #contacts = ${contacts.length}");
getMultiUserEvents(gListRelayUrls1, usersFetched, 4 * limitPerSubscription, getSecondsDaysAgo(limitSelfEvents*2), {0,3});
// get events from channels of user
Set<String> userChannels = getUserChannels(initialEvents, userPublicKey);
//getMentionEvents(gListRelayUrls1, userChannels, limitPerSubscription, getSecondsDaysAgo(limitSelfEvents), "#e");

17
lib/relays.dart
View File

@ -102,6 +102,14 @@ class Relays {
sendRequest(relayUrl, request);
}
void getIdAndMentionEvents(String relayUrl, Set<String> ids, int limit, int sinceWhen, String tagToGet) {
String subscriptionId = "id_mention_${tagToGet}" + (relays[relayUrl]?.numRequestsSent??"").toString() + "_" + relayUrl.substring(6);
String request = getIdAndMentionRequest(subscriptionId, ids, limit, sinceWhen, tagToGet);
sendRequest(relayUrl, request);
}
/*
* @connect Connect to given relay and get all events for multiple users/publicKey and insert the
* received events in the given List<Event>
@ -255,6 +263,13 @@ void getMentionEvents(Set<String> serverUrls, Set<String> ids, int numUserEvents
});
}
void getIdAndMentionEvents(Set<String> serverUrls, Set<String> ids, int numUserEvents, int sinceWhen, String tagToGet) {
serverUrls.forEach((serverUrl) {
relays.getIdAndMentionEvents(serverUrl, ids, numUserEvents, sinceWhen, tagToGet);
});
}
getKindEvents(List<int> kind, Set<String> serverUrls, int limit, int sinceWhen) {
serverUrls.forEach((serverUrl) {
relays.getKindEvents(kind, serverUrl, limit, sinceWhen);
@ -283,7 +298,7 @@ void sendEventsRequest(Set<String> serverUrls, Set<String> eventIds) {
if( eventIds.length == 0)
return;
String eventIdsStr = getJsonList(eventIds);;
String eventIdsStr = getCommaSeparatedQuotedStrs(eventIds);;
String getEventRequest = '["REQ","event_${eventIds.length}",{"ids":[$eventIdsStr]}]';
if( gDebug > 0) log.info("sending $getEventRequest");

4
lib/settings.dart
View File

@ -34,13 +34,13 @@ bool gOverWriteFile = false; // overwrite the file, and don't ju
const int gDontAddToStoreBeforeDays = 60; // events older than this are not added to the Store of all events
const int gLimitFollowPosts = 20; // when getting events, this is the since field (unless a fully formed request is given in command line)
const int gLimitPerSubscription = 10000;
const int gLimitPerSubscription = 20000;
// don't show notifications for events that are older than 5 days and come when program is running
// applicable only for notifications and not for search results. Search results set a flag in EventData and don't use this variable
const int gDontHighlightEventsOlderThan = 4;
int gDefaultNumWaitSeconds = 12000; // is used in main()
int gDefaultNumWaitSeconds = 10000; // is used in main()
const int gMaxAuthorsInOneRequest = 300; // number of author requests to send in one request
const int gMaxPtagsToGet = 100; // maximum number of p tags that are taken from the comments of feed ( the top most, most frequent)

20
lib/utils.dart
View File

@ -364,7 +364,7 @@ String getStringFromUser(String prompt, [String defaultValue=""] ) {
// returns list in form ( if 3 sized list)
// "pubkey1","pubkey2","pubkey3"
String getJsonList(Set<String> publicKeys) {
String getCommaSeparatedQuotedStrs(Set<String> publicKeys) {
String s = "";
int i = 0;
for(String pubkey in publicKeys) {
@ -448,9 +448,23 @@ String getMentionRequest(String subscriptionId, Set<String> ids, int numUserEven
}
var strSubscription1 = '["REQ","$subscriptionId",{ "$tagToGet": [';
var strSubscription2 ='], "limit": $numUserEvents $strTime } ]';
return strSubscription1 + getJsonList(ids) + strSubscription2;
return strSubscription1 + getCommaSeparatedQuotedStrs(ids) + strSubscription2;
}
String getIdAndMentionRequest(String subscriptionId, Set<String> ids, int numUserEvents, int sinceWhen, String tagToGet) {
String strTime = "";
if( sinceWhen != 0) {
strTime = ', "since": ${sinceWhen.toString()}';
}
var strSubscription1 = '["REQ","$subscriptionId",{ "$tagToGet": [';
var strSubscription2 ='], "limit": $numUserEvents $strTime } ]';
String req = '["REQ","$subscriptionId",{ "$tagToGet": [' + getCommaSeparatedQuotedStrs(ids) + '], "limit": $numUserEvents $strTime},{"authors":[' + getCommaSeparatedQuotedStrs(ids) + ']} ]';
//print("Created id and mention request: $req");
return req;
}
String getMultiUserRequest(String subscriptionId, Set<String> publicKeys, int numUserEvents, int sinceWhen, [Set<int>? kind = null]) {
String strTime = "";
if( sinceWhen != 0) {
@ -467,7 +481,7 @@ String getMultiUserRequest(String subscriptionId, Set<String> publicKeys, int nu
var strSubscription1 = '["REQ","$subscriptionId",{ "authors": [';
var strSubscription2 ='],$strKindSection"limit": $numUserEvents $strTime } ]';
String s = "";
s = getJsonList(publicKeys);
s = getCommaSeparatedQuotedStrs(publicKeys);
String request = strSubscription1 + s + strSubscription2;
//print("In getMultiUserRequest kind = $kind strKindSection = $strKindSection: request = $request");
return request;

1
pubspec.yaml
View File

@ -13,6 +13,7 @@ homepage: https://github.com/vishalxl/nostr_console
# reduced items fetched. 23/12
# reduced items more evening 23/12
# reduced more evening 23/12
environment:
sdk: '>=2.17.3 <3.0.0'

19
scripts/configfile.cfg
View File

@ -1,14 +1,33 @@
#nostr_servers=( "wss://relay.nostr.info" "wss://nostr-relay.wlvs.space" "wss://nostr-pub.wellorder.net" "wss://relay.damus.io" "wss://nostr.delo.software" "wss://nostr-pub.semisol.dev" "wss://nostr-relay-dev.wlvs.space")
nostr_servers=(
"wss://nostr-relay-dev.wlvs.space"
"wss://nostr-01.bolt.observer"
"wss://nostr.shawnyeager.net"
"wss://nostr.zerofeerouting.com"
"wss://nostr.satsophone.tk"
"wss://relay.oldcity-bitcoiners.info"
"wss://relay.damus.io"
"wss://relay.nostr.info"
"wss://nostr-2.zebedee.cloud"
"wss://nostr.zebedee.cloud"
"wss://nostr.semisol.dev"
"wss://nostr.onsats.org"
"wss://nostr-relay.wlvs.space"
"wss://nostr-relay.digitalmob.ro"
"wss://nostr.coinos.io"
"wss://nostr-pub.wellorder.net"
"wss://nostr.radixrat.com"
"wss://nostr.bitcoiner.social"
"wss://relay.valireum.net"
"wss://nostr-relay.trustbtc.org"
"wss://relay.stoner.com"
"wss://nostr.w3ird.tech"
"wss://nostr.bongbong.com"
"wss://nostr.hugo.md"
)

2
scripts/test_servers.sh
View File

@ -5,7 +5,7 @@ source ./configfile.cfg
limit=100000
numHours=1
echo -e "Requesting all evetns in last $numHours hours with a limit of $limit by executing the following command for each:"
echo -e "Requesting all events in last $numHours hours with a limit of $limit by executing the following command for each:"
sinceSeconds=`date -d "-$numHours hour" +%s` ;
req="[\"REQ\",\"l\",{\"since\":$sinceSeconds,\"limit\":$limit}]";
echo "echo $req | websocat $server | wc " ;