diff --git a/bin/nostr_console.dart b/bin/nostr_console.dart index 1eb1c04..ad204d8 100644 --- a/bin/nostr_console.dart +++ b/bin/nostr_console.dart @@ -43,7 +43,8 @@ Future main(List arguments) async { ..addOption(eventFileArg, abbr:"f", defaultsTo: gDefaultEventsFilename)..addFlag(disableFileArg, abbr:"s", defaultsTo: false) ..addFlag(translateArg, abbr: "t", defaultsTo: false) ..addOption(colorArg, abbr:"c") - ..addOption(difficultyArg, abbr:"y"); + ..addOption(difficultyArg, abbr:"y") + ..addFlag("debug"); try { ArgResults argResults = parser.parse(arguments); if( argResults[helpArg]) { @@ -51,6 +52,11 @@ Future main(List arguments) async { return; } + if( argResults["debug"]) { + gDebug = 1; + } + + if( argResults[translateArg]) { gTranslate = true; print("Going to translate comments in last $gNumTranslateDays days using Google translate service"); @@ -183,24 +189,19 @@ Future main(List arguments) async { } } + Set initialEvents = {}; // collect all events here and then create tree out of them + if( gEventsFilename != "") { - print("\n"); stdout.write('Reading events from ${whetherDefault}file.......'); // read file events and give the events to relays from where they're picked up later - Set eventsFromFile = await readEventsFromFile(gEventsFilename); - setRelaysIntialEvents(eventsFromFile); + initialEvents = await readEventsFromFile(gEventsFilename); // count events - eventsFromFile.forEach((element) { element.eventData.kind == 1? numFileEvents++: numFileEvents;}); - print("read $numFileEvents posts from file $gEventsFilename"); + initialEvents.forEach((element) { element.eventData.kind == 1? numFilePosts++: numFilePosts;}); + print("read $numFilePosts posts from file $gEventsFilename"); } - // get all events in Tree form - Store node = getTree(getRecievedEvents()); - - // call the mein UI function - clearEvents(); // process request string. If this is blank then the application only reads from file and does not connect to internet. if( argResults[requestArg] != null) { @@ -216,11 +217,14 @@ Future main(List arguments) async { Future.delayed(Duration(milliseconds: numWaitSeconds * 2), () { Set receivedEvents = getRecievedEvents(); - stdout.write("received ${receivedEvents.length - numFileEvents} events\n"); + //stdout.write("received ${receivedEvents.length - numFilePosts} events\n"); - node.insertEvents(getRecievedEvents()); - clearEvents(); + initialEvents.addAll(receivedEvents); + + // Creat tree from all events read form file + Store node = getTree(initialEvents); + clearEvents(); if( gDebug > 0) stdout.write("Total events of kind 1 in created tree: ${node.count()} events\n"); mainMenuUi(node); }); @@ -236,50 +240,65 @@ Future main(List arguments) async { stdout.write('Waiting for user posts to come in.....'); Future.delayed(const Duration(milliseconds: gDefaultNumWaitSeconds), () { // count user events - getRecievedEvents().forEach((element) { element.eventData.kind == 1? numUserEvents++: numUserEvents;}); - stdout.write("...received $numUserEvents new posts made by the user\n"); + + initialEvents.addAll(getRecievedEvents()); + clearEvents(); + + //print("numUserPosts $numUserPosts numFilePosts $numFilePosts numFeedPosts $numFeedPosts"); + initialEvents.forEach((element) { element.eventData.kind == 1? numUserPosts++: numUserPosts;}); + numUserPosts -= numFilePosts; + stdout.write("...done.\n");//received $numUserPosts new posts made by the user\n"); if( gDebug > 0) log.info("Received user events."); - getRecievedEvents().forEach((e) => processKind3Event(e)); // first process the kind 3 event + initialEvents.forEach((e) => processKind3Event(e)); // first process the kind 3 event // get the latest kind 3 event for the user, which lists his 'follows' list Event? contactEvent = getContactEvent(userPublicKey); // if contact list was found, get user's feed, and keep the contact list for later use - List contactList = []; if (contactEvent != null ) { if(gDebug > 0) print("In main: found contact list: \n ${contactEvent.originalJson}"); - contactList = getContactFeed(gListRelayUrls, contactEvent.eventData.contactList, gLimitPerSubscription, getSecondsDaysAgo(gDaysToGetEventsFor)); + getContactFeed(gListRelayUrls, contactEvent.eventData.contactList, gLimitPerSubscription, getSecondsDaysAgo(gDaysToGetEventsFor)); if( !gContactLists.containsKey(userPublicKey)) { gContactLists[userPublicKey] = contactEvent.eventData.contactList; } } else { - if( gDebug > 0) print( "could not find contact list"); + if( gDebug >= 0) log.info( "Could not find contact list"); } stdout.write('Waiting for feed to come in..............'); Future.delayed(const Duration(milliseconds: gDefaultNumWaitSeconds * 1), () { + initialEvents.addAll(getRecievedEvents()); + clearEvents(); + // count feed events - getRecievedEvents().forEach((element) { element.eventData.kind == 1? numFeedEvents++: numFeedEvents;}); - numFeedEvents = numFeedEvents - numUserEvents; - stdout.write("received $numFeedEvents new posts from the follows\n"); - if( gDebug > 0) log.info("Received feed."); + initialEvents.forEach((element) { element.eventData.kind == 1? numFeedPosts++: numFeedPosts;}); + numFeedPosts = numFeedPosts - numUserPosts - numFilePosts; + stdout.write("done\n");//received $numFeedPosts new posts from the follows\n"); // get mentioned ptags, and then get the events for those users - List pTags = getpTags(getRecievedEvents(), gMaxPtagsToGet); + List pTags = getpTags(initialEvents, gMaxPtagsToGet); getMultiUserEvents(gListRelayUrls, pTags, gLimitPerSubscription, getSecondsDaysAgo(gDaysToGetEventsFor)); + //print("before others: initialEvents = ${initialEvents.length}"); stdout.write('Waiting for rest of posts to come in.....'); Future.delayed(const Duration(milliseconds: gDefaultNumWaitSeconds * 2), () { + initialEvents.addAll(getRecievedEvents()); + clearEvents(); + + //print("after adding others: initialEvents = ${initialEvents.length}"); + // count other events - getRecievedEvents().forEach((element) { element.eventData.kind == 1? numOtherEvents++: numOtherEvents;}); - numOtherEvents = numOtherEvents - numFeedEvents - numUserEvents; - stdout.write("received $numOtherEvents new posts by others\n"); + initialEvents.forEach((element) { element.eventData.kind == 1? numOtherPosts++: numOtherPosts;}); + numOtherPosts = numOtherPosts - numFeedPosts - numUserPosts - numFilePosts; + stdout.write("done\n"); if( gDebug > 0) log.info("Received ptag events events."); - node.insertEvents(getRecievedEvents()); + // Creat tree from all events read form file + Store node = getTree(initialEvents); + clearEvents(); mainMenuUi(node); }); diff --git a/lib/console_ui.dart b/lib/console_ui.dart index 83e02a4..0f9ee7b 100644 --- a/lib/console_ui.dart +++ b/lib/console_ui.dart @@ -12,7 +12,7 @@ Future processNotifications(Store node) async { const int waitMilliSeconds = 150; Future.delayed(const Duration(milliseconds: waitMilliSeconds), () { - Set newEventIdsSet = node.insertEvents(getRecievedEvents()); + Set newEventIdsSet = node.processIncomingEvent(getRecievedEvents()); String nameToDisplay = userPrivateKey.length == 64? "$gCommentColor${getAuthorName(userPublicKey)}$gColorEndMarker": "${gWarningColor}You are not signed in$gColorEndMarker but are using public key $userPublicKey"; @@ -269,16 +269,15 @@ Future otherMenuUi(Store node) async { node.printSocialDistance(pubkey.first, authorName); print(""); - stdout.write("They follows ${contactEvent.eventData.contactList.length} accounts: "); + stdout.write("They follow ${contactEvent.eventData.contactList.length} accounts: "); contactEvent.eventData.contactList.forEach((x) => stdout.write("${getAuthorName(x.id)}, ")); print("\n"); - - List followers = node.getFollowers(pubkey.first); - stdout.write("They have ${followers.length} followers: "); - followers.forEach((x) => stdout.write("${getAuthorName(x)}, ")); - print(""); - } + + List followers = node.getFollowers(pubkey.first); + stdout.write("They have ${followers.length} followers: "); + followers.forEach((x) => stdout.write("${getAuthorName(x)}, ")); + print(""); print(""); } } @@ -584,7 +583,7 @@ Future mainMenuUi(Store node) async { bool hasRepliesAndLikes (Tree t) => t.hasRepliesAndLikes(userPublicKey); node.printTree(0, DateTime.now().subtract(Duration(days:gNumLastDays)), hasRepliesAndLikes); - + bool userContinue = true; while(userContinue) { // align the text again in case the window size has been changed @@ -602,6 +601,7 @@ Future mainMenuUi(Store node) async { } await processNotifications(node); + // the main menu int option = showMenu(['Display feed', // 1 'Post/Reply/Like', // 2 @@ -657,11 +657,11 @@ Future mainMenuUi(Store node) async { default: userContinue = false; String authorName = getAuthorName(userPublicKey); - print("\nFinished Nostr session for user with name and public key: ${authorName} ($userPublicKey)."); + print("\nFinished Nostr session for user with name and public key: ${authorName} ($userPublicKey)"); if( gEventsFilename != "") { await node.writeEventsToFile(gEventsFilename); } exit(0); - } + } // end menu switch } // end while -} +} // end mainMenu diff --git a/lib/event_ds.dart b/lib/event_ds.dart index 408a159..6616c77 100644 --- a/lib/event_ds.dart +++ b/lib/event_ds.dart @@ -1,3 +1,4 @@ +import 'dart:ffi'; import 'dart:io'; import 'dart:convert'; import 'package:intl/intl.dart'; @@ -197,47 +198,30 @@ class EventData { break; case 4: - if( pubkey == userPublicKey) - break; - + if( pubkey == userPublicKey) break; // crashes right now otherwise if(!isUserDirectMessage(this)) { break; } - /*print("in translateAndExpandMentions() for a dm \npubkey = $pubkey \ncontent = $content"); - print("tags = $tags\n"); - */ - + //print("in translateAndExpandMentions() for a dm \npubkey = $pubkey \ncontent = $content"); + //print("tags = $tags\n"); int ivIndex = content.indexOf("?iv="); var enc_str = content.substring(0, ivIndex); var iv = content.substring( ivIndex + 4, content.length); //print("enc_str = $enc_str ; iv = $iv"); - var decryptd = myPrivateDecrypt( userPrivateKey, "02" + pubkey, enc_str, iv); // use bob's privatekey and alic's publickey means bob can read message from alic - + + String userKey = userPrivateKey ; + String otherUserPubKey = "02" + pubkey; + if( pubkey == userPublicKey) { + userKey = userPrivateKey; + otherUserPubKey = "02" + pubkey; + //iv = ""; + } + var decryptd = myPrivateDecrypt( userKey, otherUserPubKey, enc_str, iv); // use bob's privatekey and alic's publickey means bob can read message from alic evaluatedContent = decryptd; - //printEventData(0); - //print("\n\n"); break; } // end switch } // end translateAndExpandMentions -Uint8List aesCbcDecrypt(Uint8List key, Uint8List iv, Uint8List cipherText) { - // Create a CBC block cipher with AES, and initialize with key and IV - - final cbc = CBCBlockCipher(AESFastEngine()) - ..init(false, ParametersWithIV(KeyParameter(key), iv)); // false=decrypt - - // Decrypt the cipherText block-by-block - - final paddedPlainText = Uint8List(cipherText.length); // allocate space - - var offset = 0; - while (offset < cipherText.length) { - offset += cbc.processBlock(cipherText, offset, paddedPlainText, offset); - } - assert(offset == cipherText.length); - - return paddedPlainText; -} // only applicable for kind 42 event String getChatRoomId() { @@ -340,28 +324,29 @@ class Event { EventData eventData; String originalJson; List seenOnRelays; + bool readFromFile; - Event(this.event, this.id, this.eventData, this.seenOnRelays, this.originalJson); + Event(this.event, this.id, this.eventData, this.seenOnRelays, this.originalJson, [this.readFromFile = false]); @override bool operator ==( other) { return (other is Event) && eventData.id == other.eventData.id; } - factory Event.fromJson(String d, String relay) { + factory Event.fromJson(String d, String relay, [bool fromFile = false]) { try { dynamic json = jsonDecode(d); if( json.length < 3) { String e = ""; e = json.length > 1? json[0]: ""; - return Event(e,"",EventData("non","", 0, 0, "", [], [], [], [[]], {}), [relay], "[json]"); + return Event(e,"",EventData("non","", 0, 0, "", [], [], [], [[]], {}), [relay], "[json]", fromFile); } - return Event(json[0] as String, json[1] as String, EventData.fromJson(json[2]), [relay], d ); + return Event(json[0] as String, json[1] as String, EventData.fromJson(json[2]), [relay], d, fromFile ); } on Exception catch(e) { if( gDebug> 0) { print("Could not create event. returning dummy event. $e"); } - return Event("","",EventData("non","", 0, 0, "", [], [], [], [[]], {}), [relay], "[json]"); + return Event("","",EventData("non","", 0, 0, "", [], [], [], [[]], {}), [relay], "[json]", fromFile); } } @@ -429,29 +414,6 @@ List getpTags(Set events, int numMostFrequent) { return ptags; } -Set readEventsFromFile(String filename) { - Set events = {}; - final File file = File(filename); - - // sync read - try { - List lines = file.readAsLinesSync(); - for( int i = 0; i < lines.length; i++ ) { - Event e = Event.fromJson(lines[i], ""); - if( e.eventData.id == gCheckEventId) { - print("read $gCheckEventId from file"); - } - events.add(e); - } - } on Exception catch(e) { - //print("cannot open file $gEventsFilename"); - if( gDebug > 0) print("Could not open file. error = $e"); - } - - if( gDebug > 0) print("In readEventsFromFile: returning ${events.length} total events"); - return events; -} - // From the list of events provided, lookup the lastst contact information for the given user/pubkey Event? getContactEvent(String pubkey) { @@ -568,8 +530,8 @@ bool processKind3Event(Event newContactEvent) { } // returns name by looking up global list gKindONames, which is populated by kind 0 events -String getAuthorName(String pubkey) { - String max3(String v) => v.length > 3? v.substring(0,3) : v.substring(0, v.length); +String getAuthorName(String pubkey, [int len = 3]) { + String max3(String v) => v.length > len? v.substring(0,len) : v.substring(0, v.length); String name = gKindONames[pubkey]?.name??max3(pubkey); return name; } @@ -578,15 +540,17 @@ String getAuthorName(String pubkey) { Set getPublicKeyFromName(String userName) { Set pubkeys = {}; - if(gDebug > 0) print("In getPublicKeyFromName: doing lookup for $userName len of gKindONames= ${gKindONames.length}"); + if(gDebug >= 0) print("In getPublicKeyFromName: doing lookup for $userName len of gKindONames= ${gKindONames.length}"); - gKindONames.forEach((pk, value) { + gKindONames.forEach((pk, userInfo) { // check both the user name, and the pubkey to search for the user - if( userName == value.name) { + //print(userInfo.name); + if( userName == userInfo.name) { pubkeys.add(pk); } if( userName.length <= pk.length) { + print("$pk $userName" ); if( pk.substring(0, userName.length) == userName) { pubkeys.add(pk); } @@ -770,19 +734,20 @@ bool isUserDirectMessage(EventData directMessageData) { } /// Decrypt data using self private key -String myPrivateDecrypt( - String privateString, String publicString, String b64encoded, - [String b64IV = ""]) { +String myPrivateDecrypt( String privateString, + String publicString, + String b64encoded, + [String b64IV = ""]) { + Uint8List encdData = convert.base64.decode(b64encoded); + final rawData = myPrivateDecryptRaw(privateString, publicString, encdData, b64IV); + convert.Utf8Decoder decode = const convert.Utf8Decoder(); + return decode.convert(rawData.toList()); +} - Uint8List encdData = convert.base64.decode(b64encoded); - final rawData = myPrivateDecryptRaw(privateString, publicString, encdData, b64IV); - convert.Utf8Decoder decode = const convert.Utf8Decoder(); - return decode.convert(rawData.toList()); - } - -Uint8List myPrivateDecryptRaw( - String privateString, String publicString, Uint8List cipherText, - [String b64IV = ""]) { +Uint8List myPrivateDecryptRaw( String privateString, + String publicString, + Uint8List cipherText, + [String b64IV = ""]) { final secretIV = Kepler.byteSecret(privateString, publicString); final key = Uint8List.fromList(secretIV[0]); @@ -790,33 +755,78 @@ Uint8List myPrivateDecryptRaw( ? convert.base64.decode(b64IV) : Uint8List.fromList(secretIV[1]); + bool debug = false; + if( debug) print("iv = $iv "); + + // pointy castle source https://github.com/PointyCastle/pointycastle/blob/master/tutorials/aes-cbc.md -// pointy castle source https://github.com/PointyCastle/pointycastle/blob/master/tutorials/aes-cbc.md -final cbc = CBCBlockCipher(AESFastEngine()) - ..init(false, ParametersWithIV(KeyParameter(key), iv)); // false=decrypt - // Decrypt the cipherText block-by-block + final cbc = CBCBlockCipher(AESFastEngine()) + ..init(false, ParametersWithIV(KeyParameter(key), iv) ) ; + - final paddedPlainText = Uint8List(cipherText.length); // allocate space + // 2 + + //PaddedBlockCipher('AES/CBC/PKCS7').KeyParameter = KeyParameter; + PaddedBlockCipher p = PaddedBlockCipher('AES/CBC/PKCS7'); + if( debug) print("p cipher: ${p.cipher}"); + p.cipher.init(false, ParametersWithIV(KeyParameter(key), iv) ) ; + + + PaddedBlockCipherParameters paddedParams = PaddedBlockCipherParameters ( ParametersWithIV(KeyParameter(key), iv), null ); + + final pbc = CBCBlockCipher(p)..init(false, ParametersWithIV(paddedParams, iv) ) ; + + + // 3 https://github.com/Dhuliang/flutter-bsv/blob/42a2d92ec6bb9ee3231878ffe684e1b7940c7d49/lib/src/aescbc.dart + + CipherParameters params = new PaddedBlockCipherParameters( + new ParametersWithIV(new KeyParameter(key), iv), null); + + PaddedBlockCipherImpl cipherImpl = new PaddedBlockCipherImpl( + new PKCS7Padding(), new CBCBlockCipher(new AESEngine())); + + cipherImpl.init(false, + params as PaddedBlockCipherParameters); + + final Uint8List paddedPlainText = Uint8List(cipherText.length); // allocate space - //print("going into while"); var offset = 0; while (offset < cipherText.length) { -/* convert.Utf8Decoder decode = const convert.Utf8Decoder(); - print('in while loop $offset $paddedPlainText len = ${paddedPlainText}'); - print( "paddedPlainText converted: ${decode.convert(paddedPlainText)}"); - print(""); -*/ - offset += cbc.processBlock(cipherText, offset, paddedPlainText, offset); + offset += cipherImpl.processBlock(cipherText, offset, paddedPlainText, offset); } assert(offset == cipherText.length); - return paddedPlainText; + + final pd = PKCS7Padding (); + Uint8List retval = paddedPlainText;// p.process(false, paddedPlainText); + return retval.sublist(0, retval.length); } - ParametersWithIV buildParams( - Uint8List key, Uint8List iv) { - return ParametersWithIV(KeyParameter(key), iv); +ParametersWithIV +buildParams( Uint8List key, Uint8List iv) { + return ParametersWithIV(KeyParameter(key), iv); +} + +Set readEventsFromFile(String filename) { + Set events = {}; + final File file = File(filename); + + // sync read + try { + List lines = file.readAsLinesSync(); + for( int i = 0; i < lines.length; i++ ) { + Event e = Event.fromJson(lines[i], "", true); + events.add(e); + } + } on Exception catch(e) { + //print("cannot open file $gEventsFilename"); + if( gDebug > 0) print("Could not open file. error = $e"); } + if( gDebug > 0) print("In readEventsFromFile: returning ${events.length} total events"); + return events; +} + diff --git a/lib/relays.dart b/lib/relays.dart index 0e35c7a..ab6fa75 100644 --- a/lib/relays.dart +++ b/lib/relays.dart @@ -51,19 +51,16 @@ class Relays { * received events in the given List */ void getUserEvents(String relayUrl, String publicKey, int numEventsToGet, int sinceWhen) { - for(int i = 0; i < gBots.length; i++) { + for(int i = 0; i < gBots.length; i++) { // ignore bots if( publicKey == gBots[i]) { - //print("In gerUserEvents: ignoring bot: $publicKey"); return; } } String subscriptionId = "single_user" + (relays[relayUrl]?.numRequestsSent??"").toString(); - if( relays.containsKey(relayUrl)) { - List? users = relays[relayUrl]?.users; - if( users != null) { + if( users != null) { // get a user only if it has not already been requested // following is too restrictive casuse changed sinceWhen is not considered. TODO improve it for(int i = 0; i < users.length; i++) { if( users[i] == publicKey) { @@ -254,7 +251,7 @@ List getContactFeed(List relayUrls, List contacts, int // send request for the users events to the relays mContacts.forEach((key, value) { - relays.getMultiUserEvents(key, value, numEventsToGet, sinceWhen); + //relays.getMultiUserEvents(key, value, numEventsToGet, sinceWhen); relayUrls.forEach((relayUrl) { relays.getMultiUserEvents(relayUrl, value, numEventsToGet, sinceWhen); diff --git a/lib/settings.dart b/lib/settings.dart index 434b164..3b8fc57 100644 --- a/lib/settings.dart +++ b/lib/settings.dart @@ -12,7 +12,7 @@ String gEventsFilename = ""; // is set in arguments, and if set, th bool gDontWriteOldEvents = true; const int gDontSaveBeforeDays = 100; // dont save events older than this many days if gDontWriteOldEvents flag is true -const int gDaysToGetEventsFor = 30; // when getting events, this is the since field (unless a fully formed request is given in command line) +const int gDaysToGetEventsFor = 100; // when getting events, this is the since field (unless a fully formed request is given in command line) const int gLimitPerSubscription = 20000; // don't show notifications for events that are older than 5 days and come when program is running @@ -23,7 +23,7 @@ const int gMaxAuthorsInOneRequest = 100; // number of author requests to send in const int gMaxPtagsToGet = 100; // maximum number of p tags that are taken from the comments of feed ( the top most, most frequent) // global counters of total events read or processed -int numFileEvents = 0, numUserEvents = 0, numFeedEvents = 0, numOtherEvents = 0; +int numFilePosts = 0, numUserPosts = 0, numFeedPosts = 0, numOtherPosts = 0; //String defaultServerUrl = 'wss://relay.damus.io'; //const String nostrRelayUnther = 'wss://nostr-relay.untethr.me'; not working @@ -34,7 +34,6 @@ List gListRelayUrls = [ defaultServerUrl, relayNostrInfo, "wss://nostr-verified.wellorder.net", "wss://nostr-relay.wlvs.space", - "wss://nostr-pub.wellorder.net", "wss://nostr.ono.re" ]; diff --git a/lib/tree_ds.dart b/lib/tree_ds.dart index c84b445..ce8a40a 100644 --- a/lib/tree_ds.dart +++ b/lib/tree_ds.dart @@ -330,17 +330,14 @@ class Tree { * This Store class holds events too in its map, and in its chatRooms structure */ class Store { - List children; // only has kind 1 events + List topPosts; // only has kind 1 events Map allChildEventsMap; // has events of kind typesInEventMap List eventsWithoutParent; - bool whetherTopMost; Map chatRooms = {}; Map directRooms = {}; - Set eventsNotReadFromFile; - - Store(this.children, this.allChildEventsMap, this.eventsWithoutParent, this.whetherTopMost, this.chatRooms, this.directRooms, this.eventsNotReadFromFile) { + Store(this.topPosts, this.allChildEventsMap, this.eventsWithoutParent, this.chatRooms, this.directRooms) { allChildEventsMap.forEach((eventId, tree) { if( tree.store == null) { tree.setStore(this); @@ -448,7 +445,7 @@ class Store { // first create a map. then process each element in the map by adding it to its parent ( if its a child tree) factory Store.fromEvents(Set events) { if( events.isEmpty) { - return Store( [], {}, [], false, {}, {}, {}); + return Store( [], {}, [], {}, {}); } // create a map tempChildEventsMap from list of events, key is eventId and value is event itself @@ -532,133 +529,138 @@ class Store { if(gDebug != 0) print("In Tree FromEvents: number of events without parent in fromEvents = ${tempWithoutParent.length}"); // create a dummy top level tree and then create the main Tree object - return Store( topLevelTrees, tempChildEventsMap, tempWithoutParent, true, rooms, tempDirectRooms, {}); + return Store( topLevelTrees, tempChildEventsMap, tempWithoutParent, rooms, tempDirectRooms); } // end fromEvents() /***********************************************************************************************************************************/ - /* @insertEvents inserts the given new events into the tree, and returns the id the ones actually - * inserted so that they can be printed as notifications - */ - Set insertEvents(Set newEventsSetToProcess) { - if( gDebug > 0) log.info("In insertEvetnts: called for ${newEventsSetToProcess.length} events"); + /* @processIncomingEvent inserts the relevant events into the tree and otherwise processes likes, delete events etc. + * returns the id of the relevant ones actually inserted so that they can be printed as notifications. + */ + Set processIncomingEvent(Set newEventsToProcess) { + if( gDebug >= 0) log.info("In insertEvetnts: allChildEventsMap size = ${allChildEventsMap.length}, called for ${newEventsToProcess.length} NEW events"); Set newEventIdsSet = {}; // add the event to the main event store thats allChildEventsMap - newEventsSetToProcess.forEach((newEvent) { - - if( allChildEventsMap.containsKey(newEvent.eventData.id)) {// don't process if the event is already present in the map - return; - } - - // handle reaction events and return if we could not find the reacted to. Continue otherwise to add this to notification set newEventIdsSet - if( newEvent.eventData.kind == 7) { - if( processReaction(newEvent) == "") { - if(gDebug > 0) print("In insertEvents: For new reaction ${newEvent.eventData.id} could not find reactedTo or reaction was already present by this reactor"); + newEventsToProcess.forEach((newEvent) { + + if( allChildEventsMap.containsKey(newEvent.eventData.id)) {// don't process if the event is already present in the map return; } - } - // handle delete events. return if its not handled for some reason ( like deleted event not found) - if( newEvent.eventData.kind == 5) { - processDeleteEvent(allChildEventsMap, newEvent); - if(gDebug > 0) print("In insertEvents: For new deleteion event ${newEvent.eventData.id} could not process it."); - return; - } - - if( !isUserDirectMessage(newEvent.eventData)) { // direct message not relevant to user are ignored - return; - } - - - // only kind 0, 1, 3, 4, 5( delete), 7, 40, 42 events are added to map, return otherwise - if( !typesInEventMap.contains(newEvent.eventData.kind) ) { - return; - } - - // expand mentions ( and translate if flag is set) and then add event to main event map - newEvent.eventData.translateAndExpandMentions(); // this also handles dm decryption for kind 4 messages, for kind 1 will do translation/expansion; - - eventsNotReadFromFile.add(newEvent.eventData.id); // used later so that only these events are appended to the file - - // add them to the main store of the Tree object - allChildEventsMap[newEvent.eventData.id] = Tree(newEvent, [], this); - - // add to new-notification list only if this is a recent event ( because relays may send old events, and we dont want to highlight stale messages) - if( newEvent.eventData.createdAt > getSecondsDaysAgo(gDontHighlightEventsOlderThan)) { - newEventIdsSet.add(newEvent.eventData.id); - } - }); - - // now go over the newly inserted event, and add its to the tree for kind 1 events, add 42 events to channels. rest ( such as kind 0, kind 3, kind 7) are ignored. - newEventIdsSet.forEach((newId) { - Tree? newTree = allChildEventsMap[newId]; - if( newTree != null) { // this should return true because we just inserted this event in the allEvents in block above - - switch(newTree.event.eventData.kind) { - case 1: - // only kind 1 events are added to the overall tree structure - if( newTree.event.eventData.eTagsRest.isEmpty) { - // if its a new parent event, then add it to the main top parents ( this.children) - children.add(newTree); - } else { - // if it has a parent , then add the newTree as the parent's child - String parentId = newTree.event.eventData.getParent(); - if( allChildEventsMap.containsKey(parentId)) { - allChildEventsMap[parentId]?.children.add(newTree); - } else { - // create top unknown parent and then add it - Event dummy = Event("","", EventData("non", gDummyAccountPubkey, newTree.event.eventData.createdAt, -1, "Unknown parent event", [], [], [], [[]], {}), [""], "[json]"); - Tree dummyTopNode = Tree.withoutStore(dummy, []); - dummyTopNode.children.add(newTree); - children.add(dummyTopNode); - } - } - break; - case 4: - // add kind 4 direct chat message event to its direct massage room - String directRoomId = getDirectRoomId(newTree.event.eventData); - //print("in insert events: got directRoomId = ${directRoomId}"); - if( directRoomId != "") { - if( directRooms.containsKey(directRoomId)) { - if( gDebug > 0) print("added event to direct room in insert event"); - addMessageToDirectRoom(directRoomId, newTree.event.eventData.id, allChildEventsMap, directRooms); - newTree.event.eventData.isNotification = true; // highlight it too in next printing - //print(" in from event: added it to a direct room"); - break; - } - } - - List temp = []; - temp.add(newTree.event.eventData.id); - directRooms[directRoomId] = DirectMessageRoom(directRoomId, temp); // TODO sort it - - break; - - case 42: - newTree.event.eventData.isNotification = true; // highlight it too in next printing - // add 42 chat message event id to its chat room - String channelId = newTree.event.eventData.getParent(); - if( channelId != "") { - if( chatRooms.containsKey(channelId)) { - if( gDebug > 0) print("added event to chat room in insert event"); - addMessageToChannel(channelId, newTree.event.eventData.id, allChildEventsMap, chatRooms); // adds in order - break; - } else { - chatRooms[channelId] = ChatRoom(channelId, "", "", "", []); - addMessageToChannel(channelId, newTree.event.eventData.id, allChildEventsMap, chatRooms); - } - } - break; - default: - break; + // handle reaction events and return if we could not find the reacted to. Continue otherwise to add this to notification set newEventIdsSet + if( newEvent.eventData.kind == 7) { + if( processReaction(newEvent) == "") { + if(gDebug > 0) print("In insertEvents: For new reaction ${newEvent.eventData.id} could not find reactedTo or reaction was already present by this reactor"); + return; + } } - } - }); - if(gDebug > 0) print("In end of insertEvents: Returning ${newEventIdsSet.length} new notification-type events, which are ${newEventIdsSet.length < 10 ? newEventIdsSet: " 0) print("In insertEvents: For new deleteion event ${newEvent.eventData.id} could not process it."); + return; + } + + if( newEvent.eventData.kind == 4) { + if( !isUserDirectMessage(newEvent.eventData)) { // direct message not relevant to user are ignored + return; + } + } + + if( newEvent.eventData.kind == 0) { + processKind0Event(newEvent); + } + + // only kind 0, 1, 3, 4, 5( delete), 7, 40, 42 events are added to map, return otherwise + if( !typesInEventMap.contains(newEvent.eventData.kind) ) { + return; + } + + // expand mentions ( and translate if flag is set) and then add event to main event map + newEvent.eventData.translateAndExpandMentions(); // this also handles dm decryption for kind 4 messages, for kind 1 will do translation/expansion; + + // add them to the main store of the Tree object + allChildEventsMap[newEvent.eventData.id] = Tree(newEvent, [], this); + + // add to new-notification list only if this is a recent event ( because relays may send old events, and we dont want to highlight stale messages) + if( newEvent.eventData.createdAt > getSecondsDaysAgo(gDontHighlightEventsOlderThan)) { + newEventIdsSet.add(newEvent.eventData.id); + } + }); + + // now go over the newly inserted event, and add its to the tree for kind 1 events, add 42 events to channels. rest ( such as kind 0, kind 3, kind 7) are ignored. + newEventIdsSet.forEach((newId) { + Tree? newTree = allChildEventsMap[newId]; + if( newTree != null) { // this should return true because we just inserted this event in the allEvents in block above + + switch(newTree.event.eventData.kind) { + case 1: + // only kind 1 events are added to the overall tree structure + if( newTree.event.eventData.eTagsRest.isEmpty) { + // if its a new parent event, then add it to the main top parents ( this.children) + topPosts.add(newTree); + } else { + // if it has a parent , then add the newTree as the parent's child + String parentId = newTree.event.eventData.getParent(); + if( allChildEventsMap.containsKey(parentId)) { + allChildEventsMap[parentId]?.children.add(newTree); + } else { + // create top unknown parent and then add it + Event dummy = Event("","", EventData("non", gDummyAccountPubkey, newTree.event.eventData.createdAt, -1, "Unknown parent event", [], [], [], [[]], {}), [""], "[json]"); + Tree dummyTopNode = Tree.withoutStore(dummy, []); + dummyTopNode.children.add(newTree); + topPosts.add(dummyTopNode); + } + } + break; + case 4: + // add kind 4 direct chat message event to its direct massage room + String directRoomId = getDirectRoomId(newTree.event.eventData); + //print("in insert events: got directRoomId = ${directRoomId}"); + if( directRoomId != "") { + if( directRooms.containsKey(directRoomId)) { + if( gDebug > 0) print("added event to direct room in insert event"); + addMessageToDirectRoom(directRoomId, newTree.event.eventData.id, allChildEventsMap, directRooms); + newTree.event.eventData.isNotification = true; // highlight it too in next printing + //print(" in from event: added it to a direct room"); + break; + } + } + + List temp = []; + temp.add(newTree.event.eventData.id); + directRooms[directRoomId] = DirectMessageRoom(directRoomId, temp); // TODO sort it + + break; + + case 42: + newTree.event.eventData.isNotification = true; // highlight it too in next printing + // add 42 chat message event id to its chat room + String channelId = newTree.event.eventData.getParent(); + if( channelId != "") { + if( chatRooms.containsKey(channelId)) { + if( gDebug > 0) print("added event to chat room in insert event"); + addMessageToChannel(channelId, newTree.event.eventData.id, allChildEventsMap, chatRooms); // adds in order + break; + } else { + chatRooms[channelId] = ChatRoom(channelId, "", "", "", []); + addMessageToChannel(channelId, newTree.event.eventData.id, allChildEventsMap, chatRooms); + } + } + break; + default: + break; + } + } + }); + int totalTreeSize = 0; + topPosts.forEach((element) {totalTreeSize += element.count();}); + if(gDebug >= 0) print("In end of insertEvents: allChildEventsMap size = ${allChildEventsMap.length}; mainTree count = $totalTreeSize"); + if(gDebug >= 0) print("Returning ${newEventIdsSet.length} new notification-type events, which are ${newEventIdsSet.length < 10 ? newEventIdsSet: " "} "); + return newEventIdsSet; + } // end insertEvents() /***********************************************************************************************************************************/ /* @@ -761,17 +763,17 @@ class Store { int numPrinted = 0; depth = depth - 1; - children.sort(sortTreeNewestReply); // sorting done only for top most threads. Lower threads aren't sorted so save cpu etc TODO improve top sorting + topPosts.sort(sortTreeNewestReply); // sorting done only for top most threads. Lower threads aren't sorted so save cpu etc TODO improve top sorting - for( int i = 0; i < children.length; i++) { + for( int i = 0; i < topPosts.length; i++) { // continue if this children isn't going to get printed anyway; selector is only called for top most tree - if( treeSelector(children[i]) == false) { + if( treeSelector(topPosts[i]) == false) { continue; } // for top Store, only print the thread that are newer than the given parameter - int newestChildTime = children[i].getMostRecentTime(0); + int newestChildTime = topPosts[i].getMostRecentTime(0); DateTime dTime = DateTime.fromMillisecondsSinceEpoch(newestChildTime *1000); if( dTime.compareTo(newerThan) < 0) { continue; @@ -781,7 +783,7 @@ class Store { stdout.write("\n"); } - numPrinted += children[i].printTree(depth+1, newerThan, treeSelector); + numPrinted += topPosts[i].printTree(depth+1, newerThan, treeSelector); } print("\n\nTotal posts/replies printed: $numPrinted for last $gNumLastDays days"); @@ -825,10 +827,12 @@ class Store { print("\n\nDirect messages inbox:"); printUnderlined(" From Num of Messages Latest Message "); directRooms.forEach((key, value) { - String name = getAuthorName(key); + String name = getAuthorName(key, 4); int numMessages = value.messageIds.length; stdout.write("${name} ${getNumSpaces(32-name.length)} $numMessages${getNumSpaces(12- numMessages.toString().length)}"); + + // print latest event in one lin List messageIds = value.messageIds; for( int i = messageIds.length - 1; i >= 0; i++) { if( allChildEventsMap.containsKey(messageIds[i])) { @@ -845,10 +849,12 @@ class Store { // shows the given directRoomId, where directRoomId is prefix-id or pubkey of the other user. returns full id of other user. String showDirectRoom(String directRoomId, [int page = 1]) { + print("In show DirectRoom $directRoomId"); if( directRoomId.length > 64) { // TODO revisit cause if name is > 64 should not return return ""; } Set lookedUpName = getPublicKeyFromName(directRoomId); + if( lookedUpName.length == 1) { DirectMessageRoom? room = directRooms[lookedUpName.first]; if( room != null) { @@ -856,7 +862,7 @@ class Store { return lookedUpName.first; } } else { - //print("got more than one pubkey for $directRoomId which are $lookedUpName"); + print("got more than one pubkey for $directRoomId which are $lookedUpName"); for( String key in directRooms.keys) { //print("in direct room key = $key"); if( key == directRoomId) { @@ -925,7 +931,7 @@ class Store { // Write the tree's events to file as one event's json per line Future writeEventsToFile(String filename) async { - //print("opening $filename to write to"); + if( gDebug > 0) print("opening $filename to write to."); try { final File file = File(filename); @@ -936,38 +942,42 @@ class Store { const int numLinesTogether = 100; // number of lines to write in one write call int linesWritten = 0; - if(gDebug > 0) log.info("eventsNotReadFromFile = ${eventsNotReadFromFile.length}. start writing."); - for( var k in eventsNotReadFromFile) { - Tree? t = allChildEventsMap[k]; - if( t != null) { - // only write if its not too old - if( gDontWriteOldEvents) { - if( t.event.eventData.createdAt < (DateTime.now().subtract(Duration(days: gDontSaveBeforeDays)).millisecondsSinceEpoch ~/ 1000)) { - continue; - } - } + for( var tree in allChildEventsMap.values) { - String line = "${t.event.originalJson}\n"; - nLinesStr += line; - eventCounter++; - if( t.event.eventData.kind == 1) { - countPosts++; + if( tree.event.readFromFile) { // ignore those already in file; only the new ones are writen/appended to file + continue; + } + + // only write if its not too old + if( gDontWriteOldEvents) { + if( tree.event.eventData.createdAt < getSecondsDaysAgo(gDontSaveBeforeDays)) { + continue; } } + //print("writing event "); + //tree.event.printEvent(0); print(""); + String line = "${tree.event.originalJson}\n"; + nLinesStr += line; + eventCounter++; + if( tree.event.eventData.kind == 1) { + countPosts++; + } + if( eventCounter % numLinesTogether == 0) { await file.writeAsString(nLinesStr, mode: FileMode.append).then( (file) => file); nLinesStr = ""; linesWritten += numLinesTogether; } - } + } // end for if( eventCounter > linesWritten) { + //print("writing.."); await file.writeAsString(nLinesStr, mode: FileMode.append).then( (file) => file); nLinesStr = ""; } - if(gDebug > 0) log.info("eventsNotReadFromFile = ${eventsNotReadFromFile.length}. finished writing eventCounter = ${eventCounter}."); + if(gDebug > 0) log.info("finished writing eventCounter = ${eventCounter}."); print("Appended $eventCounter new events to file \"$gEventsFilename\" of which ${countPosts} are posts."); } on Exception catch (e) { print("Could not open file $filename."); @@ -1114,8 +1124,8 @@ class Store { int count() { int totalEvents = 0; - for(int i = 0; i < children.length; i++) { - totalEvents += children[i].count(); // calling tree's count. + for(int i = 0; i < topPosts.length; i++) { + totalEvents += topPosts[i].count(); // calling tree's count. } return totalEvents; } @@ -1331,7 +1341,7 @@ void processReactions(Set events) { Store getTree(Set events) { if( events.isEmpty) { if(gDebug > 0) log.info("Warning: In printEventsAsTree: events length = 0"); - return Store([], {}, [], true, {}, {}, {}); + return Store([], {}, [], {}, {}); } // remove all events other than kind 0 (meta data), 1(posts replies likes), 3 (contact list), 7(reactions), 40 and 42 (chat rooms) diff --git a/test/nostr_console_test.dart b/test/nostr_console_test.dart index eb17566..90d5dd3 100644 --- a/test/nostr_console_test.dart +++ b/test/nostr_console_test.dart @@ -9,7 +9,7 @@ EventData exampleEdataChild = EventData("id2", "pubkey", 1111111, 1, "content ch Event exampleEvent = Event('event', 'id3', exampleEdata, ['relay name'], "[json]"); Event exampleEventChild = Event('event', 'id4', exampleEdataChild, ['relay name'], "[json]"); -Store exampleStore = Store([], {}, [], false, {}, {}, {}); +Store exampleStore = Store([], {}, [], {}, {}); Tree exampleTree = Tree.withoutStore(exampleEvent, []); void main() {