Add import and export for database

fix lazy loading resetting timeline
fix ridiculously long image URLs

docker-entrypoint.sh

@@ -6,7 +6,7 @@ if [ -n "$CACHE_RELAY" ]; then
   echo "Cache relay set to $CACHE_RELAY"
   sed -i 's/CACHE_RELAY_ENABLED = false/CACHE_RELAY_ENABLED = true/g' /usr/share/nginx/html/index.html
   CACHE_RELAY_PROXY="
-    location /cache-relay {
+    location /local-relay {
       proxy_pass http://$CACHE_RELAY/;
       proxy_http_version 1.1;
       proxy_set_header Upgrade \$http_upgrade;

src/classes/timeline-loader.ts

@@ -19,7 +19,7 @@ import {
   mapQueryMap,
   stringifyFilter,
 } from "../helpers/nostr/filter";
-import { localCacheRelay } from "../services/local-cache-relay";
+import { localRelay } from "../services/local-relay";
 import { relayRequest } from "../helpers/relay";
 import { Subscription } from "nostr-idb";
 
@@ -147,7 +147,7 @@ export default class TimelineLoader {
     if (isReplaceable(event.kind)) replaceableEventLoaderService.handleEvent(event);
 
     this.events.addEvent(event);
-    if (cache) localCacheRelay.publish(event);
+    if (cache) localRelay.publish(event);
   }
   private handleDeleteEvent(deleteEvent: NostrEvent) {
     const cord = deleteEvent.tags.find(isATag)?.[1];
@@ -177,7 +177,7 @@ export default class TimelineLoader {
     }
 
     for (const filters of Object.values(queries)) {
-      relayRequest(localCacheRelay, filters).then((events) => {
+      relayRequest(localRelay, filters).then((events) => {
         for (const e of events) this.handleEvent(e, false);
       });
     }

src/components/embed-types/common.tsx

@@ -6,7 +6,12 @@ import OpenGraphLink from "../open-graph-link";
 export function renderGenericUrl(match: URL) {
   return (
     <Link href={match.toString()} isExternal color="blue.500">
-      {match.toString()}
+      {match.protocol +
+        "//" +
+        match.host +
+        match.pathname +
+        (match.search && match.search.length < 20 ? "?" + match.search : "") +
+        (match.hash.length < 20 ? match.hash : "")}
     </Link>
   );
 }

src/components/note/text-note-contents.tsx

@@ -1,5 +1,5 @@
-import React from "react";
-import { Box, BoxProps } from "@chakra-ui/react";
+import React, { Suspense } from "react";
+import { Box, BoxProps, Spinner } from "@chakra-ui/react";
 
 import { DraftNostrEvent, NostrEvent } from "../../types/nostr-event";
 import { EmbedableContent, embedUrls, truncateEmbedableContent } from "../../helpers/embeds";
@@ -90,9 +90,11 @@ export const NoteContents = React.memo(
 
     return (
       <LightboxProvider>
-        <Box whiteSpace="pre-wrap" {...props}>
-          {content}
-        </Box>
+        <Suspense fallback={<Spinner />}>
+          <Box whiteSpace="pre-wrap" {...props}>
+            {content}
+          </Box>
+        </Suspense>
       </LightboxProvider>
     );
   },

src/components/timeline-page/generic-note-timeline/index.tsx

@@ -18,7 +18,7 @@ const NOTE_BUFFER = 5;
 const timelineNoteMinHeightCache = new WeakMap<TimelineLoader, Record<string, Record<string, number>>>();
 
 function GenericNoteTimeline({ timeline }: { timeline: TimelineLoader }) {
-  const events = useThrottle(useSubject(timeline.timeline), 100);
+  const events = useSubject(timeline.timeline);
   const [latest, setLatest] = useState(() => dayjs().unix());
 
   const location = useLocation();

src/helpers/nostr/filter.ts

@@ -1,6 +1,6 @@
 import stringify from "json-stringify-deterministic";
 import { NostrQuery, NostrRequestFilter, RelayQueryMap } from "../../types/nostr-query";
-import { LOCAL_CACHE_RELAY, LOCAL_CACHE_RELAY_ENABLED } from "../../services/local-cache-relay";
+import { LOCAL_CACHE_RELAY, LOCAL_CACHE_RELAY_ENABLED } from "../../services/local-relay";
 
 export function addQueryToFilter(filter: NostrRequestFilter, query: NostrQuery) {
   if (Array.isArray(filter)) {

src/services/channel-metadata.ts

@@ -12,7 +12,7 @@ import { logger } from "../helpers/debug";
 import db from "./db";
 import createDefer, { Deferred } from "../classes/deferred";
 import { getChannelPointer } from "../helpers/nostr/channel";
-import { LOCAL_CACHE_RELAY, LOCAL_CACHE_RELAY_ENABLED } from "./local-cache-relay";
+import { LOCAL_CACHE_RELAY, LOCAL_CACHE_RELAY_ENABLED } from "./local-relay";
 
 type Pubkey = string;
 type Relay = string;

src/services/db/index.ts

@@ -3,7 +3,7 @@ import { clearDB, deleteDB as nostrIDBDelete } from "nostr-idb";
 
 import { SchemaV1, SchemaV2, SchemaV3, SchemaV4, SchemaV5, SchemaV6, SchemaV7, SchemaV8 } from "./schema";
 import { logger } from "../../helpers/debug";
-import { localCacheDatabase } from "../local-cache-relay";
+import { localDatabase } from "../local-relay";
 
 const log = logger.extend("Database");
 
@@ -178,7 +178,7 @@ log("Open");
 
 export async function clearCacheData() {
   log("Clearing nostr-idb");
-  await clearDB(localCacheDatabase);
+  await clearDB(localDatabase);
 
   log("Clearing channelMetadata");
   await db.clear("channelMetadata");

src/services/event-exists.ts

@@ -8,7 +8,7 @@ import relayScoreboardService from "./relay-scoreboard";
 import { logger } from "../helpers/debug";
 import { matchFilter, matchFilters } from "nostr-tools";
 import { NostrEvent } from "../types/nostr-event";
-import { LOCAL_CACHE_RELAY, LOCAL_CACHE_RELAY_ENABLED } from "./local-cache-relay";
+import { LOCAL_CACHE_RELAY, LOCAL_CACHE_RELAY_ENABLED } from "./local-relay";
 
 function hashFilter(filter: NostrRequestFilter) {
   return stringify(filter);

src/services/local-relay.ts

@@ -3,25 +3,25 @@ import { Relay } from "nostr-tools";
 import { logger } from "../helpers/debug";
 import _throttle from "lodash.throttle";
 
-const log = logger.extend(`LocalCacheRelay`);
+const log = logger.extend(`LocalRelay`);
 const params = new URLSearchParams(location.search);
 
-const paramRelay = params.get("cacheRelay");
+const paramRelay = params.get("localRelay");
 // save the cache relay to localStorage
 if (paramRelay) {
-  localStorage.setItem("cacheRelay", paramRelay);
-  params.delete("cacheRelay");
+  localStorage.setItem("localRelay", paramRelay);
+  params.delete("localRelay");
   if (params.size === 0) location.search = params.toString();
 }
 
-const storedCacheRelayURL = localStorage.getItem("cacheRelay");
-const url = (storedCacheRelayURL && new URL(storedCacheRelayURL)) || new URL("/cache-relay", location.href);
+const storedCacheRelayURL = localStorage.getItem("localRelay");
+const url = (storedCacheRelayURL && new URL(storedCacheRelayURL)) || new URL("/local-relay", location.href);
 url.protocol = url.protocol === "https:" ? "wss:" : "ws:";
 
-export const LOCAL_CACHE_RELAY_ENABLED = !!window.CACHE_RELAY_ENABLED || !!localStorage.getItem("cacheRelay");
+export const LOCAL_CACHE_RELAY_ENABLED = !!window.CACHE_RELAY_ENABLED || !!localStorage.getItem("localRelay");
 export const LOCAL_CACHE_RELAY = url.toString();
 
-export const localCacheDatabase = await openDB();
+export const localDatabase = await openDB();
 
 function createRelay() {
   if (LOCAL_CACHE_RELAY_ENABLED) {
@@ -29,19 +29,19 @@ function createRelay() {
     return new Relay(LOCAL_CACHE_RELAY);
   } else {
     log(`Using IndexedDB`);
-    return new CacheRelay(localCacheDatabase);
+    return new CacheRelay(localDatabase, { maxEvents: 10000 });
   }
 }
 
-export const localCacheRelay = createRelay();
+export const localRelay = createRelay();
 
 function pruneLocalDatabase() {
-  if (localCacheRelay instanceof CacheRelay) {
-    pruneLastUsed(localCacheRelay.db, 20_000);
+  if (localRelay instanceof CacheRelay) {
+    pruneLastUsed(localRelay.db, 20_000);
   }
 }
 // connect without waiting
-localCacheRelay.connect().then(() => {
+localRelay.connect().then(() => {
   log("Connected");
 
   pruneLocalDatabase();
@@ -49,7 +49,7 @@ localCacheRelay.connect().then(() => {
 
 // keep the relay connection alive
 setInterval(() => {
-  if (!localCacheRelay.connected) localCacheRelay.connect().then(() => log("Reconnected"));
+  if (!localRelay.connected) localRelay.connect().then(() => log("Reconnected"));
 }, 1000 * 5);
 
 setInterval(() => {
@@ -58,5 +58,7 @@ setInterval(() => {
 
 if (import.meta.env.DEV) {
   //@ts-ignore
-  window.localCacheRelay = localCacheRelay;
+  window.localDatabase = localDatabase;
+  //@ts-ignore
+  window.localRelay = localRelay;
 }

src/services/relay-stats.ts

@@ -6,7 +6,7 @@ import SuperMap from "../classes/super-map";
 import { NostrEvent } from "../types/nostr-event";
 import relayInfoService from "./relay-info";
 import { normalizeRelayURL } from "../helpers/relay";
-import { localCacheRelay } from "./local-cache-relay";
+import { localRelay } from "./local-relay";
 import { MONITOR_STATS_KIND, SELF_REPORTED_KIND, getRelayURL } from "../helpers/nostr/relay-stats";
 
 const MONITOR_PUBKEY = "151c17c9d234320cf0f189af7b761f63419fd6c38c6041587a008b7682e4640f";
@@ -18,7 +18,7 @@ class RelayStatsService {
 
   constructor() {
     // load all stats from cache and subscribe to future ones
-    localCacheRelay.subscribe([{ kinds: [SELF_REPORTED_KIND, MONITOR_STATS_KIND] }], {
+    localRelay.subscribe([{ kinds: [SELF_REPORTED_KIND, MONITOR_STATS_KIND] }], {
       onevent: (e) => this.handleEvent(e, false),
     });
   }
@@ -34,12 +34,12 @@ class RelayStatsService {
     if (event.kind === SELF_REPORTED_KIND) {
       if (!sub.value || event.created_at > sub.value.created_at) {
         sub.next(event);
-        if (cache) localCacheRelay.publish(event);
+        if (cache) localRelay.publish(event);
       }
     } else if (event.kind === MONITOR_STATS_KIND) {
       if (!sub.value || event.created_at > sub.value.created_at) {
         sub.next(event);
-        if (cache) localCacheRelay.publish(event);
+        if (cache) localRelay.publish(event);
       }
     }
   }

src/services/replaceable-event-requester.ts

@@ -12,7 +12,7 @@ import db from "./db";
 import { nameOrPubkey } from "./user-metadata";
 import { getEventCoordinate, parseCoordinate } from "../helpers/nostr/events";
 import createDefer, { Deferred } from "../classes/deferred";
-import { LOCAL_CACHE_RELAY, LOCAL_CACHE_RELAY_ENABLED, localCacheRelay } from "./local-cache-relay";
+import { LOCAL_CACHE_RELAY, LOCAL_CACHE_RELAY_ENABLED, localRelay } from "./local-relay";
 import { relayRequest } from "../helpers/relay";
 
 type Pubkey = string;
@@ -199,7 +199,7 @@ class ReplaceableEventLoaderService {
     }
     const filters = Array.from(Object.values(kindFilters));
 
-    const events = await relayRequest(localCacheRelay, filters);
+    const events = await relayRequest(localRelay, filters);
     for (const event of events) {
       this.handleEvent(event, false);
       const cord = getEventCoordinate(event);
@@ -235,7 +235,7 @@ class ReplaceableEventLoaderService {
     if (this.writeCacheQueue.size === 0) return;
 
     this.dbLog(`Writing ${this.writeCacheQueue.size} events to database`);
-    for (const [_, event] of this.writeCacheQueue) localCacheRelay.publish(event);
+    for (const [_, event] of this.writeCacheQueue) localRelay.publish(event);
     this.writeCacheQueue.clear();
   }
   private async saveToCache(cord: string, event: NostrEvent) {
@@ -248,7 +248,7 @@ class ReplaceableEventLoaderService {
     const sub = this.events.get(cord);
 
     const relayUrls = Array.from(relays);
-    // TODO: use localCacheRelay instead
+    // TODO: use localRelay instead
     if (LOCAL_CACHE_RELAY_ENABLED) relayUrls.unshift(LOCAL_CACHE_RELAY);
 
     for (const relay of relayUrls) {

src/services/single-event.ts

@@ -4,7 +4,7 @@ import NostrRequest from "../classes/nostr-request";
 import Subject from "../classes/subject";
 import SuperMap from "../classes/super-map";
 import { NostrEvent } from "../types/nostr-event";
-import { localCacheRelay } from "./local-cache-relay";
+import { localRelay } from "./local-relay";
 import { relayRequest, safeRelayUrls } from "../helpers/relay";
 import { logger } from "../helpers/debug";
 
@@ -29,7 +29,7 @@ class SingleEventService {
   handleEvent(event: NostrEvent, cache = true) {
     this.cache.get(event.id).next(event);
 
-    if (cache) localCacheRelay.publish(event);
+    if (cache) localRelay.publish(event);
   }
 
   private batchRequestsThrottle = _throttle(this.batchRequests, RELAY_REQUEST_BATCH_TIME);
@@ -40,7 +40,7 @@ class SingleEventService {
     const loaded: string[] = [];
 
     // load from cache relay
-    const fromCache = await relayRequest(localCacheRelay, [{ ids }]);
+    const fromCache = await relayRequest(localRelay, [{ ids }]);
 
     for (const e of fromCache) {
       this.handleEvent(e, false);

src/views/settings/database-settings.tsx

@@ -1,4 +1,5 @@
-import { useState } from "react";
+import { useRef, useState } from "react";
+import { useAsync } from "react-use";
 import {
   Button,
   AccordionItem,
@@ -8,17 +9,19 @@ import {
   AccordionIcon,
   ButtonGroup,
   Text,
+  Input,
 } from "@chakra-ui/react";
-import { useAsync } from "react-use";
-import { countEvents, countEventsByKind } from "nostr-idb";
+import { addEvents, countEvents, countEventsByKind, getEventUID, updateUsed } from "nostr-idb";
+import stringify from "json-stringify-deterministic";
 
 import { clearCacheData, deleteDatabase } from "../../services/db";
 import { DatabaseIcon } from "../../components/icons";
-import { localCacheDatabase } from "../../services/local-cache-relay";
+import { localDatabase } from "../../services/local-relay";
+import { NostrEvent } from "../../types/nostr-event";
 
 function DatabaseStats() {
-  const { value: count } = useAsync(async () => await countEvents(localCacheDatabase), []);
-  const { value: kinds } = useAsync(async () => await countEventsByKind(localCacheDatabase), []);
+  const { value: count } = useAsync(async () => await countEvents(localDatabase), []);
+  const { value: kinds } = useAsync(async () => await countEventsByKind(localDatabase), []);
 
   return (
     <>
@@ -32,6 +35,49 @@ function DatabaseStats() {
   );
 }
 
+function ImportButton() {
+  const ref = useRef<HTMLInputElement | null>(null);
+  const [importing, setImporting] = useState(false);
+  const importFile = (file: File) => {
+    setImporting(true);
+    const reader = new FileReader();
+    reader.readAsText(file, "utf8");
+    reader.onload = async () => {
+      if (typeof reader.result !== "string") return;
+      const lines = reader.result.split("\n");
+      const events: NostrEvent[] = [];
+      for (const line of lines) {
+        try {
+          const event = JSON.parse(line) as NostrEvent;
+          events.push(event);
+        } catch (e) {}
+      }
+      await addEvents(localDatabase, events);
+      await updateUsed(
+        localDatabase,
+        events.map((e) => getEventUID(e)),
+      );
+      alert(`Imported ${events.length} events`);
+      setImporting(false);
+    };
+  };
+
+  return (
+    <>
+      <Input
+        hidden
+        type="file"
+        accept=".jsonl"
+        onChange={(e) => e.target.files?.[0] && importFile(e.target.files[0])}
+        ref={ref}
+      />
+      <Button onClick={() => ref.current?.click()} isLoading={importing}>
+        Import events
+      </Button>
+    </>
+  );
+}
+
 export default function DatabaseSettings() {
   const [clearing, setClearing] = useState(false);
   const handleClearData = async () => {
@@ -47,6 +93,19 @@ export default function DatabaseSettings() {
     setDeleting(false);
   };
 
+  const [exporting, setExporting] = useState(false);
+  const exportDatabase = async () => {
+    setExporting(true);
+    const rows = await localDatabase.getAll("events");
+    const lines = rows.map((row) => stringify(row.event));
+    const file = new File(lines, "noStrudel-export.jsonl", {
+      type: "application/jsonl",
+    });
+    const url = URL.createObjectURL(file);
+    window.open(url);
+    setExporting(false);
+  };
+
   return (
     <AccordionItem>
       <h2>
@@ -61,10 +120,14 @@ export default function DatabaseSettings() {
       <AccordionPanel>
         <DatabaseStats />
         <ButtonGroup mt="2">
-          <Button onClick={handleClearData} isLoading={clearing} isDisabled={clearing}>
-            Clear cache data
+          <Button onClick={handleClearData} isLoading={clearing}>
+            Clear cache
           </Button>
-          <Button colorScheme="red" onClick={handleDeleteDatabase} isLoading={deleting} isDisabled={deleting}>
+          <ImportButton />
+          <Button onClick={exportDatabase} isLoading={exporting}>
+            Export database
+          </Button>
+          <Button colorScheme="red" onClick={handleDeleteDatabase} isLoading={deleting}>
             Delete database
           </Button>
         </ButtonGroup>

src/views/settings/performance-settings.tsx

@@ -28,12 +28,9 @@ import {
 import { safeUrl } from "../../helpers/parse";
 import { AppSettings } from "../../services/settings/migrations";
 import { PerformanceIcon } from "../../components/icons";
-import { useLocalStorage } from "react-use";
-import { LOCAL_CACHE_RELAY } from "../../services/local-cache-relay";
 
 export default function PerformanceSettings() {
   const { register, formState } = useFormContext<AppSettings>();
-  const [localCacheRelay, setLocalCacheRelay] = useLocalStorage<boolean>("enable-cache-relay");
   const cacheDetails = useDisclosure();
 
   return (
@@ -129,7 +126,7 @@ export default function PerformanceSettings() {
                 <ModalCloseButton />
                 <ModalBody px="4" pb="4" pt="0">
                   <Text>
-                    When this is enabled noStrudel will connect to the relay at ws://{"<app domain>"}/cache-relay and
+                    When this is enabled noStrudel will connect to the relay at ws://{"<app domain>"}/local-relay and
                     use it to cache all events it finds.
                   </Text>
                   <Text>