highperfocused/scrapestr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update scraper

Browse Source
This commit is contained in:
mr0x50 2025-02-23 20:22:20 +01:00
parent b895c585a4
commit 01c956b786
1 changed files with 17 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
scraper/main.py
View File

@ -12,7 +12,7 @@ async def relay_websockets(input_websocket, output_websocket, kinds, sub_id):
event = json.loads(await input_websocket.recv()) event = json.loads(await input_websocket.recv())
try: try:
if(event[0] == "EVENT"): if(event[0] == "EVENT"):
print("Got event: ", event[2]['id']) print("Received ID: ",event[2]['id']," // Kind: ",event[2]['kind'])
# Forward the event to output websocket # Forward the event to output websocket
await output_websocket.send(json.dumps(["EVENT", sub_id, event[2]])) await output_websocket.send(json.dumps(["EVENT", sub_id, event[2]]))
elif(event[0] == "EOSE"): elif(event[0] == "EOSE"):
@ -30,18 +30,7 @@ async def relay_websockets(input_websocket, output_websocket, kinds, sub_id):
# If either websocket is closed, attempt to reconnect # If either websocket is closed, attempt to reconnect
print("Connection closed, attempting to reconnect...") print("Connection closed, attempting to reconnect...")
await asyncio.sleep(1) await asyncio.sleep(1)
try: break
new_sub_id = str(uuid4())
async with websockets.connect(os.environ.get("INPUT_RELAY")) as new_input_websocket, \
websockets.connect(os.environ.get("OUTPUT_RELAY")) as new_output_websocket:
message = f'["REQ", "{new_sub_id}", {{"kinds": {kinds}, "limit": 10}}]'
await new_input_websocket.send(message)
await relay_websockets(new_input_websocket, new_output_websocket, kinds, new_sub_id)
except Exception as error:
# If the reconnection attempt fails, repeat the loop and try again
print(f"Failed to reconnect: {error}")
continue
async def main(): async def main():
print("Scraper started...") print("Scraper started...")
@ -56,19 +45,22 @@ async def main():
if not output_url: if not output_url:
raise ValueError("Please set the OUTPUT_RELAY environment variable") raise ValueError("Please set the OUTPUT_RELAY environment variable")
try: while True:
sub_id = str(uuid4()) try:
async with websockets.connect(input_url) as input_websocket, \ sub_id = str(uuid4())
websockets.connect(output_url) as output_websocket: async with websockets.connect(input_url) as input_websocket, \
message = f'["REQ", "{sub_id}", {{"kinds": {kinds}}}]' websockets.connect(output_url) as output_websocket:
await input_websocket.send(message) message = f'["REQ", "{sub_id}", {{"kinds": {kinds}}}]'
await relay_websockets(input_websocket, output_websocket, kinds, sub_id) await input_websocket.send(message)
await relay_websockets(input_websocket, output_websocket, kinds, sub_id)
except Exception as error: except Exception as error:
# If the initial connection attempt fails, attempt to reconnect immediately # If the initial connection attempt fails, attempt to reconnect immediately
print(f"Failed to connect: {error}") print(f"Failed to connect: {error}")
await asyncio.sleep(1) await asyncio.sleep(1)
await main() if "maximum recursion depth exceeded" in str(error):
raise RuntimeError("Maximum recursion depth exceeded, crashing application.")
continue
# Start the script # Start the script
asyncio.run(main()) asyncio.run(main())