highperfocused/scrapestr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update scraper

Browse Source
This commit is contained in:
mr0x50 2025-02-23 20:22:20 +01:00
parent b895c585a4
commit 01c956b786
1 changed files with 17 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
scraper/main.py
View File

@ -12,7 +12,7 @@ async def relay_websockets(input_websocket, output_websocket, kinds, sub_id):
event = json.loads(await input_websocket.recv())
try:
if(event[0] == "EVENT"):
print("Got event: ", event[2]['id'])
print("Received ID: ",event[2]['id']," // Kind: ",event[2]['kind'])
# Forward the event to output websocket
await output_websocket.send(json.dumps(["EVENT", sub_id, event[2]]))
elif(event[0] == "EOSE"):
@ -30,18 +30,7 @@ async def relay_websockets(input_websocket, output_websocket, kinds, sub_id):
# If either websocket is closed, attempt to reconnect
print("Connection closed, attempting to reconnect...")
await asyncio.sleep(1)
try:
new_sub_id = str(uuid4())
async with websockets.connect(os.environ.get("INPUT_RELAY")) as new_input_websocket, \
websockets.connect(os.environ.get("OUTPUT_RELAY")) as new_output_websocket:
message = f'["REQ", "{new_sub_id}", {{"kinds": {kinds}, "limit": 10}}]'
await new_input_websocket.send(message)
await relay_websockets(new_input_websocket, new_output_websocket, kinds, new_sub_id)
except Exception as error:
# If the reconnection attempt fails, repeat the loop and try again
print(f"Failed to reconnect: {error}")
continue
break
async def main():
print("Scraper started...")
@ -56,19 +45,22 @@ async def main():
if not output_url:
raise ValueError("Please set the OUTPUT_RELAY environment variable")
try:
sub_id = str(uuid4())
async with websockets.connect(input_url) as input_websocket, \
websockets.connect(output_url) as output_websocket:
message = f'["REQ", "{sub_id}", {{"kinds": {kinds}}}]'
await input_websocket.send(message)
await relay_websockets(input_websocket, output_websocket, kinds, sub_id)
while True:
try:
sub_id = str(uuid4())
async with websockets.connect(input_url) as input_websocket, \
websockets.connect(output_url) as output_websocket:
message = f'["REQ", "{sub_id}", {{"kinds": {kinds}}}]'
await input_websocket.send(message)
await relay_websockets(input_websocket, output_websocket, kinds, sub_id)
except Exception as error:
# If the initial connection attempt fails, attempt to reconnect immediately
print(f"Failed to connect: {error}")
await asyncio.sleep(1)
await main()
except Exception as error:
# If the initial connection attempt fails, attempt to reconnect immediately
print(f"Failed to connect: {error}")
await asyncio.sleep(1)
if "maximum recursion depth exceeded" in str(error):
raise RuntimeError("Maximum recursion depth exceeded, crashing application.")
continue
# Start the script
asyncio.run(main())