Skip to content
140 changes: 140 additions & 0 deletions scripts/multiprocess_runcontrol_driver.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
#!/bin/env python3

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I thikn the top of this file is missing a bit of documentation. People should know how to run this and what this script does, as well as a brief description of how to interact with the shells. (and then maybe add a todo to link these to the PMaaS docs when we eventually make them >.<)


import asyncio
import sys
import time
from daqconf.utils import find_free_port

last_msg_time = 0

async def read_stream(stream, process_name, completion_event):
"""Asynchronously reads lines from a stream and prints them immediately."""
global last_msg_time
while True:
line = await stream.readline()
if not line:
break
decoded_line = line.decode().rstrip()

if "*** COMMAND HAS COMPLETED ***" in decoded_line:
#print("=== Setting the completion event ===", flush=True)
completion_event.set()
continue

# Decode and strip line endings
print(f"[{process_name}] {line.decode().rstrip()}", flush=True)
last_msg_time = time.time()

async def interactive_manager(commands):
processes = {}
tasks = []
command_completion_event = asyncio.Event()
global last_msg_time

# 1. Start all interactive subprocesses
for cmd in commands:
name = cmd.pop(0)
print()
print(f"*** Starting \"{cmd[0]}\" with local process name \"{name}\"...")
print()
proc = await asyncio.create_subprocess_exec(
*cmd,
stdin=asyncio.subprocess.PIPE,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.STDOUT
)
processes[name] = proc

# 2. Schedule output reading tasks concurrently
tasks.append(asyncio.create_task(read_stream(proc.stdout, name, command_completion_event)))

time.sleep(2)

print()
print(f"*** Started {len(processes)} processes. Type: '<process_name>:<input>' (e.g., shell:help)")
print("*** Type 'exit' to quit everything.")
print()

# 3. Handle interactive user input from the main terminal

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

One thing I noticed is that arrow keys don't work. Makes a bit hard to submit the previous commands.

I dont think this is a blocker for this merge tho. It would be nice if it can make it but if its too difficult I'm happy for this to be deferred

loop = asyncio.get_running_loop()
reader = asyncio.StreamReader()
protocol = asyncio.StreamReaderProtocol(reader)
await loop.connect_read_pipe(lambda: protocol, sys.stdin)

try:
while True:
print("\nmprc_drvr> ", end="", flush=True)
user_line = await reader.readline()
if not user_line:
break

command_text = user_line.decode().strip()
if command_text.lower() == 'exit':
break

# Parse target process and message (Format: Proc-1:your_command)
if ":" in command_text:
target, msg = command_text.split(":", 1)
target = target.strip()

if target in processes:
proc = processes[target]
if proc.returncode is None: # Check if still running
cmd_start_time = time.time()
proc.stdin.write((msg + "\n").encode())
await proc.stdin.drain()
print(f"[System] Sent to {target}: {msg}")
if "drunc" in target:
proc.stdin.write(("echo '*** COMMAND HAS COMPLETED ***'\n").encode())
await proc.stdin.drain()
#print(f"[System] Sent to {target}: echo '*** COMMAND HAS COMPLETED ***'")

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Here and a couple of others: Some stray comments that should be deleted before merge into develop

await command_completion_event.wait()
command_completion_event.clear()
else:
now = time.time()
#print(f"{cmd_start_time} {last_msg_time} {now}", flush=True)
while True:
if last_msg_time <= cmd_start_time:
if now - cmd_start_time > 5:
break
else:
if now - last_msg_time >= 5:
break
#print(f"{cmd_start_time} {last_msg_time} {now}", flush=True)
await asyncio.sleep(0.25)
now = time.time()
#print(f"{cmd_start_time} {last_msg_time} {now}", flush=True)
else:
print(f"[System] Error: {target} has already exited.")
else:
print(f"[System] Error: Process '{target}' not found.")
else:
print("[System] Invalid format. Use: <process_name>:<command>")
Comment on lines +76 to +112

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Quite a nested block of code. I prefer if its inverted to flatten them:

Suggested change
if ":" in command_text:
target, msg = command_text.split(":", 1)
target = target.strip()
if target in processes:
proc = processes[target]
if proc.returncode is None: # Check if still running
cmd_start_time = time.time()
proc.stdin.write((msg + "\n").encode())
await proc.stdin.drain()
print(f"[System] Sent to {target}: {msg}")
if "drunc" in target:
proc.stdin.write(("echo '*** COMMAND HAS COMPLETED ***'\n").encode())
await proc.stdin.drain()
#print(f"[System] Sent to {target}: echo '*** COMMAND HAS COMPLETED ***'")
await command_completion_event.wait()
command_completion_event.clear()
else:
now = time.time()
#print(f"{cmd_start_time} {last_msg_time} {now}", flush=True)
while True:
if last_msg_time <= cmd_start_time:
if now - cmd_start_time > 5:
break
else:
if now - last_msg_time >= 5:
break
#print(f"{cmd_start_time} {last_msg_time} {now}", flush=True)
await asyncio.sleep(0.25)
now = time.time()
#print(f"{cmd_start_time} {last_msg_time} {now}", flush=True)
else:
print(f"[System] Error: {target} has already exited.")
else:
print(f"[System] Error: Process '{target}' not found.")
else:
print("[System] Invalid format. Use: <process_name>:<command>")
if ":" not in command_text:
print("[System] Invalid format. Use: <process_name>:<command>")
continue
target, msg = command_text.split(":", 1)
target = target.strip()
if target not in processes:
print(f"[System] Error: Process '{target}' not found.")
continue
proc = processes[target]
if proc.returncode is not None:
print(f"[System] Error: {target} has already exited.")
continue
cmd_start_time = time.time()
proc.stdin.write((msg + "\n").encode())
await proc.stdin.drain()
print(f"[System] Sent to {target}: {msg}")
if "drunc" in target:
proc.stdin.write(("echo '*** COMMAND HAS COMPLETED ***'\n").encode())
await proc.stdin.drain()
await command_completion_event.wait()
command_completion_event.clear()
continue
now = time.time()
while True:
if last_msg_time <= cmd_start_time and now - cmd_start_time > 5:
break
if last_msg_time > cmd_start_time and now - last_msg_time >= 5:
break
await asyncio.sleep(0.25)
now = time.time()

Comments on each if statement as necessary too if theyre not clear enough


except asyncio.CancelledError:
pass
finally:
# 4. Cleanup and terminate remaining processes
print("\n[System] Shutting down processes...")
for name, proc in reversed(processes.items()):
if proc.returncode is None:
proc.terminate()
await proc.wait()

# Cancel background reading tasks
for task in tasks:
task.cancel()

if __name__ == "__main__":
pm_port = find_free_port(50001, 52000)

interactive_cmds = [
["pm", "drunc-process-manager", "ssh-standalone", str(pm_port)], # Launch Interactive Python Instance 1
["pmshell", "drunc-process-manager-shell", f"grpc://localhost:{pm_port}"], # Launch Interactive Python Instance 2
["drunc", "drunc-unified-shell", f"grpc://localhost:{pm_port}", "config/daqsystemtest/example-configs.data.xml", "local-1x1-config", "biery-local-test"] # Launch Interactive Python Instance 3
]
Comment on lines +131 to +135

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is the purpose of this script for a test/demonstration of the PMaaS functionality or are people expected to use this more generally / in a wider use case?

If its the latter, then maybe people might want to change these lines to fit their needs? I would suggest the following:

  • The easiest might be a comment / some docs here that says 'please change this to change what set of applications you would like to run'. In these comments, it would have to be clear that the first element of this list is the 'alias' of the application and the rest are just the usual application setups
  • Something harder, possibly not worth it, is to open this up as an runtime argument. I'm ambivalent / semi against this idea though


try:
asyncio.run(interactive_manager(interactive_cmds))
except KeyboardInterrupt:
print("\n[System] Exited via Ctrl+C.")
Loading