strix

commit a18270a9787c22fa9445eb856c843494d7102d24
parent d234adbe45786d6d64d2c838a1562ba4b74afaca
Author: Brian C. Lane <bcl@brianlane.com>
Date:   Sun,  3 Dec 2017 12:17:04 -0800

Limit the number of running processes to the # of cpus

This should prevent overwhelimg the system with too many parallel ffmpeg
instances.

Diffstat:
M
src/strix/queue.py
 | 
5
+++++

1 file changed, 5 insertions(+), 0 deletions(-)
diff --git a/src/strix/queue.py b/src/strix/queue.py
@@ -110,6 +110,11 @@ def monitor_queue(logging_queue: mp.Queue, base_dir: str, quit: mp.Event) -> Non
 
         log.debug("queue check", queue_path=queue_path)
         for event_file in glob(os.path.join(queue_path, "*")):
+            # Limit the number of processes
+            if len(threads) > mp.cpu_count():
+                log.debug("Too many running threads (%d), not adding a new one yet.", len(threads))
+                break
+
             os.unlink(event_file)
             event = os.path.split(event_file)[-1]
             thread = mp.Process(target=process_event, args=(log, base_dir, event))