25 lines
704 B
Python
25 lines
704 B
Python
import multiprocessing as mp
|
|
import queue
|
|
import threading
|
|
from typing import Union
|
|
|
|
import dill
|
|
|
|
|
|
def mapper_function(
|
|
input_queue: Union[mp.Queue, queue.Queue],
|
|
output_queue: Union[mp.Queue, queue.Queue],
|
|
should_stop: Union[mp.Event, threading.Event],
|
|
serialized_map_function: bytes,
|
|
):
|
|
map_function = dill.loads(serialized_map_function)
|
|
try:
|
|
while not should_stop.is_set():
|
|
try:
|
|
input_chunk = input_queue.get_nowait()
|
|
output_chunk = [(i, map_function(v)) for i, v in input_chunk]
|
|
output_queue.put(output_chunk)
|
|
except queue.Empty:
|
|
pass
|
|
except KeyboardInterrupt:
|
|
return
|