multiprocessing memorandum

Memorandum of multiprocessing

Environment is Windows 10

If you want to lock in Pool, create Lock from Manager

• Cannot be used with multiprocessing.Lock.

from multiprocessing import Pool,Manager
import logging

LOGGER = multiprocessing.get_logger()
LOGGER.setLevel(logging.ERROR)

def f(lock,x):
    lock.acquire()
    try:
        LOGGER.error('hoge{}'.format(x))
    finally:
        lock.release()

    return x*x

def main():
    m = Manager()
    lock = m.Lock()
    with Pool(2) as p:
        jobs = [ p.apply_async(f, (lock,i)) for i in range(10) ]
        for job in jobs:
            print(job.get())

if __name__ == '__main__':
    main()

multiprocessing.log_to_stderr () prints 2 logs to standard output

Handling that two handlers are registered?

from multiprocessing import Pool,Lock
import multiprocessing
import logging

LOGGER = multiprocessing.log_to_stderr()
LOGGER.setLevel(logging.ERROR)

def f(lock,x):
    lock.acquire()
    try:
        LOGGER.error('hoge{}'.format(x))
    finally:
        lock.release()

    return x*x

def main():
    m = multiprocessing.Manager()
    lock = m.Lock()
    with Pool(2) as p:
        jobs = [ p.apply_async(f, (lock,i)) for i in range(5) ]
        for job in jobs:
            print(job.get())

if __name__ == '__main__':
    main()

[ERROR/SpawnPoolWorker-2] hoge0
[ERROR/SpawnPoolWorker-2] hoge0
0
[ERROR/SpawnPoolWorker-3] hoge1
[ERROR/SpawnPoolWorker-3] hoge1
1
[ERROR/SpawnPoolWorker-2] hoge2
[ERROR/SpawnPoolWorker-2] hoge2
4
[ERROR/SpawnPoolWorker-3] hoge3
[ERROR/SpawnPoolWorker-3] hoge3
9
[ERROR/SpawnPoolWorker-2] hoge4
[ERROR/SpawnPoolWorker-2] hoge4
16

I tried adding a file handler

from multiprocessing import Pool,Lock
import multiprocessing
import logging

logger = logging.getLogger()
#Generate handler
file_handler = logging.FileHandler('test.log', 'a')
#handler log level setting(Level of error message output by handler)
file_handler.setLevel(logging.DEBUG)
#Log output format setting
handler_format = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
file_handler.setFormatter(handler_format)
logger.addHandler(file_handler)

LOGGER = multiprocessing.log_to_stderr()
LOGGER.setLevel(logging.ERROR)
LOGGER.addHandler(file_handler)

def f(lock,x):
    lock.acquire()
    try:
        LOGGER.error('hoge{}'.format(x))
    finally:
        lock.release()

    return x*x

def main():
    m = multiprocessing.Manager()
    lock = m.Lock()
    logger.error('process start')
    with Pool(2) as p:
        jobs = [ p.apply_async(f, (lock,i)) for i in range(5) ]
        for job in jobs:
            print(job.get())

if __name__ == '__main__':
    main()

[ERROR/SpawnPoolWorker-3] hoge0
[ERROR/SpawnPoolWorker-3] hoge0
0
[ERROR/SpawnPoolWorker-2] hoge1
[ERROR/SpawnPoolWorker-2] hoge1
1[ERROR/SpawnPoolWorker-3] hoge2

[ERROR/SpawnPoolWorker-3] hoge2
4
[ERROR/SpawnPoolWorker-2] hoge3
[ERROR/SpawnPoolWorker-2] hoge3
9[ERROR/SpawnPoolWorker-3] hoge4

[ERROR/SpawnPoolWorker-3] hoge4
16

test.log

2020-05-02 17:21:59,128 - root - ERROR - process start
2020-05-02 17:21:59,968 - multiprocessing - ERROR - hoge0
2020-05-02 17:21:59,981 - multiprocessing - ERROR - hoge1
2020-05-02 17:21:59,987 - multiprocessing - ERROR - hoge2
2020-05-02 17:21:59,994 - multiprocessing - ERROR - hoge3
2020-05-02 17:22:00,002 - multiprocessing - ERROR - hoge4