multiprocessingの備忘録
環境はWindows10
PoolでLockしたい場合はManagerからLockを生成する
※multiprocessing.Lockではだめ。
from multiprocessing import Pool,Manager
import logging
LOGGER = multiprocessing.get_logger()
LOGGER.setLevel(logging.ERROR)
def f(lock,x):
lock.acquire()
try:
LOGGER.error('hoge{}'.format(x))
finally:
lock.release()
return x*x
def main():
m = Manager()
lock = m.Lock()
with Pool(2) as p:
jobs = [ p.apply_async(f, (lock,i)) for i in range(10) ]
for job in jobs:
print(job.get())
if __name__ == '__main__':
main()
multiprocessing.log_to_stderr()は標準出力に2個ログがでる
ハンドラが2個登録されている扱い？
from multiprocessing import Pool,Lock
import multiprocessing
import logging
LOGGER = multiprocessing.log_to_stderr()
LOGGER.setLevel(logging.ERROR)
def f(lock,x):
lock.acquire()
try:
LOGGER.error('hoge{}'.format(x))
finally:
lock.release()
return x*x
def main():
m = multiprocessing.Manager()
lock = m.Lock()
with Pool(2) as p:
jobs = [ p.apply_async(f, (lock,i)) for i in range(5) ]
for job in jobs:
print(job.get())
if __name__ == '__main__':
main()
[ERROR/SpawnPoolWorker-2] hoge0
[ERROR/SpawnPoolWorker-2] hoge0
0
[ERROR/SpawnPoolWorker-3] hoge1
[ERROR/SpawnPoolWorker-3] hoge1
1
[ERROR/SpawnPoolWorker-2] hoge2
[ERROR/SpawnPoolWorker-2] hoge2
4
[ERROR/SpawnPoolWorker-3] hoge3
[ERROR/SpawnPoolWorker-3] hoge3
9
[ERROR/SpawnPoolWorker-2] hoge4
[ERROR/SpawnPoolWorker-2] hoge4
16
ファイルハンドラを追加してみた
from multiprocessing import Pool,Lock
import multiprocessing
import logging
logger = logging.getLogger()
# handlerの生成
file_handler = logging.FileHandler('test.log', 'a')
# handlerのログレベル設定(ハンドラが出力するエラーメッセージのレベル)
file_handler.setLevel(logging.DEBUG)
# ログ出力フォーマット設定
handler_format = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
file_handler.setFormatter(handler_format)
logger.addHandler(file_handler)
LOGGER = multiprocessing.log_to_stderr()
LOGGER.setLevel(logging.ERROR)
LOGGER.addHandler(file_handler)
def f(lock,x):
lock.acquire()
try:
LOGGER.error('hoge{}'.format(x))
finally:
lock.release()
return x*x
def main():
m = multiprocessing.Manager()
lock = m.Lock()
logger.error('process start')
with Pool(2) as p:
jobs = [ p.apply_async(f, (lock,i)) for i in range(5) ]
for job in jobs:
print(job.get())
if __name__ == '__main__':
main()
[ERROR/SpawnPoolWorker-3] hoge0
[ERROR/SpawnPoolWorker-3] hoge0
0
[ERROR/SpawnPoolWorker-2] hoge1
[ERROR/SpawnPoolWorker-2] hoge1
1[ERROR/SpawnPoolWorker-3] hoge2
[ERROR/SpawnPoolWorker-3] hoge2
4
[ERROR/SpawnPoolWorker-2] hoge3
[ERROR/SpawnPoolWorker-2] hoge3
9[ERROR/SpawnPoolWorker-3] hoge4
[ERROR/SpawnPoolWorker-3] hoge4
16
test.log
2020-05-02 17:21:59,128 - root - ERROR - process start
2020-05-02 17:21:59,968 - multiprocessing - ERROR - hoge0
2020-05-02 17:21:59,981 - multiprocessing - ERROR - hoge1
2020-05-02 17:21:59,987 - multiprocessing - ERROR - hoge2
2020-05-02 17:21:59,994 - multiprocessing - ERROR - hoge3
2020-05-02 17:22:00,002 - multiprocessing - ERROR - hoge4