1
0
mirror of synced 2024-11-25 18:54:22 +01:00

Thread the cleaning process

This commit is contained in:
jvoisin 2018-04-14 16:10:45 +02:00
parent cef5068fe9
commit 6f4ed2490f

32
main.py
View File

@ -3,6 +3,9 @@
import os import os
import mimetypes import mimetypes
import argparse import argparse
from threading import Thread
import multiprocessing
from queue import Queue
from src import parser_factory from src import parser_factory
@ -75,6 +78,14 @@ def __get_files_recursively(files):
for _f in _files: for _f in _files:
yield os.path.join(path, _f) yield os.path.join(path, _f)
def __do_clean_async(q):
while True:
f = q.get()
if f is None: # nothing more to process
return
clean_meta(f)
q.task_done()
def main(): def main():
arg_parser = create_arg_parser() arg_parser = create_arg_parser()
@ -86,12 +97,27 @@ def main():
show_parsers() show_parsers()
return return
if args.show: elif args.show:
for f in __get_files_recursively(args.files): for f in __get_files_recursively(args.files):
show_meta(f) show_meta(f)
else: return
else: # Thread the cleaning
q = Queue(maxsize=0)
threads = list()
for f in __get_files_recursively(args.files): for f in __get_files_recursively(args.files):
clean_meta(f) q.put(f)
for _ in range(multiprocessing.cpu_count()):
worker = Thread(target=__do_clean_async, args=(q, ))
worker.start()
threads.append(worker)
for _ in range(multiprocessing.cpu_count()):
q.put(None) # stop the threads
for worker in threads:
worker.join()
if __name__ == '__main__': if __name__ == '__main__':