[releng/abderrahim/threadpool] convert-to-tarballs: use multiple threads to speed up conversion
- From: Abderrahim Kitouni <akitouni src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [releng/abderrahim/threadpool] convert-to-tarballs: use multiple threads to speed up conversion
- Date: Mon, 4 May 2020 14:11:19 +0000 (UTC)
commit 4447465033a1a475119aba1a51be8ff7d4959b9c
Author: Abderrahim Kitouni <akitouni gnome org>
Date: Sun May 3 11:45:41 2020 +0100
convert-to-tarballs: use multiple threads to speed up conversion
tools/smoketesting/convert-to-tarballs.py | 35 +++++++++++++++++++++++--------
1 file changed, 26 insertions(+), 9 deletions(-)
---
diff --git a/tools/smoketesting/convert-to-tarballs.py b/tools/smoketesting/convert-to-tarballs.py
index 88ac545..8784455 100755
--- a/tools/smoketesting/convert-to-tarballs.py
+++ b/tools/smoketesting/convert-to-tarballs.py
@@ -29,6 +29,7 @@ import os
from xml.etree import ElementTree
from ruamel import yaml
from collections import defaultdict
+from concurrent.futures import ThreadPoolExecutor, as_completed
from tqdm import tqdm
from downloadsites import SITE_KINDS
@@ -175,8 +176,8 @@ class ConvertToTarballs:
print("- Can't update tarball for module '{}'".format(module))
def convert_modules(self, directories):
- to_convert = []
- to_update = []
+ to_convert = {}
+ to_update = {}
for directory in directories:
for filename in os.listdir(directory):
@@ -191,18 +192,34 @@ class ConvertToTarballs:
module_kind = self._get_module_kind(element)
if module_kind == 'git':
- to_convert.append((name, fullpath, element))
+ to_convert[name] = fullpath, element
elif module_kind == 'tarball':
- to_update.append((name, fullpath, element))
+ to_update[name] = fullpath, element
+
+ executor = ThreadPoolExecutor()
+
+ converted = None
+ if self.convert:
+ converted = {executor.submit(self._convert_one_module, name, True): name for name in to_convert}
+
+ updated = {executor.submit(self._convert_one_module, name, False): name for name in to_update}
+
+ if converted:
+ for future in tqdm(as_completed(converted), 'Converting git repos',
+ unit='', total=len(converted)):
+ name = converted[future]
+ fullpath, element = to_convert[name]
+ location, checksum = future.result()
- if self.convert and to_convert:
- for name, fullpath, element in tqdm(to_convert, 'Converting git repos', unit=''):
- location, checksum = self._convert_one_module(name, True)
if location:
self._write_bst_file(fullpath, element, location, checksum)
- for name, fullpath, element in tqdm(to_update, 'Updating existing tarballs', unit=''):
- location, checksum = self._convert_one_module(name, False)
+ for future in tqdm(as_completed(updated), 'Updating existing tarballs',
+ unit='', total=len(updated)):
+ name = updated[future]
+ fullpath, element = to_update[name]
+ location, checksum = future.result()
+
if location:
self._write_bst_file(fullpath, element, location, checksum)
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]