[jhbuild] Fix performance bottleneck in jhbuild tinderbox
- From: Craig Keogh <cskeogh src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [jhbuild] Fix performance bottleneck in jhbuild tinderbox
- Date: Sun, 9 Dec 2012 11:32:26 +0000 (UTC)
commit 302573d3e5ff4744b78a3b1e7b9618e2d026d558
Author: Ryan Lortie <desrt desrt ca>
Date: Tue Nov 27 21:40:24 2012 -0500
Fix performance bottleneck in jhbuild tinderbox
The HTML logging for jhbuild tinderbox contained code that would read
1024 bytes of build stdout and then sleep for 0.1 seconds. This
effectively rate-limited builds to producing 10k of output per second,
which is easily overrun by chatty builds like WebKit on fast systems.
This commit changes the buffer size to 10000 and eliminates the sleep
entirely. It also reworks the handling of when an output line is
printed, always waiting until a newline is seen, except at the very end
of builds.
This prevents this sort of output:
checking for sys/types.h...
yes
that was often seen (on separate lines) in the logfiles before.
https://bugzilla.gnome.org/show_bug.cgi?id=689194
jhbuild/utils/cmds.py | 24 +++++++++++-------------
1 files changed, 11 insertions(+), 13 deletions(-)
---
diff --git a/jhbuild/utils/cmds.py b/jhbuild/utils/cmds.py
index 9075948..d0bbcac 100644
--- a/jhbuild/utils/cmds.py
+++ b/jhbuild/utils/cmds.py
@@ -182,7 +182,7 @@ def pprint_output(pipe, format_line):
rlist, wlist, xlist = select.select(read_set, [], [])
if pipe.stdout in rlist:
- out_chunk = os.read(pipe.stdout.fileno(), 1024)
+ out_chunk = os.read(pipe.stdout.fileno(), 10000)
if out_chunk == '':
pipe.stdout.close()
read_set.remove(pipe.stdout)
@@ -193,16 +193,9 @@ def pprint_output(pipe, format_line):
pos = out_data.find('\n')
format_line(out_data[:pos+1], False)
out_data = out_data[pos+1:]
- # FIXME: A bug here: If the last out_chunk is exactly 1024
- # and there is no final newline, the last line will not be
- # shown. Due to the '1024 chunk to line with no lookahead'
- # algorithm used here. I can't see a solution.
- if len(out_chunk) < 1024 and out_data:
- format_line(out_data, False)
- out_data = ''
-
+
if pipe.stderr in rlist:
- err_chunk = os.read(pipe.stderr.fileno(), 1024)
+ err_chunk = os.read(pipe.stderr.fileno(), 10000)
if err_chunk == '':
pipe.stderr.close()
read_set.remove(pipe.stderr)
@@ -214,11 +207,16 @@ def pprint_output(pipe, format_line):
# safeguard against tinderbox that close stdin
if sys.stdin in rlist and sys.stdin.isatty():
- in_chunk = os.read(sys.stdin.fileno(), 1024)
+ in_chunk = os.read(sys.stdin.fileno(), 10000)
if pipe.stdin:
os.write(pipe.stdin.fileno(), in_chunk)
-
- select.select([],[],[],.1) # give a little time for buffers to fill
+
+ # flush the remainder of stdout/stderr data lacking newlines
+ if out_data:
+ format_line(out_data, False)
+ if err_data:
+ format_line(err_data, True)
+
except KeyboardInterrupt:
# interrupt received. Send SIGINT to child process.
try:
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]