cli: refactor progress indication

This patch:

  * Increases the encapsulation of the different progress
    indicators: the progress code does not access its
    owner's attributes anymore
  * Makes the progress update at most every 100 ms
  * Makes the progress indication use the current timestamp
    and the trace collection's begin/end timestamps to
    approximate its position; falls back on approximating
    with the size if those timestamps are not available
  * Adds the general --progress-use-size CLI option to
    reverse to the old default behaviour of approximating
    with the size

Signed-off-by: Philippe Proulx <eeppeliteloop@gmail.com>

diff --git a/lttnganalyses/cli/command.py b/lttnganalyses/cli/command.py
index c277709ad74a2e6ffb9f21fe91b980de2560e533..3250a36bb9491114c34d4ac7b9f984ab3ee8dc06 100644 (file)
--- a/lttnganalyses/cli/command.py
+++ b/lttnganalyses/cli/command.py
@@ -172,6 +172,8 @@ class Command:
             self._gen_error('Failed to open ' + self._args.path, -1)
         self._handles = handles
         self._traces = traces
+        self._ts_begin = traces.timestamp_begin
+        self._ts_end = traces.timestamp_end
         self._process_date_args()
         self._read_tracer_version()
         if not self._args.skip_validation:
@@ -251,32 +253,40 @@ class Command:
         self._mi_print()
 
     def _pb_setup(self):
-        if self._mi_mode:
-            if self._args.output_progress:
-                progressbar.mi_progress_setup(self)
-        else:
-            progressbar.progressbar_setup(self)
+        if self._args.no_progress:
+            return
+
+        ts_end = self._ts_end
+
+        if self._analysis_conf.end_ts is not None:
+            ts_end = self._analysis_conf.end_ts
 
-    def _pb_update(self):
         if self._mi_mode:
-            if self._args.output_progress:
-                progressbar.mi_progress_update(self)
+            cls = progressbar.MiProgress
         else:
-            progressbar.progressbar_update(self)
+            cls = progressbar.FancyProgressBar
+
+        self._progress = cls(self._ts_begin, ts_end, self._args.path,
+                             self._args.progress_use_size)
+
+    def _pb_update(self, event):
+        if self._args.no_progress:
+            return
+
+        self._progress.update(event)
 
     def _pb_finish(self):
-        if self._mi_mode:
-            if self._args.output_progress:
-                progressbar.mi_progress_finish(self)
-        else:
-            progressbar.progressbar_finish(self)
+        if self._args.no_progress:
+            return
+
+        self._progress.finalize()
 
     def _run_analysis(self):
         self._pre_analysis()
         self._pb_setup()
 
         for event in self._traces.events:
-            self._pb_update()
+            self._pb_update(event)
             self._analysis.process_event(event)
             if self._analysis.ended:
                 break
@@ -434,6 +444,8 @@ class Command:
                         'CPU IDs')
         ap.add_argument('--timerange', type=str, help='time range: '
                                                       '[begin,end]')
+        ap.add_argument('--progress-use-size', action='store_true',
+                        help='use trace size to approximate progress')
         ap.add_argument('-V', '--version', action='version',
                         version='LTTng Analyses v' + __version__)
 
@@ -455,6 +467,13 @@ class Command:
         self._add_arguments(ap)
 
         args = ap.parse_args()
+
+        if self._mi_mode:
+            args.no_progress = True
+
+            if args.output_progress:
+                args.no_progress = False
+
         self._validate_transform_common_args(args)
         self._validate_transform_args(args)
         self._args = args
@@ -551,8 +570,8 @@ class Command:
                 # it always is in older versions of babeltrace. In
                 # that case, the test is simply skipped and an invalid
                 # --end value will cause an empty analysis
-                if self._traces.timestamp_begin is not None and \
-                   end_ts < self._traces.timestamp_begin:
+                if self._ts_begin is not None and \
+                   end_ts < self._ts_begin:
                     self._cmdline_error(
                         '--end timestamp before beginning of trace')
 

diff --git a/lttnganalyses/cli/progressbar.py b/lttnganalyses/cli/progressbar.py
index cd30d4e31d3b9cb634117ec961040e280453d893..b7938e9d5dcd4531cf075d3f7c092ee49e4e9c90 100644 (file)
--- a/lttnganalyses/cli/progressbar.py
+++ b/lttnganalyses/cli/progressbar.py
@@ -22,8 +22,11 @@
 
 import os
 import sys
+import time
 from . import mi
 from collections import namedtuple
+from ..common import format_utils
+
 
 try:
     from progressbar import ETA, Bar, Percentage, ProgressBar
@@ -31,8 +34,9 @@ try:
 except ImportError:
     progressbar_available = False
 
+
 # approximation for the progress bar
-BYTES_PER_EVENT = 30
+_BYTES_PER_EVENT = 30
 
 
 def get_folder_size(folder):
@@ -46,85 +50,106 @@ def get_folder_size(folder):
     return total_size
 
 
-def _get_maxval(obj):
-    size = get_folder_size(obj._args.path)
+class _Progress:
+    def __init__(self, ts_begin, ts_end, path, use_size=False):
+        if ts_begin is None or ts_end is None or use_size:
+            size = get_folder_size(path)
+            self._maxval = size / _BYTES_PER_EVENT
+            self._use_time = False
+        else:
+            self._maxval = ts_end - ts_begin
+            self._ts_begin = ts_begin
+            self._ts_end = ts_end
+            self._use_time = True
 
-    return size / BYTES_PER_EVENT
+        self._at = 0
+        self._event_count = 0
+        self._last_event_count_check = 0
+        self._last_time_check = time.time()
 
+    def update(self, event):
+        self._event_count += 1
 
-def progressbar_setup(obj):
-    if obj._args.no_progress:
-        obj.pbar = None
-        return
+        if self._use_time:
+            self._at = event.timestamp - self._ts_begin
+        else:
+            self._at = self._event_count
 
-    if progressbar_available:
-        widgets = ['Processing the trace: ', Percentage(), ' ',
-                   Bar(marker='#', left='[', right=']'),
-                   ' ', ETA(), ' ']  # see docs for other options
-        obj.pbar = ProgressBar(widgets=widgets,
-                               maxval=_get_maxval(obj))
-        obj.pbar.start()
-    else:
-        print('Warning: progressbar module not available, '
-              'using --no-progress.', file=sys.stderr)
-        obj._args.no_progress = True
-        obj.pbar = None
-    obj.event_count = 0
+        if self._at > self._maxval:
+            self._at = self._maxval
 
+        if self._event_count - self._last_event_count_check >= 101:
+            self._last_event_count_check = self._event_count
+            now = time.time()
 
-def progressbar_update(obj):
-    if obj._args.no_progress or obj.pbar is None:
-        return
+            if now - self._last_time_check >= .1:
+                self._update_progress()
+                self._last_time_check = now
 
-    try:
-        obj.pbar.update(obj.event_count)
-    except ValueError:
+    def _update_progress(self):
         pass
-    obj.event_count += 1
 
+    def finalize(self):
+        pass
 
-def progressbar_finish(obj):
-    if obj._args.no_progress:
-        return
-    obj.pbar.finish()
 
+class FancyProgressBar(_Progress):
+    def __init__(self, ts_begin, ts_end, path, use_size):
+        super().__init__(ts_begin, ts_end, path, use_size)
+        self._pbar = None
 
-class _MiProgress:
-    def __init__(self, maxval):
-        self._maxval = maxval
-        self._events = 0
-        self._step = maxval // 997
+        if progressbar_available:
+            widgets = ['Processing the trace: ', Percentage(), ' ',
+                       Bar(marker='#', left='[', right=']'),
+                       ' ', ETA(), ' ']  # see docs for other options
+            self._pbar = ProgressBar(widgets=widgets,
+                                   maxval=self._maxval)
+            self._pbar.start()
+        else:
+            print('Warning: progressbar module not available, '
+                  'using --no-progress.', file=sys.stderr)
 
-        if self._step == 0:
-            self._step = 1
+    def _update_progress(self):
+        if self._pbar is None:
+            return
 
-    def init(self):
-        msg = 'Starting analysis: {} estimated events'.format(round(self._maxval))
-        mi.print_progress(0, msg)
+        self._pbar.update(self._at)
 
-    def update(self):
-        if (self._events % self._step) == 0:
-            if self._events > self._maxval:
-                mi.print_progress(1, 'Almost done...')
-            else:
-                at = round(self._events / self._maxval, 4)
-                msg = '{} events processed'.format(self._events)
-                mi.print_progress(at, msg)
+    def finalize(self):
+        if self._pbar is None:
+            return
 
-        self._events += 1
+        self._pbar.finish()
 
-    def finish(self):
-        mi.print_progress(1, 'Done!')
 
+class MiProgress(_Progress):
+    def __init__(self, ts_begin, ts_end, path, use_size):
+        super().__init__(ts_begin, ts_end, path, use_size)
+
+        if self._use_time:
+            fmt = 'Starting analysis from {} to {}'
+            begin = format_utils.format_timestamp(self._ts_begin)
+            end = format_utils.format_timestamp(self._ts_end)
+            msg = fmt.format(begin, end)
+        else:
+            msg = 'Starting analysis: {} estimated events'.format(round(self._maxval))
 
-def mi_progress_setup(obj):
-    obj.pbar = _MiProgress(_get_maxval(obj))
-    obj.pbar.init()
+        mi.print_progress(0, msg)
 
+    def _update_progress(self):
+        if self._at == self._maxval:
+            mi.print_progress(1, 'Done!')
+            return
 
-def mi_progress_update(obj):
-    obj.pbar.update()
+        if self._use_time:
+            ts_at = self._at + self._ts_begin
+            at_ts = format_utils.format_timestamp(ts_at)
+            end = format_utils.format_timestamp(self._ts_end)
+            msg = '{}/{}; {} events processed'.format(at_ts, end, self._event_count)
+        else:
+            msg = '{} events processed'.format(self._event_count)
 
+        mi.print_progress(round(self._at / self._maxval, 4), msg)
 
-def mi_progress_finish(obj):
-    obj.pbar.finish()
+    def finalize(self):
+        mi.print_progress(1, 'Done!')