block and net state providers

Signed-off-by: Julien Desfossez <jdesfossez@efficios.com>

diff --git a/iostats b/iostats
new file mode 100755 (executable)
index 0000000..a77db56
--- /dev/null
+++ b/iostats
@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+
+import sys
+
+sys.path += ['linuxautomaton', 'lttnganalyses', 'lttnganalysescli']
+
+import lttnganalysescli.io
+
+
+if __name__ == '__main__':
+    lttnganalysescli.io.run()

diff --git a/linuxautomaton/linuxautomaton/automaton.py b/linuxautomaton/linuxautomaton/automaton.py
index 8368aac2f0c325ad0475188e7b433ce6958b01b8..c4e6955061a1038c6795e459e5306a3b3e035dae 100644 (file)
--- a/linuxautomaton/linuxautomaton/automaton.py
+++ b/linuxautomaton/linuxautomaton/automaton.py
@@ -3,6 +3,8 @@ from .mem import MemStateProvider
 from .irq import IrqStateProvider
 from .syscalls import SyscallsStateProvider
 from .statedump import StatedumpStateProvider
+from .block import BlockStateProvider
+from .net import NetStateProvider
 
 
 class State:
@@ -27,6 +29,8 @@ class Automaton:
             IrqStateProvider(self._state),
             SyscallsStateProvider(self._state),
             StatedumpStateProvider(self._state),
+            BlockStateProvider(self._state),
+            NetStateProvider(self._state),
         ]
 
     def process_event(self, ev):

diff --git a/linuxautomaton/linuxautomaton/block.py b/linuxautomaton/linuxautomaton/block.py
new file mode 100644 (file)
index 0000000..739eb40
--- /dev/null
+++ b/linuxautomaton/linuxautomaton/block.py
@@ -0,0 +1,136 @@
+from linuxautomaton import sp, sv, common
+
+
+class BlockStateProvider(sp.StateProvider):
+    def __init__(self, state):
+        self.state = state
+        self.cpus = state.cpus
+        self.disks = state.disks
+        self.tids = state.tids
+        self.remap_requests = []
+        cbs = {
+            'block_rq_complete': self._process_block_rq_complete,
+            'block_rq_issue': self._process_block_rq_issue,
+            'block_bio_remap': self._process_block_bio_remap,
+            'block_bio_backmerge': self._process_block_bio_backmerge,
+        }
+        self._register_cbs(cbs)
+
+    def process_event(self, ev):
+        self._process_event_cb(ev)
+
+    def _process_block_bio_remap(self, event):
+        dev = event["dev"]
+        sector = event["sector"]
+        old_dev = event["old_dev"]
+        old_sector = event["old_sector"]
+
+        for req in self.remap_requests:
+            if req["dev"] == old_dev and req["sector"] == old_sector:
+                req["dev"] = dev
+                req["sector"] = sector
+                return
+
+        req = {}
+        req["orig_dev"] = old_dev
+        req["dev"] = dev
+        req["sector"] = sector
+        self.remap_requests.append(req)
+
+    # For backmerge requests, just remove the request from the
+    # remap_requests queue, because we rely later on the nr_sector
+    # which has all the info we need.
+    def _process_block_bio_backmerge(self, event):
+        dev = event["dev"]
+        sector = event["sector"]
+        for req in self.remap_requests:
+            if req["dev"] == dev and req["sector"] == sector:
+                self.remap_requests.remove(req)
+
+    def _process_block_rq_issue(self, event):
+        dev = event["dev"]
+        sector = event["sector"]
+        nr_sector = event["nr_sector"]
+        # Note: since we don't know, we assume a sector is 512 bytes
+        block_size = 512
+        if nr_sector == 0:
+            return
+
+        rq = {}
+        rq["nr_sector"] = nr_sector
+        rq["rq_time"] = event.timestamp
+        rq["iorequest"] = sv.IORequest()
+        rq["iorequest"].iotype = sv.IORequest.IO_BLOCK
+        rq["iorequest"].begin = event.timestamp
+        rq["iorequest"].size = nr_sector * block_size
+
+        d = None
+        for req in self.remap_requests:
+            if req["dev"] == dev and req["sector"] == sector:
+                d = common.get_disk(req["orig_dev"], self.disks)
+        if not d:
+            d = common.get_disk(dev, self.disks)
+
+        d.nr_requests += 1
+        d.nr_sector += nr_sector
+        d.pending_requests[sector] = rq
+
+        if "tid" in event.keys():
+            tid = event["tid"]
+            if tid not in self.tids:
+                p = sv.Process()
+                p.tid = tid
+                self.tids[tid] = p
+            else:
+                p = self.tids[tid]
+            if p.pid != -1 and p.tid != p.pid:
+                p = self.tids[p.pid]
+            rq["pid"] = p
+            # even rwbs means read, odd means write
+            if event["rwbs"] % 2 == 0:
+                p.block_read += nr_sector * block_size
+                rq["iorequest"].operation = sv.IORequest.OP_READ
+            else:
+                p.block_write += nr_sector * block_size
+                rq["iorequest"].operation = sv.IORequest.OP_WRITE
+
+    def _process_block_rq_complete(self, event):
+        dev = event["dev"]
+        sector = event["sector"]
+        nr_sector = event["nr_sector"]
+        if nr_sector == 0:
+            return
+
+        d = None
+        for req in self.remap_requests:
+            if req["dev"] == dev and req["sector"] == sector:
+                d = common.get_disk(req["orig_dev"], self.disks)
+                self.remap_requests.remove(req)
+
+        if not d:
+            d = common.get_disk(dev, self.disks)
+
+        # ignore the completion of requests we didn't see the issue
+        # because it would mess up the latency totals
+        if sector not in d.pending_requests.keys():
+            return
+
+        rq = d.pending_requests[sector]
+        if rq["nr_sector"] != nr_sector:
+            return
+        d.completed_requests += 1
+        if rq["rq_time"] > event.timestamp:
+            print("Weird request TS", event.timestamp)
+        time_per_sector = (event.timestamp - rq["rq_time"]) / rq["nr_sector"]
+        d.request_time += time_per_sector
+        rq["iorequest"].duration = time_per_sector
+        rq["iorequest"].end = event.timestamp
+        d.rq_list.append(rq["iorequest"])
+        if "pid" in rq.keys():
+            rq["pid"].iorequests.append(rq["iorequest"])
+        del d.pending_requests[sector]
+
+    def dump_orphan_requests(self):
+        for req in self.remap_requests:
+            print("Orphan : %d : %d %d" % (req["orig_dev"], req["dev"],
+                                           req["sector"]))

diff --git a/linuxautomaton/linuxautomaton/net.py b/linuxautomaton/linuxautomaton/net.py
new file mode 100644 (file)
index 0000000..146c6fc
--- /dev/null
+++ b/linuxautomaton/linuxautomaton/net.py
@@ -0,0 +1,55 @@
+from linuxautomaton import sp, sv
+
+
+class NetStateProvider(sp.StateProvider):
+    def __init__(self, state):
+        self.state = state
+        self.ifaces = state.ifaces
+        self.cpus = state.cpus
+        self.tids = state.tids
+        cbs = {
+            'net_dev_xmit': self._process_net_dev_xmit,
+            'netif_receive_skb': self._process_netif_receive_skb,
+        }
+        self._register_cbs(cbs)
+
+    def process_event(self, ev):
+        self._process_event_cb(ev)
+
+    def get_dev(self, dev):
+        if dev not in self.ifaces:
+            d = sv.Iface()
+            d.name = dev
+            self.ifaces[dev] = d
+        else:
+            d = self.ifaces[dev]
+        return d
+
+    def _process_net_dev_xmit(self, event):
+        dev = event["name"]
+        sent_len = event["len"]
+        cpu_id = event["cpu_id"]
+
+        d = self.get_dev(dev)
+        d.send_packets += 1
+        d.send_bytes += sent_len
+
+        if cpu_id not in self.cpus.keys():
+            return
+        c = self.cpus[cpu_id]
+        if c.current_tid == -1:
+            return
+        t = self.tids[c.current_tid]
+        if not t.current_syscall:
+            return
+        if t.current_syscall["name"] in sv.SyscallConsts.WRITE_SYSCALLS:
+            if t.current_syscall["fd"].fdtype == sv.FDType.unknown:
+                t.current_syscall["fd"].fdtype = sv.FDType.maybe_net
+
+    def _process_netif_receive_skb(self, event):
+        dev = event["name"]
+        recv_len = event["len"]
+
+        d = self.get_dev(dev)
+        d.recv_packets += 1
+        d.recv_bytes += recv_len

diff --git a/lttnganalysescli/lttnganalysescli/io.py b/lttnganalysescli/lttnganalysescli/io.py
index 8e5295745a6043d115afed1fcc6e2ffc09923e09..f48d18bd76a52581a3a1faa0f91cd1c0cc892cf3 100644 (file)
--- a/lttnganalysescli/lttnganalysescli/io.py
+++ b/lttnganalysescli/lttnganalysescli/io.py
@@ -304,7 +304,7 @@ class IoAnalysis(Command):
             if disk.completed_requests == 0:
                 continue
             total = (disk.request_time / disk.completed_requests) \
-                / sv.MSEC_PER_NSEC
+                / common.MSEC_PER_NSEC
             total = float("%0.03f" % total)
             values.append(("%s" % disk.prettyname, total))
         for line in graph.graph('Disk request time/sector', values, sort=2,