import json
import re
import sys


events = []

with open(sys.argv[1]) as raw:
    for line in raw.readlines():
        events.append(json.loads(line))


old_pypy_events = []
pypy_events = []
cpython_events = []

for e in events:
    name = e["Resource"]["k8s"]["deployment"]["name"]
    podname = e["Resource"]["k8s"]["pod"]["name"]
    if name == "loader-git-pypy":
        if podname.endswith("8v7"):
            old_pypy_events.append(e)
        else:
            pypy_events.append(e)
    else:
        cpython_events.append(e)


class Task:

    def __init__(self):
        self.events = []
        self.runtime = None
        self.objects = 0
        self.new = 0

    def add(self, event):
        self.events.append(event)

AFTER_RE = re.compile(r"After (\S+): processed (\d+) objects, (\d+) are new")

def combine_events(events):
    current_tasks = None
    tasks = []
    for e in events:
        func_name = e["Attributes"].get("funcName")
        if func_name == "task_message_handler":
            tasks.append(current_tasks := Task())

        if current_tasks is not None:
            current_tasks.add(e)

            if func_name == "maybe_log":
                msg = e["Attributes"]["message"]
                after = AFTER_RE.fullmatch(msg)
                if after is not None:
                    current_tasks.objects += int(after.group(2))
                    current_tasks.new += int(after.group(3))

        runtime = e["Attributes"].get("data", {}).get("runtime")
        if runtime is not None and current_tasks is not None:
            current_tasks.runtime = runtime
            current_tasks = None
    return tasks


for some_events in (cpython_events, old_pypy_events, pypy_events):
    print("=====")
    print('  events:', len(some_events))
    tasks = combine_events(some_events)
    tasks = [t for t in tasks if t.runtime is not None]
    tasks.sort(key=lambda x: x.runtime)
    start = 0.0
    end = 1.0
    tasks = tasks[int(len(tasks) * start):int(len(tasks) * end)]
    print('  tasks:', len(tasks))
    total_runtime = sum(t.runtime for t in tasks)
    total_object = sum(t.objects for t in tasks)
    total_new = sum(t.new for t in tasks)
    print('  total-runtime:', total_runtime)
    print('  average-runtime:', total_runtime / len(tasks))
    print('  total-object:', total_object)
    print('  total-new:', total_new)
    print('  object-rate:', total_object / total_runtime, "/s")
    print('  new-rate:', total_new / total_runtime, "/s")

``` import json import re import sys events = [] with open(sys.argv[1]) as raw: for line in raw.readlines(): events.append(json.loads(line)) old_pypy_events = [] pypy_events = [] cpython_events = [] for e in events: name = e["Resource"]["k8s"]["deployment"]["name"] podname = e["Resource"]["k8s"]["pod"]["name"] if name == "loader-git-pypy": if podname.endswith("8v7"): old_pypy_events.append(e) else: pypy_events.append(e) else: cpython_events.append(e) class Task: def __init__(self): self.events = [] self.runtime = None self.objects = 0 self.new = 0 def add(self, event): self.events.append(event) AFTER_RE = re.compile(r"After (\S+): processed (\d+) objects, (\d+) are new") def combine_events(events): current_tasks = None tasks = [] for e in events: func_name = e["Attributes"].get("funcName") if func_name == "task_message_handler": tasks.append(current_tasks := Task()) if current_tasks is not None: current_tasks.add(e) if func_name == "maybe_log": msg = e["Attributes"]["message"] after = AFTER_RE.fullmatch(msg) if after is not None: current_tasks.objects += int(after.group(2)) current_tasks.new += int(after.group(3)) runtime = e["Attributes"].get("data", {}).get("runtime") if runtime is not None and current_tasks is not None: current_tasks.runtime = runtime current_tasks = None return tasks for some_events in (cpython_events, old_pypy_events, pypy_events): print("=====") print(' events:', len(some_events)) tasks = combine_events(some_events) tasks = [t for t in tasks if t.runtime is not None] tasks.sort(key=lambda x: x.runtime) start = 0.0 end = 1.0 tasks = tasks[int(len(tasks) * start):int(len(tasks) * end)] print(' tasks:', len(tasks)) total_runtime = sum(t.runtime for t in tasks) total_object = sum(t.objects for t in tasks) total_new = sum(t.new for t in tasks) print(' total-runtime:', total_runtime) print(' average-runtime:', total_runtime / len(tasks)) print(' total-object:', total_object) print(' total-new:', total_new) print(' object-rate:', total_object / total_runtime, "/s") print(' new-rate:', total_new / total_runtime, "/s") ```