Compare TensorRT Engines

Use this Jupyter worksheet to compare two or more TensorRT Engine plans.

Load JSON Files

python

import IPython
from ipywidgets import widgets
from trex import *
from trex.notebook import *
from trex.report_card import *
from trex.compare_engines import *

# Configure a wider output (for the wide graphs)
set_wide_display()

python

engine_name_1 = "../tests/inputs/mobilenet.qat.onnx.engine"
engine_name_2 = "../tests/inputs/mobilenet_v2_residuals.qat.onnx.engine"

def extract_engine_name(engine_path):
    from pathlib import Path 
    return Path(engine_path).name

def make_plan(engine_path, engine_name=None):
    plan = EnginePlan(f'{engine_path}.graph.json', f'{engine_path}.profile.json', f"{engine_path}.profile.metadata.json", name=engine_name)
    return plan
    
plan1 = make_plan(engine_name_1, "mobilenet.qat")
plan2 = make_plan(engine_name_2, "mobilenet_v2_residuals.qat")
plans = (plan1, plan2)

Summary

It is helpful to look at a high-level summary of the engine plan before diving into the details.

python

compare_engines_overview(plans)

python

compare_engines_summaries_tbl(plans, orientation='vertical')

python

compare_engines_layer_latencies(
    plan1, plan2,
    # Allow for 3% error grace threshold when color highlighting performance differences
    threshold=0.03,
    # Inexact matching uses only the layer's first input and output to match to other layers.
    exact_matching=True)

python

compare_engines_layer_details(plans[0], plans[1])

python

print(plan1.name)
report_card_perf_overview_widget(plan1)

python

print(plan2.name)
report_card_perf_overview_widget(plan2)

python

print(plan1.name)
report_card_table_view(plan1)

python

print(plan2.name)
report_card_table_view(plan2)

python

for plan in plans:
    graph = graphing.to_dot(plan, graphing.layer_type_formatter, display_regions=True, expand_layer_details=True)
    graphing.render_dot(graph, plan.name, 'svg')