evlib 0.8.6

Event Camera Data Processing Library
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211

#!/usr/bin/env python3
"""
Performance comparison tool showing the improvements from the memory optimization.

This script demonstrates the concrete benefits of the new implementation.
"""

import evlib
import time
import psutil
import os
import gc
from pathlib import Path


def get_memory_usage():
    """Get current memory usage in MB"""
    process = psutil.Process(os.getpid())
    return process.memory_info().rss / 1024 / 1024


def demonstrate_improvements():
    """Demonstrate the key improvements in the new implementation"""

    print("TARGET: EVLIB MEMORY OPTIMIZATION IMPROVEMENTS")
    print("=" * 60)

    # Test file
    test_file = "data/slider_depth/events.txt"
    if not Path(test_file).exists():
        print(f"FAIL: Test file not found: {test_file}")
        return

    print(f"FILE: Test file: {test_file}")

    # === IMPROVEMENT 1: Memory Efficiency ===
    print("\nIMPROVEMENT 1: MEMORY EFFICIENCY")
    print("-" * 40)

    gc.collect()
    start_mem = get_memory_usage()

    # Load data
    lf = evlib.load_events(test_file)
    df = lf.collect()

    peak_mem = get_memory_usage()
    memory_used = peak_mem - start_mem
    bytes_per_event = (memory_used * 1024 * 1024) / len(df)

    print(f"STATS: Events loaded: {len(df):,}")
    print(f"MEMORY: Memory used: {memory_used:.1f} MB")
    print(f"TREND: Efficiency: {bytes_per_event:.1f} bytes/event")

    # Theoretical comparison
    print("\nSTATS: MEMORY EFFICIENCY COMPARISON:")
    old_estimate = len(df) * 37  # Old: ~37 bytes/event
    new_actual = memory_used * 1024 * 1024  # New: actual usage
    improvement = (old_estimate - new_actual) / old_estimate * 100

    print(f"   OLD (estimated): {old_estimate / 1024 / 1024:.1f} MB (~37 bytes/event)")
    print(
        f"   TREND: NEW (measured):  {new_actual / 1024 / 1024:.1f} MB ({bytes_per_event:.1f} bytes/event)"
    )
    print(f"   PASS: IMPROVEMENT:     {improvement:.1f}% memory reduction")

    # === IMPROVEMENT 2: Processing Speed ===
    print("\nFAST: IMPROVEMENT 2: PROCESSING SPEED")
    print("-" * 40)

    # Test loading speed
    start_time = time.time()
    lf2 = evlib.load_events(test_file)
    df2 = lf2.collect()
    load_time = time.time() - start_time

    events_per_second = len(df2) / load_time
    print(f"TIMING: Load time: {load_time:.2f}s")
    print(f"PERFORMANCE: Speed: {events_per_second:,.0f} events/s")

    # Test filtering speed (LazyFrame optimization)
    start_time = time.time()
    import polars as pl

    filtered = lf2.filter(pl.col("polarity") == 1).collect()
    filter_time = time.time() - start_time

    filter_speed = len(df2) / filter_time
    print(f"ANALYSIS: Filter speed: {filter_speed:,.0f} events/s ({filter_time:.3f}s)")

    # === IMPROVEMENT 3: Data Type Optimization ===
    print("\nLABEL: IMPROVEMENT 3: DATA TYPE OPTIMIZATION")
    print("-" * 40)

    print("Optimized data types:")
    for col in df.columns:
        dtype = str(df[col].dtype)
        print(f"   • {col}: {dtype}")

    # Calculate type efficiency
    # type_sizes = {"Int64": 8, "Int32": 4, "Int16": 2, "Int8": 1, "Duration": 8, "Float64": 8, "Float32": 4}

    total_bytes_per_event = 0
    for col in df.columns:
        dtype = str(df[col].dtype)
        if "Int64" in dtype:
            total_bytes_per_event += 8
        elif "Int32" in dtype:
            total_bytes_per_event += 4
        elif "Int16" in dtype:
            total_bytes_per_event += 2
        elif "Int8" in dtype:
            total_bytes_per_event += 1
        elif "Duration" in dtype:
            total_bytes_per_event += 8
        else:
            total_bytes_per_event += 8  # Conservative estimate

    print(f"Core data size: {total_bytes_per_event} bytes/event (theoretical minimum)")
    print(
        f"MEMORY: Actual memory: {bytes_per_event:.1f} bytes/event (includes overhead)"
    )
    overhead = bytes_per_event - total_bytes_per_event
    print(
        f"CONFIG: Memory overhead: {overhead:.1f} bytes/event ({overhead / bytes_per_event * 100:.1f}%)"
    )

    # === IMPROVEMENT 4: Architecture Benefits ===
    print("\nBUILD: IMPROVEMENT 4: ARCHITECTURE BENEFITS")
    print("-" * 40)

    print("PASS: BEFORE (Old Architecture):")
    print("   Events → 4x Vec<T> → Python Dict → Polars DataFrame")
    print("   • Multiple memory allocations")
    print("   • Data copying at each step")
    print("   • Python object overhead")

    print("\nPASS: AFTER (New Architecture):")
    print("   Events → Direct Polars Series → DataFrame")
    print("   • Single allocation per column")
    print("   • Zero intermediate copies")
    print("   • Native Arrow memory layout")

    # === IMPROVEMENT 5: Format-Specific Optimizations ===
    print("\nTARGET: IMPROVEMENT 5: FORMAT-SPECIFIC OPTIMIZATIONS")
    print("-" * 40)

    # Test different formats
    formats_to_test = [
        ("data/slider_depth/events.txt", "Text", [0, 1]),
        ("data/eTram/h5/val_2/val_night_011_td.h5", "HDF5", [0, 1]),
        ("data/eTram/raw/val_2/val_night_011.raw", "EVT2", [-1, 1]),
    ]

    for file_path, format_name, expected_polarities in formats_to_test:
        if Path(file_path).exists():
            lf_test = evlib.load_events(file_path)
            df_test = lf_test.collect()
            polarities = sorted(df_test["polarity"].unique().to_list())

            status = "PASS:" if polarities == expected_polarities else "FAIL:"
            print(
                f"   {status} {format_name}: {polarities} (expected {expected_polarities})"
            )

            del lf_test, df_test
        else:
            print(f"   PAUSE: {format_name}: File not available for testing")

    # === FINAL SUMMARY ===
    print("\nOPTIMIZATION SUMMARY")
    print("=" * 60)

    print("STATS: Performance Metrics:")
    print(f"   • Load Speed: {events_per_second:,.0f} events/s")
    print(f"   • Filter Speed: {filter_speed:,.0f} events/s")
    print(f"   • Memory Efficiency: {bytes_per_event:.1f} bytes/event")

    print("\nTARGET: Key Achievements:")
    print("   PASS: Zero-copy memory architecture")
    print("   PASS: Direct Polars Series construction")
    print("   PASS: Format-specific optimizations")
    print("   PASS: Maintained API compatibility")
    print("   PASS: Enhanced type efficiency")

    # Performance classification
    if events_per_second > 1_000_000:
        speed_rating = "PERFORMANCE: EXCELLENT"
    elif events_per_second > 500_000:
        speed_rating = "PASS: VERY GOOD"
    else:
        speed_rating = "WARNING: ADEQUATE"

    if bytes_per_event < 50:
        memory_rating = "PERFORMANCE: EXCELLENT"
    elif bytes_per_event < 100:
        memory_rating = "PASS: VERY GOOD"
    else:
        memory_rating = "WARNING: ADEQUATE"

    print("\nOverall Rating:")
    print(f"   • Speed: {speed_rating}")
    print(f"   • Memory: {memory_rating}")

    # Cleanup
    del lf, df, lf2, df2, filtered
    gc.collect()


if __name__ == "__main__":
    demonstrate_improvements()