Initial benchmaks

Author: TomGillen

.gitignore

@@ -0,0 +1,2 @@
+/target
+Cargo.lock

Cargo.toml

@@ -0,0 +1,28 @@
+[package]
+name = "ecs_bench_suite"
+version = "0.1.0"
+authors = ["Thomas Gillen <[email protected]>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+cgmath = { version = "0.17", feature = ["serde"] }
+bincode = "1.3"
+ron = "0.6"
+serde = { version = "1.0", features = ["derive"] }
+rayon = "1.3"
+legion = { git = "https://github.com/TomGillen/legion" }
+#legion_typeuuid = { git = "https://github.com/TomGillen/legion_typeuuid" }
+legion_2_4 = { package = "legion", version = "0.2.4" }
+bevy_ecs = "0.1"
+hecs = "0.2"
+shipyard = "0.4"
+
+[dev-dependencies]
+criterion = "0.3"
+rayon = "1.3"
+
+[[bench]]
+name = "benchmarks"
+harness = false

benches/benchmarks.rs

@@ -0,0 +1,161 @@
+use criterion::*;
+use ecs_bench_suite::*;
+
+fn bench_simple_insert(c: &mut Criterion) {
+    let mut group = c.benchmark_group("simple_insert");
+    group.bench_function("legion", |b| {
+        let mut bench = legion::simple_insert::Benchmark::new();
+        b.iter(move || bench.run());
+    });
+    group.bench_function("legion_0.2.4", |b| {
+        let mut bench = legion_2_4::simple_insert::Benchmark::new();
+        b.iter(move || bench.run());
+    });
+    group.bench_function("bevy", |b| {
+        let mut bench = bevy::simple_insert::Benchmark::new();
+        b.iter(move || bench.run());
+    });
+    group.bench_function("shipyard", |b| {
+        let mut bench = shipyard::simple_insert::Benchmark::new();
+        b.iter(move || bench.run());
+    });
+}
+
+fn bench_simple_iter(c: &mut Criterion) {
+    let mut group = c.benchmark_group("simple_iter");
+    group.bench_function("legion", |b| {
+        let mut bench = legion::simple_iter::Benchmark::new();
+        b.iter(move || bench.run());
+    });
+    group.bench_function("legion_0.2.4", |b| {
+        let mut bench = legion_2_4::simple_iter::Benchmark::new();
+        b.iter(move || bench.run());
+    });
+    group.bench_function("bevy", |b| {
+        let mut bench = bevy::simple_iter::Benchmark::new();
+        b.iter(move || bench.run());
+    });
+    group.bench_function("shipyard", |b| {
+        let mut bench = shipyard::simple_iter::Benchmark::new();
+        b.iter(move || bench.run());
+    });
+}
+
+fn bench_frag_iter_bc(c: &mut Criterion) {
+    let mut group = c.benchmark_group("fragmented_iter");
+    group.bench_function("legion", |b| {
+        let mut bench = legion::frag_iter::Benchmark::new();
+        b.iter(move || bench.run());
+    });
+    group.bench_function("legion_0.2.4", |b| {
+        let mut bench = legion_2_4::frag_iter::Benchmark::new();
+        b.iter(move || bench.run());
+    });
+    group.bench_function("bevy", |b| {
+        let mut bench = bevy::frag_iter::Benchmark::new();
+        b.iter(move || bench.run());
+    });
+    group.bench_function("shipyard", |b| {
+        let mut bench = shipyard::frag_iter::Benchmark::new();
+        b.iter(move || bench.run());
+    });
+}
+
+fn bench_schedule(c: &mut Criterion) {
+    let mut group = c.benchmark_group("schedule");
+    group.bench_function("legion", |b| {
+        let mut bench = legion::schedule::Benchmark::new();
+        b.iter(move || bench.run());
+    });
+    group.bench_function("legion_0.2.4", |b| {
+        let mut bench = legion_2_4::schedule::Benchmark::new();
+        b.iter(move || bench.run());
+    });
+    group.bench_function("bevy", |b| {
+        let mut bench = bevy::schedule::Benchmark::new();
+        b.iter(move || bench.run());
+    });
+    group.bench_function("shipyard", |b| {
+        let mut bench = shipyard::schedule::Benchmark::new();
+        b.iter(move || bench.run());
+    });
+}
+
+fn bench_heavy_compute(c: &mut Criterion) {
+    let mut group = c.benchmark_group("heavy_compute");
+    group.bench_function("legion", |b| {
+        let mut bench = legion::heavy_compute::Benchmark::new();
+        b.iter(move || bench.run());
+    });
+    group.bench_function("legion_0.2.4", |b| {
+        let mut bench = legion_2_4::heavy_compute::Benchmark::new();
+        b.iter(move || bench.run());
+    });
+    group.bench_function("bevy", |b| {
+        let mut bench = bevy::heavy_compute::Benchmark::new();
+        b.iter(move || bench.run());
+    });
+    group.bench_function("shipyard", |b| {
+        let mut bench = shipyard::heavy_compute::Benchmark::new();
+        b.iter(move || bench.run());
+    });
+}
+
+fn bench_add_remove(c: &mut Criterion) {
+    let mut group = c.benchmark_group("add_remove_component");
+    group.bench_function("legion", |b| {
+        let mut bench = legion::add_remove::Benchmark::new();
+        b.iter(move || bench.run());
+    });
+    group.bench_function("legion_0.2.4", |b| {
+        let mut bench = legion_2_4::add_remove::Benchmark::new();
+        b.iter(move || bench.run());
+    });
+    group.bench_function("shipyard", |b| {
+        let mut bench = shipyard::add_remove::Benchmark::new();
+        b.iter(move || bench.run());
+    });
+
+    // todo Bevy appears to crash in this benchmark
+    // group.bench_function("bevy", |b| {
+    //     let mut bench = bevy::add_remove::Benchmark::new();
+    //     b.iter(move || bench.run());
+    // });
+}
+
+fn bench_serialize_text(c: &mut Criterion) {
+    let mut group = c.benchmark_group("serialize_text");
+    group.bench_function("legion", |b| {
+        let mut bench = legion::serialize_text::Benchmark::new();
+        b.iter(move || bench.run());
+    });
+    // group.bench_function("bevy", |b| {
+    //     let mut bench = bevy::serialize_text::Benchmark::new();
+    //     b.iter(move || bench.run());
+    // });
+}
+
+fn bench_serialize_binary(c: &mut Criterion) {
+    let mut group = c.benchmark_group("serialize_binary");
+    group.bench_function("legion", |b| {
+        let mut bench = legion::serialize_binary::Benchmark::new();
+        b.iter(move || bench.run());
+    });
+    // group.bench_function("bevy", |b| {
+    //     let mut bench = bevy::serialize_text::Benchmark::new();
+    //     b.iter(move || bench.run());
+    // });
+}
+
+criterion_group!(
+    benchmarks,
+    bench_simple_insert,
+    bench_simple_iter,
+    bench_frag_iter_bc,
+    bench_schedule,
+    bench_heavy_compute,
+    bench_add_remove,
+    bench_serialize_text,
+    bench_serialize_binary,
+);
+criterion_main!(benchmarks);

readme.md

@@ -0,0 +1,70 @@
+# ECS Bench Suite
+
+A suite of benchmarks designed to test and compare Rust ECS library performance across a variety of challenging circumstances.
+
+## The Benchmarks
+
+### Simple Insert
+
+This benchmark is designed to test the base cost of constructing entities and moving components into the ECS.
+
+Inserts 10,000 entities, each with 4 components: `Transform(mat4x4)`, `Position(vec3)`, `Rotation(vec3)` and `Velocity(vec3)`.
+
+### Simple Iter
+
+This benchmark is designed to test the core overheads involved in component iteration in best-case conditions. The iteration should occur on a single CPU core.
+
+Dataset: 10,000 entities, each with 4 components: `Transform(mat4x4)`, `Position(vec3)`, `Rotation(vec3)` and `Velocity(vec3)`.
+
+Test: Iterate through all entities with `Position` and `Velocity`, and add velocity onto position.
+
+### Fragmented Iter
+
+This benchmark is designed to test how the ECS handles iteration through a fragmented dataset. The iteration should occur on a single CPU core.
+
+Dataset: 26 component types (`A(f32)` through `Z(f32)`), each with 20 entities plus a `Data(f32)` component.
+
+Test: Iterate through all entities with a `Data` component and double its value.
+
+### System Scheduling
+
+This benchmark is designed to test how efficiently the ECS can schedule multiple independent systems on a multi-core CPU. Each system should execute on a single CPU core.
+
+Dataset:
+
+* 10,000 entities with `(A, B)` components.
+* 10,000 entities with `(A, B, C)` components.
+* 10,000 entities with `(A, B, C, D)` components.
+* 10,000 entities with `(A, B, C, E)` components.
+
+Test:
+
+Three systems accessing the following components mutably, where each system swaps the values stored in each component:
+
+* `(A, B)`
+* `(C, D)`
+* `(C, E)`
+
+### Heavy Compute
+
+This benchmark is designed to test the ECS's ability to scale when it is allowed to run a system over multiple CPU cores.
+
+Dataset: 10,000 entities with a `mat4x4` component.
+
+Test: Iterate through all `mat4x4` components, and invert the matrix 10 times.
+
+### Add/Remove Component
+
+This benchmark is designed to test how quickly the ECS can add and then remove a component from an existing entity.
+
+Dataset: 1,000 entities with a single `A` component.
+
+Test: Iterate through all entities, adding a `B` component. Then iterate through all entities again, removing their `B` component.
+
+### Serialize
+
+This benchmark is designed to test how quickly the ECS and serialize and deserialize its entities in both text (RON) and binary (bincode) formats.
+
+Dataset: 1000 entities with `Transform(mat4x4)`, `Position(vec3)`, `Rotation(vec3)` and `Velocity(vec3)` components.
+
+Test: Serialize all entities to RON and bincode formats in-memory. Then deserialize back into the ECS. The RON and bincode formats should be separate benchmark tests.

src/bevy/add_remove.rs

@@ -0,0 +1,28 @@
+use bevy_ecs::prelude::*;
+
+struct A(f32);
+struct B(f32);
+
+pub struct Benchmark(World, Vec<Entity>);
+
+impl Benchmark {
+    pub fn new() -> Self {
+        let mut world = World::default();
+
+        let entities = world
+            .spawn_batch((0..10000).map(|_| (A(0.0),)))
+            .collect::<Vec<_>>();
+
+        Self(world, entities)
+    }
+
+    pub fn run(&mut self) {
+        for entity in &self.1 {
+            self.0.insert_one(*entity, B(0.0)).unwrap();
+        }
+
+        for entity in &self.1 {
+            self.0.remove_one::<B>(*entity).unwrap();
+        }
+    }
+}

src/bevy/frag_iter.rs

@@ -0,0 +1,30 @@
+use bevy_ecs::prelude::*;
+
+macro_rules! create_entities {
+    ($world:ident; $( $variants:ident ),*) => {
+        $(
+            struct $variants(f32);
+            $world.spawn_batch((0..20).map(|_| ($variants(0.0), Data(1.0))));
+        )*
+    };
+}
+
+struct Data(f32);
+
+pub struct Benchmark(World);
+
+impl Benchmark {
+    pub fn new() -> Self {
+        let mut world = World::default();
+
+        create_entities!(world; A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V, W, X, Y, Z);
+
+        Self(world)
+    }
+
+    pub fn run(&mut self) {
+        for mut data in self.0.query::<&mut Data>().iter() {
+            data.0 *= 2.0;
+        }
+    }
+}

src/bevy/heavy_compute.rs

@@ -0,0 +1,40 @@
+use bevy_ecs::prelude::*;
+use cgmath::*;
+
+#[derive(Copy, Clone)]
+struct Position(Vector3<f32>);
+
+#[derive(Copy, Clone)]
+struct Rotation(Vector3<f32>);
+
+#[derive(Copy, Clone)]
+struct Velocity(Vector3<f32>);
+
+pub struct Benchmark(World);
+
+impl Benchmark {
+    pub fn new() -> Self {
+        let mut world = World::default();
+
+        world.spawn_batch((0..1000).map(|_| {
+            (
+                Matrix4::<f32>::from_angle_x(Rad(1.2)),
+                Position(Vector3::unit_x()),
+                Rotation(Vector3::unit_x()),
+                Velocity(Vector3::unit_x()),
+            )
+        }));
+
+        Self(world)
+    }
+
+    pub fn run(&mut self) {
+        for (mut pos, mut mat) in self.0.query::<(&mut Position, &mut Matrix4<f32>)>().iter() {
+            for _ in 0..100 {
+                *mat = mat.invert().unwrap();
+            }
+
+            pos.0 = mat.transform_vector(pos.0);
+        }
+    }
+}

src/bevy/mod.rs

@@ -0,0 +1,6 @@
+pub mod add_remove;
+pub mod frag_iter;
+pub mod heavy_compute;
+pub mod schedule;
+pub mod simple_insert;
+pub mod simple_iter;