More wip des stuff

ewext/Cargo.lock

@@ -38,6 +38,12 @@ dependencies = [
  "windows-targets",
 ]
 
+[[package]]
+name = "bimap"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "230c5f1ca6a325a32553f8640d31ac9b49f2411e901e427570154868b46da4f7"
+
 [[package]]
 name = "bitcode"
 version = "0.6.3"
@@ -85,12 +91,14 @@ name = "ewext"
 version = "0.4.1"
 dependencies = [
  "backtrace",
+ "bimap",
  "eyre",
  "iced-x86",
  "libloading",
  "noita_api",
  "noita_api_macro",
  "rand",
+ "rustc-hash",
  "shared",
 ]
 
@@ -295,6 +303,12 @@ version = "0.1.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
 
+[[package]]
+name = "rustc-hash"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c7fb8039b3032c191086b10f11f319a6e99e1e82889c5cc6046f515c9db1d497"
+
 [[package]]
 name = "ryu"
 version = "1.0.18"

ewext/Cargo.toml

@@ -21,6 +21,8 @@ noita_api = {path = "noita_api"}
 shared = {path = "../shared"}
 libloading = "0.8.6"
 rand = "0.8.5"
+rustc-hash = "2.0.0"
+bimap = "0.6.3"
 
 [features]
 #enables cross-compilation on older systems (for example, when compiling on ubuntu 20.04)

ewext/noita_api/src/lib.rs

@@ -7,17 +7,17 @@ use eyre::{eyre, Context};
 
 pub mod lua;
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 pub struct EntityID(pub NonZero<isize>);
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 pub struct ComponentID(pub NonZero<isize>);
 
 pub struct Obj(pub usize);
 
 pub struct Color(pub u32);
 
-pub trait Component: From<ComponentID> {
+pub trait Component: From<ComponentID> + Into<ComponentID> {
     const NAME_STR: &'static str;
 }
 
@@ -31,6 +31,10 @@ impl EntityID {
         raw::entity_get_is_alive(self).unwrap_or(false)
     }
 
+    pub fn add_tag(self, tag: impl AsRef<str>) -> eyre::Result<()> {
+        raw::entity_add_tag(self, tag.as_ref().into())
+    }
+
     /// Returns true if entity has a tag.
     ///
     /// Corresponds to EntityGetTag from lua api.
@@ -38,8 +42,13 @@ impl EntityID {
         raw::entity_has_tag(self, tag.as_ref().into()).unwrap_or(false)
     }
 
-    pub fn max_in_use() -> eyre::Result<Self> {
+    pub fn kill(self) {
-        Ok(Self::try_from(raw::entities_get_max_id()? as isize)?)
+        // Shouldn't ever error.
+        let _ = raw::entity_kill(self);
+    }
+
+    pub fn set_position(self, x: f32, y: f32) -> eyre::Result<()> {
+        raw::entity_set_transform(self, x as f64, Some(y as f64), None, None, None)
     }
 
     /// Returns the first component of this type if an entity has it.
@@ -58,6 +67,26 @@ impl EntityID {
             .ok_or_else(|| eyre!("Entity {self:?} has no component {}", C::NAME_STR))
     }
 
+    pub fn remove_all_components_of_type<C: Component>(self) -> eyre::Result<()> {
+        while let Some(c) = self.try_get_first_component::<C>(None)? {
+            raw::entity_remove_component(self, c.into())?;
+        }
+        Ok(())
+    }
+
+    pub fn load(
+        filename: impl AsRef<str>,
+        pos_x: Option<f64>,
+        pos_y: Option<f64>,
+    ) -> eyre::Result<Self> {
+        raw::entity_load(filename.as_ref().into(), pos_x, pos_y)?
+            .ok_or_else(|| eyre!("Failed to spawn entity from filename {}", filename.as_ref()))
+    }
+
+    pub fn max_in_use() -> eyre::Result<Self> {
+        Ok(Self::try_from(raw::entities_get_max_id()? as isize)?)
+    }
+
     /// Returns id+1
     pub fn next(self) -> eyre::Result<Self> {
         Ok(Self(NonZero::try_from(isize::from(self.0) + 1)?))

ewext/noita_api_macro/src/lib.rs

@@ -197,6 +197,12 @@ fn generate_code_for_component(com: Component) -> proc_macro2::TokenStream {
             }
         }
 
+        impl From<#component_name> for ComponentID {
+            fn from(com: #component_name) -> Self {
+                com.0
+            }
+        }
+
         impl #component_name {
             #(#impls)*
         }

ewext/src/lib.rs

@@ -54,6 +54,15 @@ struct ExtState {
     modules: Modules,
 }
 
+impl ExtState {
+    fn with_global<T>(f: impl FnOnce(&mut Self) -> T) -> T {
+        STATE.with(|state| {
+            let mut state = state.borrow_mut();
+            f(&mut state)
+        })
+    }
+}
+
 fn init_particle_world_state(lua: LuaState) {
     println!("\nInitializing particle world state");
     let world_pointer = lua.to_integer(1);
@@ -150,18 +159,26 @@ fn netmanager_connect(_lua: LuaState) -> eyre::Result<Vec<RawString>> {
 fn netmanager_recv(_lua: LuaState) -> eyre::Result<Option<RawString>> {
     let mut binding = NETMANAGER.lock().unwrap();
     let netmanager = binding.as_mut().unwrap();
-    Ok(match netmanager.try_recv()? {
+    while let Some(msg) = netmanager.try_recv()? {
-        Some(NoitaInbound::RawMessage(msg)) => Some(msg.into()),
+        match msg {
-        Some(NoitaInbound::Ready) => {
+            NoitaInbound::RawMessage(vec) => return Ok(Some(vec.into())),
-            bail!("Unexpected Ready message")
+            NoitaInbound::Ready => bail!("Unexpected Ready message"),
+            NoitaInbound::ProxyToDes(proxy_to_des) => ExtState::with_global(|state| {
+                if let Some(entity_sync) = &mut state.modules.entity_sync {
+                    entity_sync.handle_proxytodes(proxy_to_des);
+                }
+            }),
+            NoitaInbound::RemoteMessage {
+                source,
+                message: shared::RemoteMessage::RemoteDes(remote_des),
+            } => ExtState::with_global(|state| {
+                if let Some(entity_sync) = &mut state.modules.entity_sync {
+                    entity_sync.handle_remotedes(source, remote_des);
+                }
+            }),
         }
-        Some(NoitaInbound::ProxyToDes(proxy_to_des)) => todo!(),
+    }
-        Some(NoitaInbound::RemoteMessage {
+    Ok(None)
-            source,
-            message: shared::RemoteMessage::RemoteDes(remote_des),
-        }) => todo!(),
-        None => None,
-    })
 }
 
 fn netmanager_send(lua: LuaState) -> eyre::Result<()> {

ewext/src/modules/entity_sync.rs

@@ -1,11 +1,25 @@
+//! Distibuted Entity Sync, a.k.a. DES.
+//! The idea is that we completely disregard the normal saving system for entities we sync.
+//! Also, each entity gets an owner.
+//! Each peer broadcasts an "Interest" zone. If it intersects any peer they receive all information about entities this peer owns.
+
+use diff_model::{LocalDiffModel, RemoteDiffModel};
 use eyre::Context;
+use interest::InterestTracker;
 use noita_api::EntityID;
+use rustc_hash::FxHashMap;
+use shared::{
+    des::{Gid, InterestRequest, RemoteDes},
+    Destination, NoitaOutbound, PeerId, RemoteMessage, WorldPos,
+};
 
 use super::Module;
 
+mod diff_model;
+mod interest;
+
 struct TrackedEntity {
-    /// 64 bit globally unique id. Assigned randomly, should only have 50% chance of collision with 2^32 entities at once.
+    gid: Gid,
-    gid: u64,
     entity: EntityID,
 }
 
@@ -14,6 +28,10 @@ pub(crate) struct EntitySync {
     look_current_entity: EntityID,
     /// List of entities that we have authority over.
     tracked: Vec<TrackedEntity>,
+
+    interest_tracker: InterestTracker,
+    local_diff_model: LocalDiffModel,
+    remote_models: FxHashMap<PeerId, RemoteDiffModel>,
 }
 
 impl Default for EntitySync {
@@ -21,6 +39,10 @@ impl Default for EntitySync {
         Self {
             look_current_entity: EntityID::try_from(1).unwrap(),
             tracked: Vec::new(),
+
+            interest_tracker: InterestTracker::new(512.0),
+            local_diff_model: LocalDiffModel::default(),
+            remote_models: Default::default(),
         }
     }
 }
@@ -50,13 +72,92 @@ impl EntitySync {
         self.look_current_entity = max_entity;
         Ok(())
     }
+
+    pub(crate) fn handle_proxytodes(&mut self, proxy_to_des: shared::des::ProxyToDes) {
+        todo!()
+    }
+
+    pub(crate) fn handle_remotedes(&mut self, source: PeerId, remote_des: RemoteDes) {
+        match remote_des {
+            RemoteDes::InterestRequest(interest_request) => self
+                .interest_tracker
+                .handle_interest_request(source, interest_request),
+            RemoteDes::EntityUpdate(vec) => {
+                self.remote_models
+                    .entry(source)
+                    .or_default()
+                    .apply_diff(&vec);
+            }
+            RemoteDes::ExitedInterest => {
+                self.remote_models.remove(&source);
+            }
+        }
+    }
 }
 
 impl Module for EntitySync {
-    fn on_world_update(&mut self, _ctx: &mut super::ModuleCtx) -> eyre::Result<()> {
+    fn on_world_update(&mut self, ctx: &mut super::ModuleCtx) -> eyre::Result<()> {
         self.look_for_tracked()
             .wrap_err("Error in look_for_tracked")?;
 
+        let (x, y) = noita_api::raw::game_get_camera_pos()?;
+        self.interest_tracker.set_center(x, y);
+        let frame_num = noita_api::raw::game_get_frame_num()?;
+        if frame_num % 20 == 0 {
+            send_remotedes(
+                ctx,
+                false,
+                Destination::Broadcast,
+                RemoteDes::InterestRequest(InterestRequest {
+                    pos: WorldPos::from_f64(x, y),
+                    radius: 1024,
+                }),
+            )?;
+        }
+
+        for lost in self.interest_tracker.drain_lost_interest() {
+            send_remotedes(
+                ctx,
+                true,
+                Destination::Peer(lost),
+                RemoteDes::ExitedInterest,
+            )?;
+        }
+
+        if frame_num % 2 == 0 {
+            if self.interest_tracker.got_any_new_interested() {
+                self.local_diff_model.reset_diff_encoding();
+            }
+            let diff = self.local_diff_model.make_diff();
+            // FIXME (perf): allow a Destination that can send to several peers at once, to prevent cloning and stuff.
+            for peer in self.interest_tracker.iter_interested() {
+                send_remotedes(
+                    ctx,
+                    true,
+                    Destination::Peer(peer),
+                    RemoteDes::EntityUpdate(diff.clone()),
+                )?;
+            }
+        } else {
+            for remote_model in self.remote_models.values_mut() {
+                remote_model.apply_entities()?;
+            }
+        }
+
         Ok(())
     }
 }
+
+fn send_remotedes(
+    ctx: &mut super::ModuleCtx<'_>,
+    reliable: bool,
+    destination: Destination<PeerId>,
+    remote_des: RemoteDes,
+) -> Result<(), eyre::Error> {
+    ctx.net.send(&NoitaOutbound::RemoteMessage {
+        reliable,
+        destination,
+        message: RemoteMessage::RemoteDes(remote_des),
+    })?;
+    Ok(())
+}

ewext/src/modules/entity_sync/diff_model.rs

@@ -0,0 +1,113 @@
+use bimap::BiHashMap;
+use noita_api::{
+    AIAttackComponent, AdvancedFishAIComponent, AnimalAIComponent, CameraBoundComponent, EntityID,
+    PhysicsAIComponent,
+};
+use rustc_hash::FxHashMap;
+use shared::des::{EntityData, EntityUpdate, Gid};
+
+struct EntityEntry {
+    entity_data: EntityData,
+    x: f32,
+    y: f32,
+}
+
+#[derive(Default)]
+pub(crate) struct LocalDiffModel {}
+
+#[derive(Default)]
+pub(crate) struct RemoteDiffModel {
+    tracked: BiHashMap<Gid, EntityID>,
+    entity_entries: FxHashMap<Gid, EntityEntry>,
+}
+
+impl EntityEntry {
+    fn new(entity_data: EntityData) -> Self {
+        Self {
+            entity_data,
+            x: 0.0,
+            y: 0.0,
+        }
+    }
+}
+
+impl LocalDiffModel {
+    pub(crate) fn reset_diff_encoding(&mut self) {
+        todo!();
+    }
+
+    pub(crate) fn make_diff(&mut self) -> Vec<EntityUpdate> {
+        todo!()
+    }
+}
+
+impl RemoteDiffModel {
+    pub(crate) fn apply_diff(&mut self, diff: &[EntityUpdate]) {
+        let mut current_gid = 0;
+        for entry in diff {
+            match entry {
+                EntityUpdate::CurrentEntity(gid) => current_gid = *gid,
+                EntityUpdate::EntityData(entity_data) => {
+                    self.entity_entries
+                        .insert(current_gid, EntityEntry::new(entity_data.clone()));
+                }
+                EntityUpdate::SetPosition(x, y) => {
+                    let Some(ent_data) = self.entity_entries.get_mut(&current_gid) else {
+                        continue;
+                    };
+                    ent_data.x = *x;
+                    ent_data.y = *y;
+                }
+                EntityUpdate::RemoveEntity(gid) => {
+                    if let Some((_, entity)) = self.tracked.remove_by_left(gid) {
+                        entity.kill();
+                    }
+                    self.entity_entries.remove(&gid);
+                }
+            }
+        }
+    }
+
+    pub(crate) fn apply_entities(&mut self) -> eyre::Result<()> {
+        for (gid, entity_entry) in &self.entity_entries {
+            match self.tracked.get_by_left(gid) {
+                Some(entity) => {
+                    entity.set_position(entity_entry.x, entity_entry.y)?;
+                }
+                None => {
+                    let entity = self.spawn_entity_by_data(&entity_entry.entity_data)?;
+                    self.remove_unnecessary_components(entity)?;
+                    self.tracked.insert(*gid, entity);
+                }
+            }
+        }
+
+        Ok(())
+    }
+
+    fn spawn_entity_by_data(&self, entity_data: &EntityData) -> eyre::Result<EntityID> {
+        match entity_data {
+            EntityData::Filename(filename) => EntityID::load(filename, None, None),
+        }
+    }
+
+    /// Removes components that shouldn't be on entities that were replicated from a remote,
+    /// generally because they interfere with things we're supposed to sync.
+    fn remove_unnecessary_components(&self, entity: EntityID) -> eyre::Result<()> {
+        entity.remove_all_components_of_type::<CameraBoundComponent>()?;
+        entity.remove_all_components_of_type::<AnimalAIComponent>()?;
+        entity.remove_all_components_of_type::<PhysicsAIComponent>()?;
+        entity.remove_all_components_of_type::<AdvancedFishAIComponent>()?;
+        entity.remove_all_components_of_type::<AIAttackComponent>()?;
+        Ok(())
+    }
+}
+
+impl Drop for RemoteDiffModel {
+    fn drop(&mut self) {
+        // Cleanup all entities tracked by this model.
+        for ent in self.tracked.right_values() {
+            ent.kill();
+        }
+    }
+}

ewext/src/modules/entity_sync/interest.rs

@@ -0,0 +1,62 @@
+use rustc_hash::FxHashSet;
+use shared::{des::InterestRequest, PeerId};
+
+pub(crate) struct InterestTracker {
+    radius_hysteresis: f64,
+    x: f64,
+    y: f64,
+    interested_peers: FxHashSet<PeerId>,
+    added_any: bool,
+    lost_interest: Vec<PeerId>,
+}
+
+impl InterestTracker {
+    pub(crate) fn new(radius_hysteresis: f64) -> Self {
+        assert!(radius_hysteresis > 0.0);
+        Self {
+            radius_hysteresis,
+            x: 0.0,
+            y: 0.0,
+            interested_peers: Default::default(),
+            lost_interest: Vec::with_capacity(4),
+            added_any: false,
+        }
+    }
+
+    pub(crate) fn set_center(&mut self, x: f64, y: f64) {
+        self.x = x;
+        self.y = y;
+    }
+
+    pub(crate) fn handle_interest_request(&mut self, peer: PeerId, request: InterestRequest) {
+        let rx = request.pos.x as f64;
+        let ry = request.pos.y as f64;
+
+        let dist_sq = (rx - self.x).powi(2) + (ry - self.y).powi(2);
+        if dist_sq < (request.radius as f64).powi(2) {
+            if self.interested_peers.insert(peer) {
+                self.added_any = true;
+            }
+        }
+
+        if dist_sq > ((request.radius as f64) + self.radius_hysteresis).powi(2) {
+            if self.interested_peers.remove(&peer) {
+                self.lost_interest.push(peer);
+            }
+        }
+    }
+
+    pub(crate) fn got_any_new_interested(&mut self) -> bool {
+        let ret = self.added_any;
+        self.added_any = false;
+        ret
+    }
+
+    pub(crate) fn drain_lost_interest(&mut self) -> impl Iterator<Item = PeerId> + '_ {
+        self.lost_interest.drain(..)
+    }
+
+    pub(crate) fn iter_interested(&mut self) -> impl Iterator<Item = PeerId> + '_ {
+        self.interested_peers.iter().copied()
+    }
+}

shared/src/basic_types.rs

@@ -6,7 +6,16 @@ pub struct WorldPos {
     pub y: i32,
 }
 
-#[derive(Encode, Decode)]
+impl WorldPos {
+    pub fn from_f64(x: f64, y: f64) -> Self {
+        Self {
+            x: x as i32,
+            y: y as i32,
+        }
+    }
+}
+
+#[derive(Encode, Decode, Hash, PartialEq, Eq, Clone, Copy)]
 pub struct PeerId(pub u64);
 
 #[derive(Encode, Decode, Debug, PartialEq, Eq)]

shared/src/des.rs

@@ -4,9 +4,13 @@ use bitcode::{Decode, Encode};
 
 use crate::WorldPos;
 
-#[derive(Encode, Decode)]
+/// 64 bit globally unique id. Assigned randomly, should only have 50% chance of collision with 2^32 entities at once.
+pub type Gid = u64;
+
+#[derive(Encode, Decode, Clone)]
 pub enum EntityData {
-    Serialized(Vec<u8>),
+    Filename(String),
+    // Serialized(Vec<u8>),
 }
 
 #[derive(Encode, Decode)]
@@ -45,15 +49,17 @@ pub struct InterestRequest {
 
 #[derive(Encode, Decode, Clone)]
 pub enum EntityUpdate {
-    CurrentEntity(NonZero<i32>),
+    /// Sets the gid that following EntityUpdates will act on.
-    SetPosition(WorldPos),
+    CurrentEntity(Gid),
+    EntityData(EntityData),
+    SetPosition(f32, f32),
     // TODO...
+    RemoveEntity(Gid),
 }
 
 #[derive(Encode, Decode, Clone)]
 pub enum RemoteDes {
     InterestRequest(InterestRequest),
-    EnteredInterest,
-    ExitedInterest,
     EntityUpdate(Vec<EntityUpdate>),
+    ExitedInterest,
 }