diff --git a/modules/jolt_physics/spaces/jolt_space_3d.cpp b/modules/jolt_physics/spaces/jolt_space_3d.cpp
index 69b73ea6f7..5eb28d3205 100644
--- a/modules/jolt_physics/spaces/jolt_space_3d.cpp
+++ b/modules/jolt_physics/spaces/jolt_space_3d.cpp
@@ -436,6 +436,12 @@ void JoltSpace3D::remove_object(const JPH::BodyID &p_jolt_id) {
 	}
 
 	body_iface.DestroyBody(p_jolt_id);
+
+	// If we're never going to step this space, like in the editor viewport, we need to manually clean up Jolt's broad phase instead, otherwise performance can degrade when doing things like switching scenes.
+	// We'll never actually have zero bodies in any space though, since we always have the default area, so we check if there's one or fewer left instead.
+	if (!JoltPhysicsServer3D::get_singleton()->is_active() && physics_system->GetNumBodies() <= 1) {
+		physics_system->OptimizeBroadPhase();
+	}
 }
 
 void JoltSpace3D::flush_pending_objects() {
diff --git a/thirdparty/jolt_physics/Jolt/Physics/Collision/BroadPhase/BroadPhaseQuadTree.cpp b/thirdparty/jolt_physics/Jolt/Physics/Collision/BroadPhase/BroadPhaseQuadTree.cpp
index bd26a2cf56..db427e94fc 100644
--- a/thirdparty/jolt_physics/Jolt/Physics/Collision/BroadPhase/BroadPhaseQuadTree.cpp
+++ b/thirdparty/jolt_physics/Jolt/Physics/Collision/BroadPhase/BroadPhaseQuadTree.cpp
@@ -73,6 +73,7 @@ void BroadPhaseQuadTree::Optimize()
 {
 	JPH_PROFILE_FUNCTION();
 
+	// Free the previous tree so we can create a new optimized tree
 	FrameSync();
 
 	LockModifications();
@@ -80,7 +81,7 @@ void BroadPhaseQuadTree::Optimize()
 	for (uint l = 0; l < mNumLayers; ++l)
 	{
 		QuadTree &tree = mLayers[l];
-		if (tree.HasBodies())
+		if (tree.HasBodies() || tree.IsDirty())
 		{
 			QuadTree::UpdateState update_state;
 			tree.UpdatePrepare(mBodyManager->GetBodies(), mTracking, update_state, true);
@@ -90,6 +91,9 @@ void BroadPhaseQuadTree::Optimize()
 
 	UnlockModifications();
 
+	// Free the tree from before we created a new optimized tree
+	FrameSync();
+
 	mNextLayerToUpdate = 0;
 }
 
diff --git a/thirdparty/jolt_physics/Jolt/Physics/Collision/BroadPhase/QuadTree.cpp b/thirdparty/jolt_physics/Jolt/Physics/Collision/BroadPhase/QuadTree.cpp
index 6966b31c52..0446027b9f 100644
--- a/thirdparty/jolt_physics/Jolt/Physics/Collision/BroadPhase/QuadTree.cpp
+++ b/thirdparty/jolt_physics/Jolt/Physics/Collision/BroadPhase/QuadTree.cpp
@@ -301,7 +301,7 @@ void QuadTree::UpdatePrepare(const BodyVector &inBodies, TrackingVector &ioTrack
 #endif
 
 	// Create space for all body ID's
-	NodeID *node_ids = new NodeID [mNumBodies];
+	NodeID *node_ids = mNumBodies > 0? new NodeID [mNumBodies] : nullptr;
 	NodeID *cur_node_id = node_ids;
 
 	// Collect all bodies