Implement serialization.hpp, remove cereal submodule

Bump RSX capture version, use new serializer.

.gitmodules

@@ -23,10 +23,6 @@
 	path = 3rdparty/SPIRV/SPIRV-Headers
 	url = ../../KhronosGroup/SPIRV-Headers.git
 	ignore = dirty
-[submodule "3rdparty/cereal"]
-	path = 3rdparty/cereal
-	url = ../../RPCS3/cereal.git
-	ignore = dirty
 [submodule "3rdparty/zlib"]
 	path = 3rdparty/zlib/zlib
 	url = ../../madler/zlib

3rdparty/CMakeLists.txt

@@ -103,12 +103,6 @@ else()
 	target_include_directories(xxhash INTERFACE xxHash)
 endif()
 
-
-# cereal
-add_library(3rdparty_cereal INTERFACE)
-target_include_directories(3rdparty_cereal INTERFACE cereal/include)
-
-
 # OpenGL
 
 # Prefer GLVND for OpenGL rather than legacy, unless it's been defined elsewhere, in the case of AppImage builds
@@ -351,7 +345,6 @@ add_library(3rdparty::yaml-cpp ALIAS yaml-cpp)
 add_library(3rdparty::xxhash ALIAS xxhash)
 add_library(3rdparty::hidapi ALIAS 3rdparty_hidapi)
 add_library(3rdparty::libpng ALIAS ${LIBPNG_TARGET})
-add_library(3rdparty::cereal ALIAS 3rdparty_cereal)
 add_library(3rdparty::opengl ALIAS 3rdparty_opengl)
 add_library(3rdparty::stblib ALIAS 3rdparty_stblib)
 add_library(3rdparty::discordRPC ALIAS 3rdparty_discordRPC)

buildfiles/msvc/rpcs3_default.props

@@ -3,7 +3,7 @@
   <ImportGroup Label="PropertySheets" />
   <PropertyGroup Label="UserMacros" />
   <PropertyGroup>
-    <IncludePath>.\;..\;..\3rdparty\asmjit\asmjit\src;..\3rdparty\yaml-cpp\include;..\3rdparty\ffmpeg\include;..\3rdparty\cereal\include;$(VC_IncludePath);$(WindowsSDK_IncludePath);$(UniversalCRT_IncludePath);..\3rdparty\libpng\libpng;..\3rdparty\GL;..\3rdparty\stblib\include;..\3rdparty\OpenAL\include;..\3rdparty\pugixml\src;..\3rdparty\hidapi\hidapi;..\3rdparty\Optional;..\3rdparty\xxhash</IncludePath>
+    <IncludePath>.\;..\;..\3rdparty\asmjit\asmjit\src;..\3rdparty\yaml-cpp\include;..\3rdparty\ffmpeg\include;$(VC_IncludePath);$(WindowsSDK_IncludePath);$(UniversalCRT_IncludePath);..\3rdparty\libpng\libpng;..\3rdparty\GL;..\3rdparty\stblib\include;..\3rdparty\OpenAL\include;..\3rdparty\pugixml\src;..\3rdparty\hidapi\hidapi;..\3rdparty\Optional;..\3rdparty\xxhash</IncludePath>
     <OutDir>$(SolutionDir)lib\$(Configuration)-$(Platform)\</OutDir>
     <LibraryPath>$(SolutionDir)lib\$(Configuration)-$(Platform)\;$(UniversalCRT_LibraryPath_x64);$(LibraryPath)</LibraryPath>
     <IntDir>$(SolutionDir)tmp\$(ProjectName)-$(Configuration)-$(Platform)\</IntDir>

rpcs3/Emu/CMakeLists.txt

@@ -38,7 +38,6 @@ target_sources(rpcs3_emu PRIVATE
     ../util/atomic.cpp
     ../util/logs.cpp
     ../util/yaml.cpp
-    ../util/cereal.cpp
     ../util/vm_native.cpp
     ../util/dyn_lib.cpp
     ../util/sysinfo.cpp
@@ -76,12 +75,6 @@ else()
     set_source_files_properties("../util/yaml.cpp" PROPERTIES COMPILE_FLAGS -fexceptions)
 endif()
 
-if(MSVC)
-    set_source_files_properties("../util/cereal.cpp" PROPERTIES COMPILE_FLAGS /GR)
-else()
-    set_source_files_properties("../util/cereal.cpp" PROPERTIES COMPILE_FLAGS -frtti)
-endif()
-
 # Crypto
 target_sources(rpcs3_emu PRIVATE
     ../Crypto/aes.cpp
@@ -499,7 +492,7 @@ endif()
 
 target_link_libraries(rpcs3_emu
     PUBLIC
-        3rdparty::ffmpeg 3rdparty::cereal
+        3rdparty::ffmpeg
         3rdparty::opengl 3rdparty::stblib
         3rdparty::vulkan 3rdparty::glew
         3rdparty::libusb 3rdparty::wolfssl

rpcs3/Emu/RSX/Capture/rsx_replay.h

@@ -8,35 +8,27 @@
 
 namespace rsx
 {
-	constexpr u32 FRAME_CAPTURE_MAGIC = 0x52524300; // ascii 'RRC/0'
-	constexpr u32 FRAME_CAPTURE_VERSION = 0x4;
+	enum : u32
+	{
+		c_fc_magic = "RRC"_u32,
+		c_fc_version = 0x5,
+	};
+
 	struct frame_capture_data
 	{
 		struct memory_block_data
 		{
 			std::vector<u8> data{};
-
-			template<typename Archive>
-			void serialize(Archive& ar)
-			{
-				ar(data);
-			}
 		};
 
 		// simple block to hold ps3 address and data
 		struct memory_block
 		{
+			static constexpr bool enable_bitcopy = true;
+
 			u32 offset; // Offset in rsx address space
 			u32 location; // rsx memory location of the block
 			u64 data_state;
-
-			template<typename Archive>
-			void serialize(Archive & ar)
-			{
-				ar(offset);
-				ar(location);
-				ar(data_state);
-			}
 		};
 
 		struct replay_command
@@ -45,101 +37,61 @@ namespace rsx
 			std::unordered_set<u64> memory_state{}; // index into memory_map for the various memory blocks that need applying before this command can run
 			u64 tile_state{0};                      // tile state for this command
 			u64 display_buffer_state{0};
-
-			template<typename Archive>
-			void serialize(Archive & ar)
-			{
-				ar(rsx_command);
-				ar(memory_state);
-				ar(tile_state);
-				ar(display_buffer_state);
-			}
 		};
 
 		struct tile_info
 		{
+			static constexpr bool enable_bitcopy = true;
+
 			u32 tile;
 			u32 limit;
 			u32 pitch;
 			u32 format;
-
-			template<typename Archive>
-			void serialize(Archive & ar)
-			{
-				ar(tile);
-				ar(limit);
-				ar(pitch);
-				ar(format);
-			}
 		};
 
 		struct zcull_info
 		{
+			static constexpr bool enable_bitcopy = true;
+
 			u32 region;
 			u32 size;
 			u32 start;
 			u32 offset;
 			u32 status0;
 			u32 status1;
-
-			template<typename Archive>
-			void serialize(Archive & ar)
-			{
-				ar(region);
-				ar(size);
-				ar(start);
-				ar(offset);
-				ar(status0);
-				ar(status1);
-			}
 		};
 
 		// bleh, may need to break these out, might be unnecessary to do both always
 		struct tile_state
 		{
+			static constexpr bool enable_bitcopy = true;
+
 			tile_info tiles[15]{};
 			zcull_info zculls[8]{};
-
-			template<typename Archive>
-			void serialize(Archive & ar)
-			{
-				ar(tiles);
-				ar(zculls);
-			}
 		};
 
 		struct buffer_state
 		{
+			static constexpr bool enable_bitcopy = true;
+
 			u32 width{0};
 			u32 height{0};
 			u32 pitch{0};
 			u32 offset{0};
-
-			template<typename Archive>
-			void serialize(Archive & ar)
-			{
-				ar(width);
-				ar(height);
-				ar(pitch);
-				ar(offset);
-			}
 		};
 
 		struct display_buffers_state
 		{
+			static constexpr bool enable_bitcopy = true;
+
 			std::array<buffer_state, 8> buffers{};
 			u32 count{0};
-
-			template<typename Archive>
-			void serialize(Archive & ar)
-			{
-				ar(buffers);
-				ar(count);
-			}
 		};
 
-		u32 magic;
-		u32 version;
+		u32 magic = c_fc_magic;
+		u32 version = c_fc_version;
+		u32 LE_format = std::endian::little == std::endian::native;
+
 		// hashmap of holding various states for tile
 		std::unordered_map<u64, tile_state> tile_map;
 		// hashmap of various memory 'changes' that can be applied to ps3 memory
@@ -153,23 +105,10 @@ namespace rsx
 		// Initial registers state at the beginning of the capture
 		rsx::rsx_state reg_state;
 
-		template<typename Archive>
-		void serialize(Archive & ar)
-		{
-			ar(magic);
-			ar(version);
-			ar(tile_map);
-			ar(memory_map);
-			ar(memory_data_map);
-			ar(display_buffers_map);
-			ar(replay_commands);
-			ar(reg_state);
-		}
-
 		void reset()
 		{
-			magic = FRAME_CAPTURE_MAGIC;
-			version = FRAME_CAPTURE_VERSION;
+			magic = c_fc_magic;
+			version = c_fc_version;
 			tile_map.clear();
 			memory_map.clear();
 			replay_commands.clear();

rpcs3/Emu/RSX/RSXThread.cpp

@@ -19,7 +19,7 @@
 #include "Utilities/date_time.h"
 #include "Utilities/StrUtil.h"
 
-#include "util/cereal.hpp"
+#include "util/serialization.hpp"
 #include "util/asm.hpp"
 
 #include <span>
@@ -39,6 +39,37 @@ rsx::frame_capture_data frame_capture;
 extern CellGcmOffsetTable offsetTable;
 extern thread_local std::string(*g_tls_log_prefix)();
 
+template <>
+bool serialize<rsx::rsx_state>(utils::serial& ar, rsx::rsx_state& o)
+{
+	return ar(o.transform_program, /*o.transform_constants,*/ o.registers);
+}
+
+template <>
+bool serialize<rsx::frame_capture_data>(utils::serial& ar, rsx::frame_capture_data& o)
+{
+	ar(o.magic, o.version, o.LE_format);
+
+	if (o.magic != rsx::c_fc_magic || o.version != rsx::c_fc_version || o.LE_format != (std::endian::little == std::endian::native))
+	{
+		return false;
+	}
+
+	return ar(o.tile_map, o.memory_map, o.memory_data_map, o.display_buffers_map, o.replay_commands, o.reg_state);
+}
+
+template <>
+bool serialize<rsx::frame_capture_data::memory_block_data>(utils::serial& ar, rsx::frame_capture_data::memory_block_data& o)
+{
+	return ar(o.data);
+}
+
+template <>
+bool serialize<rsx::frame_capture_data::replay_command>(utils::serial& ar, rsx::frame_capture_data::replay_command& o)
+{
+	return ar(o.rsx_command, o.memory_state, o.tile_state, o.display_buffer_state);
+}
+
 namespace rsx
 {
 	std::function<bool(u32 addr, bool is_writing)> g_access_violation_handler;
@@ -2861,11 +2892,14 @@ namespace rsx
 			const std::string file_path = fs::get_config_dir() + "captures/" + Emu.GetTitleID() + "_" + date_time::current_time_narrow() + "_capture.rrc";
 
 			// todo: may want to compress this data?
-			const std::string file_data = cereal_serialize(frame_capture);
+			utils::serial save_manager;
+			save_manager.reserve(0x800'0000); // 128MB
+
+			save_manager(frame_capture);
 
 			fs::pending_file temp(file_path);
 
-			if (temp.file && (temp.file.write(file_data), temp.commit(false)))
+			if (temp.file && (temp.file.write(save_manager.data), temp.commit(false)))
 			{
 				rsx_log.success("Capture successful: %s", file_path);
 			}

rpcs3/Emu/RSX/rsx_methods.h

@@ -554,15 +554,6 @@ namespace rsx
 
 		void init();
 
-		template<typename Archive>
-		void serialize(Archive & ar)
-		{
-			ar(transform_program,
-//				transform_constants,
-				registers
-				);
-		}
-
 		u16 viewport_width() const
 		{
 			return decode<NV4097_SET_VIEWPORT_HORIZONTAL>().width();

rpcs3/Emu/System.cpp

@@ -30,7 +30,7 @@
 #include "../Crypto/unself.h"
 #include "util/yaml.hpp"
 #include "util/logs.hpp"
-#include "util/cereal.hpp"
+#include "util/serialization.hpp"
 
 #include <fstream>
 #include <memory>
@@ -380,18 +380,31 @@ bool Emulator::BootRsxCapture(const std::string& path)
 	}
 
 	std::unique_ptr<rsx::frame_capture_data> frame = std::make_unique<rsx::frame_capture_data>();
-	cereal_deserialize(*frame, in_file.to_string());
+	utils::serial load_manager;
+	load_manager.set_reading_state(in_file.to_vector<u8>());
+
+	load_manager(*frame);
 	in_file.close();
 
-	if (frame->magic != rsx::FRAME_CAPTURE_MAGIC)
+	if (frame->magic != rsx::c_fc_magic)
 	{
 		sys_log.error("Invalid rsx capture file!");
 		return false;
 	}
 
-	if (frame->version != rsx::FRAME_CAPTURE_VERSION)
+	if (frame->version != rsx::c_fc_version)
 	{
-		sys_log.error("Rsx capture file version not supported! Expected %d, found %d", rsx::FRAME_CAPTURE_VERSION, frame->version);
+		sys_log.error("Rsx capture file version not supported! Expected %d, found %d", +rsx::c_fc_version, frame->version);
+		return false;
+	}
+
+	if (frame->LE_format != (std::endian::little == std::endian::native))
+	{
+		static constexpr std::string_view machines[2]{"Big-Endian", "Little-Endian"};
+
+		sys_log.error("Rsx capture byte endianness not supported! Expected %s format, found %s format"
+			, machines[frame->LE_format ^ 1], machines[frame->LE_format]);
+
 		return false;
 	}