From 0252291bb907635c2c5255b158d60f57ae902ea8 Mon Sep 17 00:00:00 2001 From: Taku Kudo Date: Mon, 10 Feb 2025 09:04:59 +0000 Subject: [PATCH] Remove tiny_storage and registory. They were mainly used in usage_stats. PiperOrigin-RevId: 725108576 --- src/session/BUILD.bazel | 2 - src/session/session_handler_test_util.cc | 2 - src/session/session_handler_tool.cc | 2 - src/storage/BUILD.bazel | 77 +----- src/storage/registry.cc | 115 -------- src/storage/registry.h | 132 --------- src/storage/registry_test.cc | 96 ------- src/storage/storage.gyp | 2 - src/storage/storage_interface.h | 84 ------ src/storage/storage_test.gyp | 2 - src/storage/tiny_storage.cc | 324 ----------------------- src/storage/tiny_storage.h | 60 ----- src/storage/tiny_storage_test.cc | 144 ---------- 13 files changed, 3 insertions(+), 1039 deletions(-) delete mode 100644 src/storage/registry.cc delete mode 100644 src/storage/registry.h delete mode 100644 src/storage/registry_test.cc delete mode 100644 src/storage/storage_interface.h delete mode 100644 src/storage/tiny_storage.cc delete mode 100644 src/storage/tiny_storage.h delete mode 100644 src/storage/tiny_storage_test.cc diff --git a/src/session/BUILD.bazel b/src/session/BUILD.bazel index 45a84ec05..6fa373bae 100644 --- a/src/session/BUILD.bazel +++ b/src/session/BUILD.bazel @@ -206,7 +206,6 @@ mozc_cc_library( "//prediction:user_history_predictor", "//protocol:commands_cc_proto", "//protocol:config_cc_proto", - "//storage:registry", "//testing:mozctest", "@com_google_absl//absl/flags:flag", ], @@ -491,7 +490,6 @@ mozc_cc_library( "//protocol:commands_cc_proto", "//protocol:config_cc_proto", "//request:request_test_util", - "//storage:registry", "@com_google_absl//absl/log", "@com_google_absl//absl/log:check", "@com_google_absl//absl/status", diff --git a/src/session/session_handler_test_util.cc b/src/session/session_handler_test_util.cc index c908c58e5..5a3dba4ec 100644 --- a/src/session/session_handler_test_util.cc +++ b/src/session/session_handler_test_util.cc @@ -41,7 +41,6 @@ #include "protocol/commands.pb.h" #include "protocol/config.pb.h" #include "session/session_handler.h" -#include "storage/registry.h" ABSL_DECLARE_FLAG(int32_t, max_session_size); ABSL_DECLARE_FLAG(int32_t, create_session_min_interval); @@ -142,7 +141,6 @@ void SessionHandlerTestBase::ClearState() { // Some destructors may save the state on storages. To clear the state, we // explicitly call destructors before clearing storages. - storage::Registry::Clear(); FileUtil::UnlinkOrLogError( ConfigFileStream::GetFileName("user://boundary.db")); FileUtil::UnlinkOrLogError( diff --git a/src/session/session_handler_tool.cc b/src/session/session_handler_tool.cc index 377f5d4ac..b2ca9fb73 100644 --- a/src/session/session_handler_tool.cc +++ b/src/session/session_handler_tool.cc @@ -65,7 +65,6 @@ #include "protocol/config.pb.h" #include "request/request_test_util.h" #include "session/session_handler.h" -#include "storage/registry.h" namespace mozc { namespace session { @@ -335,7 +334,6 @@ void SessionHandlerInterpreter::ClearState() { // Some destructors may save the state on storages. To clear the state, we // explicitly call destructors before clearing storages. - storage::Registry::Clear(); FileUtil::UnlinkOrLogError( ConfigFileStream::GetFileName("user://boundary.db")); FileUtil::UnlinkOrLogError( diff --git a/src/storage/BUILD.bazel b/src/storage/BUILD.bazel index a5b8b250b..19be219c3 100644 --- a/src/storage/BUILD.bazel +++ b/src/storage/BUILD.bazel @@ -132,79 +132,6 @@ mozc_cc_test( ], ) -mozc_cc_library( - name = "storage_interaface", - hdrs = ["storage_interface.h"], - deps = ["//base:port"], -) - -mozc_cc_library( - name = "tiny_storage", - srcs = ["tiny_storage.cc"], - hdrs = ["tiny_storage.h"], - deps = [ - ":storage_interaface", - "//base:bits", - "//base:file_stream", - "//base:file_util", - "//base:mmap", - "//base:vlog", - "@com_google_absl//absl/container:flat_hash_map", - "@com_google_absl//absl/log", - "@com_google_absl//absl/log:check", - "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/strings", - ], -) - -mozc_cc_test( - name = "tiny_storage_test", - size = "small", - srcs = ["tiny_storage_test.cc"], - deps = [ - ":storage_interaface", - ":tiny_storage", - "//base/file:temp_dir", - "//testing:gunit_main", - "//testing:mozctest", - "@com_google_absl//absl/container:flat_hash_map", - "@com_google_absl//absl/strings", - ], -) - -mozc_cc_library( - name = "registry", - srcs = ["registry.cc"], - hdrs = ["registry.h"], - deps = [ - ":storage_interaface", - ":tiny_storage", - "//base:bits", - "//base:file_util", - "//base:port", - "//base:singleton", - "//base:system_util", - "//base:vlog", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/log", - "@com_google_absl//absl/log:check", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/synchronization", - ], -) - -mozc_cc_test( - name = "registry_test", - size = "small", - srcs = ["registry_test.cc"], - deps = [ - ":registry", - "//testing:gunit_main", - "//testing:mozctest", - ], -) - mozc_cc_library( name = "encrypted_string_storage", srcs = ["encrypted_string_storage.cc"], @@ -228,7 +155,9 @@ mozc_cc_test( name = "encrypted_string_storage_test", size = "small", srcs = ["encrypted_string_storage_test.cc"], - tags = ["nowin"], # TODO(yuryu): depends on //base:encryptor + tags = [ + "nowin", + ], # TODO(yuryu): depends on //base:encryptor deps = [ ":encrypted_string_storage", "//base:file_stream", diff --git a/src/storage/registry.cc b/src/storage/registry.cc deleted file mode 100644 index cf6f4f062..000000000 --- a/src/storage/registry.cc +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2010-2021, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -#include "storage/registry.h" - -#include -#include -#include - -#include "absl/base/attributes.h" -#include "absl/base/const_init.h" -#include "absl/log/log.h" -#include "absl/strings/string_view.h" -#include "absl/synchronization/mutex.h" -#include "base/file_util.h" -#include "base/port.h" -#include "base/singleton.h" -#include "base/system_util.h" -#include "base/vlog.h" -#include "storage/storage_interface.h" -#include "storage/tiny_storage.h" - -namespace mozc { -namespace storage { -namespace { - -ABSL_CONST_INIT absl::Mutex g_mutex(absl::kConstInit); - -constexpr absl::string_view RegistryFileName() { - if constexpr (TargetIsWindows()) { - return "registry.db"; - } else { - return ".registry.db"; // hidden file - } -} - -class StorageInitializer { - public: - StorageInitializer() { SetStorage(TinyStorage::New()); } - - StorageInterface *GetStorage() const { return storage_.get(); } - - void SetStorage(std::unique_ptr storage) { - storage_ = std::move(storage); - if (!storage_->Open(FileUtil::JoinPath( - SystemUtil::GetUserProfileDirectory(), RegistryFileName()))) { - LOG(ERROR) << "cannot open registry"; - } - } - - private: - std::unique_ptr storage_; -}; -} // namespace - -bool Registry::Erase(const std::string &key) { - absl::MutexLock l(&g_mutex); - return Singleton::get()->GetStorage()->Erase(key); -} - -bool Registry::Sync() { - absl::MutexLock l(&g_mutex); - return Singleton::get()->GetStorage()->Sync(); -} - -// clear internal keys and values -bool Registry::Clear() { - absl::MutexLock l(&g_mutex); - return Singleton::get()->GetStorage()->Clear(); -} - -void Registry::SetStorage(std::unique_ptr handler) { - MOZC_VLOG(1) << "New storage interface is set"; - absl::MutexLock l(&g_mutex); - Singleton::get()->SetStorage(std::move(handler)); -} - -bool Registry::LookupInternal(const std::string &key, std::string *value) { - absl::MutexLock l(&g_mutex); // just for safe - return Singleton::get()->GetStorage()->Lookup(key, value); -} - -bool Registry::InsertInternal(const std::string &key, - const std::string &value) { - absl::MutexLock l(&g_mutex); - return Singleton::get()->GetStorage()->Insert(key, value); -} -} // namespace storage -} // namespace mozc diff --git a/src/storage/registry.h b/src/storage/registry.h deleted file mode 100644 index 5993b9f3a..000000000 --- a/src/storage/registry.h +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright 2010-2021, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -#ifndef MOZC_STORAGE_REGISTRY_H_ -#define MOZC_STORAGE_REGISTRY_H_ - -#include -#include -#include - -#include "absl/log/check.h" -#include "base/bits.h" -#include "storage/storage_interface.h" - -namespace mozc { -namespace storage { - -// The idea of Registry module is the same as Windows Registry. -// You can use it for saving small data like timestamp, auth_token. -// DO NOT USE it to save big data or data which are frequently updated. -// Lookup() and Insert() do process-wide global lock and it may be slow. -// All methods are thread-safe. -// -// TODO(taku): Currently, Registry won't guarantee that two processes -// can con-currently share the same data. We will replace the backend -// of Registry (Storage Interface) with sqlite -// -// Example: -// uint64_t timestamp = 0; -// CHECK(Registry::Lookup("timestamp", ×tamp)); -// -// string value = "hello world"; -// CHECK(Registry::Insert("hello", value)); -class Registry { - public: - Registry() = delete; - Registry(const Registry &) = delete; - Registry &operator=(const Registry &) = delete; - - template - static bool Lookup(const std::string &key, T *value) { - DCHECK(value); - std::string tmp; - if (!LookupInternal(key, &tmp)) { - return false; - } - if (sizeof(*value) != tmp.size()) { - return false; - } - *value = LoadUnaligned(tmp.data()); - return true; - } - - static bool Lookup(const std::string &key, std::string *value) { - return LookupInternal(key, value); - } - - static bool Lookup(const std::string &key, bool *value) { - uint8_t v = 0; - const bool result = Lookup(key, &v); - *value = (v != 0); - return result; - } - - // insert key and data - // It is not guaranteed that the data is synced to the disk - template - static bool Insert(const std::string &key, const T &value) { - std::string tmp(reinterpret_cast(&value), sizeof(value)); - return InsertInternal(key, tmp); - } - - // Insert key and string value - // It is not guaranteed that the data is synced to the disk - static bool Insert(const std::string &key, const std::string &value) { - return InsertInternal(key, value); - } - - // Insert key and bool value - // It is not guaranteed that the data is synced to the disk - static bool Insert(const std::string &key, const bool value) { - const uint8_t tmp = static_cast(value); - return Insert(key, tmp); - } - - // Syncing the data into disk - static bool Sync(); - - // Erase key - static bool Erase(const std::string &key); - - // clear internal keys and values - static bool Clear(); - - // Inject internal storage for unittesting. - // TinyStorage is used by default - // TODO(taku) replace it with SQLITE - static void SetStorage(std::unique_ptr handler); - - private: - static bool LookupInternal(const std::string &key, std::string *value); - static bool InsertInternal(const std::string &key, const std::string &value); -}; -} // namespace storage -} // namespace mozc -#endif // MOZC_STORAGE_REGISTRY_H_ diff --git a/src/storage/registry_test.cc b/src/storage/registry_test.cc deleted file mode 100644 index 5616661e0..000000000 --- a/src/storage/registry_test.cc +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2010-2021, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -#include "storage/registry.h" - -#include -#include - -#include "testing/gunit.h" -#include "testing/mozctest.h" - -namespace mozc { -namespace storage { -namespace { - -class RegistryTest : public testing::TestWithTempUserProfile {}; - -TEST_F(RegistryTest, TinyStorageTest) { - { - uint64_t value = 20; - EXPECT_TRUE(Registry::Insert("uint64_t", value)); - uint64_t expected = 0; - EXPECT_TRUE(Registry::Lookup("uint64_t", &expected)); - EXPECT_EQ(value, expected); - } - - { - uint32_t value = 20; - EXPECT_TRUE(Registry::Insert("uint32_t", value)); - uint32_t expected = 0; - EXPECT_TRUE(Registry::Lookup("uint32_t", &expected)); - EXPECT_EQ(value, expected); - } - - { - uint16_t value = 20; - EXPECT_TRUE(Registry::Insert("uint16_t", value)); - uint16_t expected = 0; - EXPECT_TRUE(Registry::Lookup("uint16_t", &expected)); - EXPECT_EQ(value, expected); - } - - { - uint8_t value = 20; - EXPECT_TRUE(Registry::Insert("uint8_t", value)); - uint8_t expected = 0; - EXPECT_TRUE(Registry::Lookup("uint8_t", &expected)); - EXPECT_EQ(value, expected); - } - - { - bool value = true; - EXPECT_TRUE(Registry::Insert("bool", value)); - bool expected = false; - EXPECT_TRUE(Registry::Lookup("bool", &expected)); - EXPECT_EQ(value, expected); - } - - { - std::string value = "test"; - EXPECT_TRUE(Registry::Insert("string", value)); - std::string expected; - EXPECT_TRUE(Registry::Lookup("string", &expected)); - EXPECT_EQ(value, expected); - } -} - -} // namespace -} // namespace storage -} // namespace mozc diff --git a/src/storage/storage.gyp b/src/storage/storage.gyp index 7e9a0629b..9acd89c92 100644 --- a/src/storage/storage.gyp +++ b/src/storage/storage.gyp @@ -37,8 +37,6 @@ 'encrypted_string_storage.cc', 'existence_filter.cc', 'lru_storage.cc', - 'registry.cc', - 'tiny_storage.cc', ], 'dependencies': [ '<(mozc_oss_src_dir)/base/absl.gyp:absl_strings', diff --git a/src/storage/storage_interface.h b/src/storage/storage_interface.h deleted file mode 100644 index 1204a15a3..000000000 --- a/src/storage/storage_interface.h +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2010-2021, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -#ifndef MOZC_STORAGE_STORAGE_INTERFACE_H_ -#define MOZC_STORAGE_STORAGE_INTERFACE_H_ - -#include -#include - - -namespace mozc { -namespace storage { - -class StorageInterface { - public: - StorageInterface() = default; - virtual ~StorageInterface() = default; - - // Binds |filename| to the storage but the interpretation of the - // |filename| depends on the implementation. implementations can - // - ignore the specified |filename|. - // - load the existing data from |filename|. - // - return true if the specified |filename| does not exist. - // Despite its name, an implementation may not keep the underlaying - // data storage specified by |filename| opened. - // StorageInterface::Sync may try to open |filename| again but it - // may fail even when this method returns true. - virtual bool Open(const std::string &filename) = 0; - - // Flushes on-memory data into persistent storage (usually on - // disk) specified by |filename|. - virtual bool Sync() = 0; - - // Looks up key and returns the value. - // If key is not found, returns false. - // It is not guaranteed that the data is synced to the disk. - virtual bool Lookup(const std::string &key, std::string *value) const = 0; - - // Inserts key and value. - // It is not guaranteed that the data is synced to the disk. - virtual bool Insert(const std::string &key, const std::string &value) = 0; - - // Erases the key-value pair specified by |key|. - // It is not guaranteed that the data is synced to the disk - virtual bool Erase(const std::string &key) = 0; - - // clears internal keys and values - // Sync() is automatically called. - virtual bool Clear() = 0; - - // Returns the number of keys, not a number of bytes. - virtual size_t Size() const = 0; -}; - -} // namespace storage -} // namespace mozc - -#endif // MOZC_STORAGE_STORAGE_INTERFACE_H_ diff --git a/src/storage/storage_test.gyp b/src/storage/storage_test.gyp index 990c65251..409d838ae 100644 --- a/src/storage/storage_test.gyp +++ b/src/storage/storage_test.gyp @@ -37,8 +37,6 @@ 'existence_filter_test.cc', 'lru_cache_test.cc', 'lru_storage_test.cc', - 'registry_test.cc', - 'tiny_storage_test.cc', ], 'dependencies': [ '<(mozc_oss_src_dir)/testing/testing.gyp:gtest_main', diff --git a/src/storage/tiny_storage.cc b/src/storage/tiny_storage.cc deleted file mode 100644 index ab9e38e21..000000000 --- a/src/storage/tiny_storage.cc +++ /dev/null @@ -1,324 +0,0 @@ -// Copyright 2010-2021, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -#include "storage/tiny_storage.h" - -#include -#include -#include -#include -#include -#include -#include - -#include "absl/container/flat_hash_map.h" -#include "absl/log/check.h" -#include "absl/log/log.h" -#include "absl/status/status.h" -#include "absl/status/statusor.h" -#include "absl/strings/str_cat.h" -#include "absl/strings/string_view.h" -#include "base/bits.h" -#include "base/file_stream.h" -#include "base/file_util.h" -#include "base/mmap.h" -#include "base/vlog.h" -#include "storage/storage_interface.h" - -namespace mozc { -namespace storage { -namespace { - -constexpr uint32_t kStorageVersion = 0; -constexpr uint32_t kStorageMagicId = 0x431fe241; // random seed -constexpr size_t kMaxElementSize = 1024; // max map size -constexpr size_t kMaxKeySize = 4096; // 4k for key/value -constexpr size_t kMaxValueSize = 4096; // 4k for key/value -// 1024 * (4096 + 4096) =~ 8MByte -// so 10Mbyte data is reasonable upper bound for file size -constexpr size_t kMaxFileSize = 1024 * 1024 * 10; // 10Mbyte - -bool IsInvalid(const absl::string_view key, const absl::string_view value, - const size_t size) { - if (size >= kMaxElementSize) { - LOG(ERROR) << "too many elements"; - return true; - } - if (key.size() >= kMaxKeySize) { - LOG(ERROR) << "too long key"; - return true; - } - if (value.size() >= kMaxValueSize) { - LOG(ERROR) << "too long value"; - return true; - } - return false; -} - -class TinyStorageImpl : public StorageInterface { - public: - TinyStorageImpl(); - TinyStorageImpl(const TinyStorageImpl &) = delete; - TinyStorageImpl &operator=(const TinyStorageImpl &) = delete; - ~TinyStorageImpl() override; - - bool Open(const std::string &filename) override; - bool Sync() override; - bool Lookup(const std::string &key, std::string *value) const override; - bool Insert(const std::string &key, const std::string &value) override; - bool Erase(const std::string &key) override; - bool Clear() override; - size_t Size() const override { return dic_.size(); } - - private: - std::string filename_; - bool should_sync_; - absl::flat_hash_map dic_; -}; - -TinyStorageImpl::TinyStorageImpl() : should_sync_(true) { - // the each entry consumes at most - // sizeof(uint32_t) * 2 (key/value length) + - // kMaxKeySize + kMaxValueSize - DCHECK_GT(kMaxFileSize, kMaxElementSize * (kMaxKeySize + kMaxValueSize + - sizeof(uint32_t) * 2)); -} - -TinyStorageImpl::~TinyStorageImpl() { - if (should_sync_) { - Sync(); - } -} - -bool TinyStorageImpl::Open(const std::string &filename) { - dic_.clear(); - filename_ = filename; - absl::StatusOr mmap = Mmap::Map(filename, Mmap::READ_ONLY); - if (!mmap.ok()) { - LOG(WARNING) << "cannot open:" << filename << ": " << mmap.status(); - // here we return true if we cannot open the file. - // it happens mostly when file doesn't exist. - // we just make an empty file from scratch here. - return true; - } - - if (mmap->size() < 20) { - LOG(ERROR) << "the file is missing the header."; - return false; - } - if (mmap->size() > kMaxFileSize) { - LOG(ERROR) << "tring to open too big file"; - return false; - } - - const char *begin = mmap->begin(); - const char *const end = mmap->end(); - - // magic is used for checking whether given file is correct or not. - // magic = (file_size ^ kStorageMagicId) - const uint32_t magic = LoadUnalignedAdvance(begin); - if ((magic ^ kStorageMagicId) != mmap->size()) { - LOG(ERROR) << "file magic is broken"; - return false; - } - const uint32_t version = LoadUnalignedAdvance(begin); - - if (version != kStorageVersion) { - LOG(ERROR) << "Incompatible version"; - return false; - } - const uint32_t size = LoadUnalignedAdvance(begin); - - for (size_t i = 0; i < size; ++i) { - if (std::distance(begin, end) < sizeof(uint32_t)) { - LOG(ERROR) << "key_size is invalid"; - return false; - } - const uint32_t key_size = LoadUnalignedAdvance(begin); - if (begin + key_size > end) { - LOG(ERROR) << "Too long key is passed"; - return false; - } - - const absl::string_view key(begin, key_size); - begin += key_size; - - if (std::distance(begin, end) < sizeof(uint32_t)) { - LOG(ERROR) << "value_size is invalid"; - return false; - } - const uint32_t value_size = LoadUnalignedAdvance(begin); - if (begin + value_size > end) { - LOG(ERROR) << "Too long value is passed"; - return false; - } - - const absl::string_view value(begin, value_size); - begin += value_size; - - if (IsInvalid(key, value, dic_.size())) { - return false; - } - - dic_.emplace(key, value); - } - - if (static_cast(begin - mmap->begin()) != mmap->size()) { - LOG(ERROR) << "file is broken: " << filename_; - dic_.clear(); - return false; - } - - return true; -} - -// Format of storage: -// |magic(uint32_t file_size ^ -// kStorageVersion)|version(uint32_t)|size(uint32_t)| -// |key_size(uint32_t)|key(variable length)| -// |value_size(uint32_t)|value(variable length)| ... -bool TinyStorageImpl::Sync() { - if (!should_sync_) { - MOZC_VLOG(2) << "Already synced"; - return true; - } - - const std::string output_filename = absl::StrCat(filename_, ".tmp"); - - OutputFileStream ofs(output_filename, std::ios::binary | std::ios::out); - if (!ofs) { - LOG(ERROR) << "cannot open " << output_filename; - return false; - } - - uint32_t magic = 0; - uint32_t size = 0; - ofs.write(reinterpret_cast(&magic), sizeof(magic)); - ofs.write(reinterpret_cast(&kStorageVersion), - sizeof(kStorageVersion)); - ofs.write(reinterpret_cast(&size), sizeof(size)); - - for (auto &[key, value] : dic_) { - if (key.empty()) { - continue; - } - const uint32_t key_size = static_cast(key.size()); - const uint32_t value_size = static_cast(value.size()); - ofs.write(reinterpret_cast(&key_size), sizeof(key_size)); - ofs.write(key.data(), key_size); - ofs.write(reinterpret_cast(&value_size), sizeof(value_size)); - ofs.write(value.data(), value_size); - ++size; - } - - magic = static_cast(ofs.tellp()); - ofs.seekp(0); - magic ^= kStorageMagicId; - - ofs.write(reinterpret_cast(&magic), sizeof(magic)); - ofs.write(reinterpret_cast(&kStorageVersion), - sizeof(kStorageVersion)); - ofs.write(reinterpret_cast(&size), sizeof(size)); - - // should call close(). Othrwise AtomicRename will be failed. - ofs.close(); - - if (absl::Status s = FileUtil::AtomicRename(output_filename, filename_); - !s.ok()) { - LOG(ERROR) << "AtomicRename failed: " << s << "; from: " << output_filename - << ", to:" << filename_; - return false; - } - -#ifdef _WIN32 - if (!FileUtil::HideFile(filename_)) { - LOG(ERROR) << "Cannot make hidden: " << filename_ << " " - << ::GetLastError(); - } -#endif // _WIN32 - - should_sync_ = false; - - return true; -} - -bool TinyStorageImpl::Insert(const std::string &key, const std::string &value) { - if (IsInvalid(key, value, dic_.size())) { - LOG(WARNING) << "invalid key/value is passed"; - return false; - } - dic_[key] = value; - should_sync_ = true; - return true; -} - -bool TinyStorageImpl::Erase(const std::string &key) { - auto node = dic_.extract(key); - if (node.empty()) { - MOZC_VLOG(2) << "cannot erase key: " << key; - return false; - } - should_sync_ = true; - return true; -} - -bool TinyStorageImpl::Lookup(const std::string &key, std::string *value) const { - const auto it = dic_.find(key); - if (it == dic_.end()) { - MOZC_VLOG(3) << "cannot find key: " << key; - return false; - } - *value = it->second; - return true; -} - -bool TinyStorageImpl::Clear() { - dic_.clear(); - should_sync_ = true; - return Sync(); -} - -} // namespace - -std::unique_ptr TinyStorage::Create( - const std::string &filename) { - auto storage = std::make_unique(); - if (!storage->Open(filename)) { - LOG(ERROR) << "cannot open " << filename; - return nullptr; - } - return storage; -} - -std::unique_ptr TinyStorage::New() { - return std::make_unique(); -} - -} // namespace storage -} // namespace mozc diff --git a/src/storage/tiny_storage.h b/src/storage/tiny_storage.h deleted file mode 100644 index 77174b909..000000000 --- a/src/storage/tiny_storage.h +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2010-2021, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -#ifndef MOZC_STORAGE_TINY_STORAGE_H_ -#define MOZC_STORAGE_TINY_STORAGE_H_ - -#include -#include - -#include "storage/storage_interface.h" - -namespace mozc { -namespace storage { - -// Very simple and tiny key/value storage. -// Use it just for saving small data which -// are not updated frequently, like timestamp, auth_token, etc. -// We will replace it with faster and more robust implementation. -class TinyStorage { - public: - TinyStorage() = delete; - TinyStorage(const TinyStorage &) = delete; - TinyStorage &operator=(const TinyStorage &) = delete; - // Returns an implementation of StorageInterface. - // Caller must take ownership of the returned object. - // Returns NULL if fails. - static std::unique_ptr New(); - static std::unique_ptr Create(const std::string &filename); -}; - -} // namespace storage -} // namespace mozc - -#endif // MOZC_STORAGE_TINY_STORAGE_H_ diff --git a/src/storage/tiny_storage_test.cc b/src/storage/tiny_storage_test.cc deleted file mode 100644 index b8395296c..000000000 --- a/src/storage/tiny_storage_test.cc +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright 2010-2021, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -#include "storage/tiny_storage.h" - -#include -#include -#include -#include -#include -#include -#include - -#include "absl/container/flat_hash_map.h" -#include "absl/strings/str_cat.h" -#include "base/file/temp_dir.h" -#include "storage/storage_interface.h" -#include "testing/gunit.h" -#include "testing/mozctest.h" - -namespace mozc { -namespace storage { -namespace { - -using TargetMap = absl::flat_hash_map; - -void CreateKeyValue(TargetMap *output, int size) { - output->clear(); - for (int i = 0; i < size; ++i) { - output->emplace(absl::StrCat("key", i), absl::StrCat("value", i)); - } -} - -} // namespace - -class TinyStorageTest : public ::testing::Test { - protected: - static std::unique_ptr CreateStorage() { - return TinyStorage::New(); - } -}; - -TEST_F(TinyStorageTest, TinyStorageTest) { - static constexpr int kSize[] = {10, 100, 1000}; - - for (int i = 0; i < std::size(kSize); ++i) { - TempFile temp_file(testing::MakeTempFileOrDie()); - std::unique_ptr storage(CreateStorage()); - - // Insert - TargetMap target; - CreateKeyValue(&target, kSize[i]); - { - EXPECT_TRUE(storage->Open(temp_file.path())); - for (const auto &[key, value] : target) { - EXPECT_TRUE(storage->Insert(key, value)); - } - } - - // Lookup - for (const auto &[key, expected] : target) { - std::string value; - EXPECT_TRUE(storage->Lookup(key, &value)); - EXPECT_EQ(value, expected); - } - - for (const auto &it : target) { - const std::string key = absl::StrCat(it.first, ".dummy"); - std::string value; - EXPECT_FALSE(storage->Lookup(key, &value)); - } - - storage->Sync(); - - std::unique_ptr storage2(CreateStorage()); - EXPECT_TRUE(storage2->Open(temp_file.path())); - EXPECT_EQ(storage2->Size(), storage->Size()); - - // Lookup - for (const auto &[key, expected] : target) { - std::string value; - EXPECT_TRUE(storage2->Lookup(key, &value)); - EXPECT_EQ(value, expected); - } - - for (const auto &it : target) { - const std::string key = it.first + ".dummy"; - std::string value; - EXPECT_FALSE(storage2->Lookup(key, &value)); - } - - // Erase - int id = 0; - for (const auto &it : target) { - if (id % 2 == 0) { - EXPECT_TRUE(storage->Erase(it.first)); - const std::string key = absl::StrCat(it.first, ".dummy"); - EXPECT_FALSE(storage->Erase(key)); - } - ++id; - } - - id = 0; - for (const auto &it : target) { - std::string value; - const std::string &key = it.first; - if (id % 2 == 0) { - EXPECT_FALSE(storage->Lookup(key, &value)); - } else { - EXPECT_TRUE(storage->Lookup(key, &value)); - } - ++id; - } - } -} - -} // namespace storage -} // namespace mozc