Add configuration options to HTTP server

contrib/epee/include/net/abstract_tcp_server2.h

@@ -65,6 +65,7 @@
 #define MONERO_DEFAULT_LOG_CATEGORY "net"
 
 #define ABSTRACT_SERVER_SEND_QUE_MAX_COUNT 1000
+#define ABSTRACT_SERVER_SEND_QUE_MAX_BYTES_DEFAULT 100 * 1024 * 1024
 
 namespace epee
 {
@@ -170,6 +171,7 @@ namespace net_utils
         } read;
         struct {
           std::deque<epee::byte_slice> queue;
+          std::size_t total_bytes;
           bool wait_consume;
         } write;
       };
@@ -268,11 +270,17 @@ namespace net_utils
     struct shared_state : connection_basic_shared_state, t_protocol_handler::config_type
     {
       shared_state()
-        : connection_basic_shared_state(), t_protocol_handler::config_type(), pfilter(nullptr), plimit(nullptr), stop_signal_sent(false)
+        : connection_basic_shared_state(),
+          t_protocol_handler::config_type(),
+          pfilter(nullptr),
+          plimit(nullptr),
+          response_soft_limit(ABSTRACT_SERVER_SEND_QUE_MAX_BYTES_DEFAULT), 
+          stop_signal_sent(false)
       {}
 
       i_connection_filter* pfilter;
       i_connection_limit* plimit;
+      std::size_t response_soft_limit;
       bool stop_signal_sent;
     };
 
@@ -380,6 +388,7 @@ namespace net_utils
 
     void set_connection_filter(i_connection_filter* pfilter);
     void set_connection_limit(i_connection_limit* plimit);
+    void set_response_soft_limit(std::size_t limit);
 
     void set_default_remote(epee::net_utils::network_address remote)
     {

contrib/epee/include/net/abstract_tcp_server2.inl

@@ -497,10 +497,12 @@ namespace net_utils
       if (m_state.socket.cancel_write) {
         m_state.socket.cancel_write = false;
         m_state.data.write.queue.clear();
+        m_state.data.write.total_bytes = 0;
         state_status_check();
       }
       else if (ec.value()) {
         m_state.data.write.queue.clear();
+        m_state.data.write.total_bytes = 0;
         interrupt();
       }
       else {
@@ -525,8 +527,11 @@ namespace net_utils
 
           start_timer(get_default_timeout(), true);
         }
-        assert(bytes_transferred == m_state.data.write.queue.back().size());
+        const std::size_t byte_count = m_state.data.write.queue.back().size();
+        assert(bytes_transferred == byte_count);
         m_state.data.write.queue.pop_back();
+        m_state.data.write.total_bytes -=
+          std::min(m_state.data.write.total_bytes, byte_count);
         m_state.condition.notify_all();
         start_write();
       }
@@ -757,6 +762,8 @@ namespace net_utils
     std::lock_guard<std::mutex> guard(m_state.lock);
     if (m_state.status != status_t::RUNNING || m_state.socket.wait_handshake)
       return false;
+    if (std::numeric_limits<std::size_t>::max() - m_state.data.write.total_bytes < message.size())
+      return false;
 
     // Wait for the write queue to fall below the max. If it doesn't after a
     // randomized delay, drop the connection.
@@ -774,7 +781,14 @@ namespace net_utils
           std::uniform_int_distribution<>(5000, 6000)(rng)
         );
       };
-      if (m_state.data.write.queue.size() <= ABSTRACT_SERVER_SEND_QUE_MAX_COUNT)
+
+      // The bytes check intentionally does not include incoming message size.
+      // This allows for a soft overflow; a single http response will never fail
+      // this check, but multiple responses could. Clients can avoid this case
+      // by reading the entire response before making another request. P2P
+      // should never hit the MAX_BYTES check (when using default values).
+      if (m_state.data.write.queue.size() <= ABSTRACT_SERVER_SEND_QUE_MAX_COUNT &&
+          m_state.data.write.total_bytes <= static_cast<shared_state&>(connection_basic::get_state()).response_soft_limit)
         return true;
       m_state.data.write.wait_consume = true;
       bool success = m_state.condition.wait_for(
@@ -783,8 +797,12 @@ namespace net_utils
         [this]{
           return (
             m_state.status != status_t::RUNNING ||
-            m_state.data.write.queue.size() <=
-              ABSTRACT_SERVER_SEND_QUE_MAX_COUNT
+            (
+              m_state.data.write.queue.size() <=
+                ABSTRACT_SERVER_SEND_QUE_MAX_COUNT &&
+              m_state.data.write.total_bytes <=
+                static_cast<shared_state&>(connection_basic::get_state()).response_soft_limit
+            )
           );
         }
       );
@@ -816,7 +834,9 @@ namespace net_utils
     ) {
       if (!wait_consume())
         return false;
+      const std::size_t byte_count = message.size();
       m_state.data.write.queue.emplace_front(std::move(message));
+      m_state.data.write.total_bytes += byte_count;
       start_write();
     }
     else {
@@ -826,6 +846,7 @@ namespace net_utils
         m_state.data.write.queue.emplace_front(
           message.take_slice(CHUNK_SIZE)
         );
+        m_state.data.write.total_bytes += m_state.data.write.queue.front().size();
         start_write();
       }
     }
@@ -1369,6 +1390,13 @@ namespace net_utils
   }
   //---------------------------------------------------------------------------------
   template<class t_protocol_handler>
+  void boosted_tcp_server<t_protocol_handler>::set_response_soft_limit(const std::size_t limit)
+  {
+    assert(m_state != nullptr); // always set in constructor
+    m_state->response_soft_limit = limit;
+  }
+  //---------------------------------------------------------------------------------
+  template<class t_protocol_handler>
   bool boosted_tcp_server<t_protocol_handler>::run_server(size_t threads_count, bool wait, const boost::thread::attributes& attrs)
   {
     TRY_ENTRY();

contrib/epee/include/net/http_protocol_handler.h

@@ -32,6 +32,7 @@
 
 #include <boost/optional/optional.hpp>
 #include <string>
+#include <unordered_map>
 #include "net_utils_base.h"
 #include "http_auth.h"
 #include "http_base.h"
@@ -54,8 +55,10 @@ namespace net_utils
 		{
 			std::string m_folder;
 			std::vector<std::string> m_access_control_origins;
+			std::unordered_map<std::string, std::size_t> m_connections;
 			boost::optional<login> m_user;
 			size_t m_max_content_length{std::numeric_limits<size_t>::max()};
+			std::size_t m_max_connections{3};
 			critical_section m_lock;
 		};
 
@@ -70,7 +73,7 @@ namespace net_utils
 			typedef http_server_config config_type;
 
 			simple_http_connection_handler(i_service_endpoint* psnd_hndlr, config_type& config, t_connection_context& conn_context);
-			virtual ~simple_http_connection_handler(){}
+			virtual ~simple_http_connection_handler();
 
 			bool release_protocol()
 			{
@@ -86,10 +89,7 @@ namespace net_utils
 			{
 				return true;
 			}
-			bool after_init_connection()
-			{
-				return true;
-			}
+			bool after_init_connection();
 			virtual bool handle_recv(const void* ptr, size_t cb);
 			virtual bool handle_request(const http::http_request_info& query_info, http_response_info& response);
 
@@ -146,6 +146,7 @@ namespace net_utils
 		protected:
 			i_service_endpoint* m_psnd_hndlr; 
 			t_connection_context& m_conn_context;
+			bool m_initialized;
 		};
 
 		template<class t_connection_context>
@@ -212,10 +213,6 @@ namespace net_utils
 			}
 			void handle_qued_callback()
 			{}
-			bool after_init_connection()
-			{
-				return true;
-			}
 
 		private:
 			//simple_http_connection_handler::config_type m_stub_config;

contrib/epee/include/net/http_protocol_handler.inl

@@ -208,11 +208,43 @@ namespace net_utils
 		m_newlines(0),
 		m_bytes_read(0),
 		m_psnd_hndlr(psnd_hndlr),
-		m_conn_context(conn_context)
+		m_conn_context(conn_context),
+		m_initialized(false)
 	{
 
 	}
 	//--------------------------------------------------------------------------------------------
+	template<class t_connection_context>
+	simple_http_connection_handler<t_connection_context>::~simple_http_connection_handler()
+	{
+	  try
+	  {
+	    if (m_initialized)
+	    {
+	      CRITICAL_REGION_LOCAL(m_config.m_lock);
+	      auto elem = m_config.m_connections.find(m_conn_context.m_remote_address.host_str());
+	      if (elem != m_config.m_connections.end())
+	      {
+	        if (elem->second == 1 || elem->second == 0)
+	          m_config.m_connections.erase(elem);
+	        else
+	          --(elem->second);
+	      }
+	    }
+	  }
+	  catch (...)
+	  {}
+	}
+	//--------------------------------------------------------------------------------------------
+	template<class t_connection_context>
+	bool simple_http_connection_handler<t_connection_context>::after_init_connection()
+	{
+	  CRITICAL_REGION_LOCAL(m_config.m_lock);
+	  ++m_config.m_connections[m_conn_context.m_remote_address.host_str()];
+	  m_initialized = true;
+	  return true;
+	}
+	//--------------------------------------------------------------------------------------------
     template<class t_connection_context>
 	bool simple_http_connection_handler<t_connection_context>::set_ready_state()
 	{

contrib/epee/include/net/http_server_impl_base.h

@@ -44,7 +44,8 @@ namespace epee
 {
 
   template<class t_child_class, class t_connection_context = epee::net_utils::connection_context_base>
-  class http_server_impl_base: public net_utils::http::i_http_server_handler<t_connection_context>
+  class http_server_impl_base: public net_utils::http::i_http_server_handler<t_connection_context>,
+                                      net_utils::i_connection_limit
   {
 
   public:
@@ -60,7 +61,8 @@ namespace epee
       const std::string& bind_ipv6_address = "::", bool use_ipv6 = false, bool require_ipv4 = true,
       std::vector<std::string> access_control_origins = std::vector<std::string>(),
       boost::optional<net_utils::http::login> user = boost::none,
-      net_utils::ssl_options_t ssl_options = net_utils::ssl_support_t::e_ssl_support_autodetect)
+      net_utils::ssl_options_t ssl_options = net_utils::ssl_support_t::e_ssl_support_autodetect,
+      const std::size_t max_ip_connections = 3, const std::size_t response_soft_limit = 0)
     {
 
       //set self as callback handler
@@ -75,6 +77,9 @@ namespace epee
       m_net_server.get_config_object().m_access_control_origins = std::move(access_control_origins);
 
       m_net_server.get_config_object().m_user = std::move(user);
+      m_net_server.get_config_object().m_max_connections = max_ip_connections;
+      m_net_server.set_response_soft_limit(response_soft_limit);
+      m_net_server.set_connection_limit(this);
 
       MGINFO("Binding on " << bind_ip << " (IPv4):" << bind_port);
       if (use_ipv6)
@@ -131,6 +136,17 @@ namespace epee
     }
 
   protected: 
+
+    virtual bool is_host_limit(const net_utils::network_address& na) override final
+    {
+      auto& config = m_net_server.get_config_object();
+      CRITICAL_REGION_LOCAL(config.m_lock);
+      const auto elem = config.m_connections.find(na.host_str());
+      if (elem != config.m_connections.end())
+        return config.m_max_connections <= elem->second;
+      return false;
+    }
+
     net_utils::boosted_tcp_server<net_utils::http::http_custom_handler<t_connection_context> > m_net_server;
   };
 }

src/rpc/core_rpc_server.cpp

@@ -163,6 +163,8 @@ namespace cryptonote
     command_line::add_arg(desc, arg_rpc_payment_difficulty);
     command_line::add_arg(desc, arg_rpc_payment_credits);
     command_line::add_arg(desc, arg_rpc_payment_allow_free_loopback);
+    command_line::add_arg(desc, arg_rpc_max_ip_connections);
+    command_line::add_arg(desc, arg_rpc_response_soft_limit);
   }
   //------------------------------------------------------------------------------------------------------------------------------
   core_rpc_server::core_rpc_server(
@@ -400,7 +402,9 @@ namespace cryptonote
     const bool inited = epee::http_server_impl_base<core_rpc_server, connection_context>::init(
       rng, std::move(port), std::move(bind_ip_str),
       std::move(bind_ipv6_str), std::move(rpc_config->use_ipv6), std::move(rpc_config->require_ipv4),
-      std::move(rpc_config->access_control_origins), std::move(http_login), std::move(rpc_config->ssl_options)
+      std::move(rpc_config->access_control_origins), std::move(http_login), std::move(rpc_config->ssl_options),
+      command_line::get_arg(vm, arg_rpc_max_ip_connections),
+      command_line::get_arg(vm, arg_rpc_response_soft_limit)
     );
 
     m_net_server.get_config_object().m_max_content_length = MAX_RPC_CONTENT_LENGTH;
@@ -3885,4 +3889,16 @@ namespace cryptonote
     , "Allow free access from the loopback address (ie, the local host)"
     , false
     };
+
+  const command_line::arg_descriptor<std::size_t> core_rpc_server::arg_rpc_max_ip_connections = {
+      "rpc-max-ip-connections"
+    , "Max RPC connections per IP permitted"
+    , 3
+  };
+
+  const command_line::arg_descriptor<std::size_t> core_rpc_server::arg_rpc_response_soft_limit = {
+      "rpc-response-soft-limit"
+    , "Max response bytes that can be queued, enforced at next response attempt"
+    , 25 * 1024 * 1024
+  };
 }  // namespace cryptonote

src/rpc/core_rpc_server.h

@@ -56,7 +56,6 @@ namespace cryptonote
   {
   public:
 
-    static const command_line::arg_descriptor<bool> arg_public_node;
     static const command_line::arg_descriptor<std::string, false, true, 2> arg_rpc_bind_port;
     static const command_line::arg_descriptor<std::string> arg_rpc_restricted_bind_port;
     static const command_line::arg_descriptor<bool> arg_restricted_rpc;
@@ -73,6 +72,8 @@ namespace cryptonote
     static const command_line::arg_descriptor<uint64_t> arg_rpc_payment_difficulty;
     static const command_line::arg_descriptor<uint64_t> arg_rpc_payment_credits;
     static const command_line::arg_descriptor<bool> arg_rpc_payment_allow_free_loopback;
+    static const command_line::arg_descriptor<std::size_t> arg_rpc_max_ip_connections;
+    static const command_line::arg_descriptor<std::size_t> arg_rpc_response_soft_limit;
 
     typedef epee::net_utils::connection_context_base connection_context;
 

src/wallet/wallet_rpc_server.cpp

@@ -129,6 +129,8 @@ namespace
   const command_line::arg_descriptor<std::string> arg_wallet_dir = {"wallet-dir", "Directory for newly created wallets"};
   const command_line::arg_descriptor<bool> arg_prompt_for_password = {"prompt-for-password", "Prompts for password when not provided", false};
   const command_line::arg_descriptor<bool> arg_no_initial_sync = {"no-initial-sync", "Skips the initial sync before listening for connections", false};
+  const command_line::arg_descriptor<std::size_t> arg_rpc_max_ip_connections = {"rpc-max-ip-connections", "Max RPC connections per IP permitted", 3};
+  const command_line::arg_descriptor<std::size_t> arg_rpc_response_soft_limit = {"rpc-response-soft-limit", "Max response bytes that can be queued, enforced at next response attempt", 25 * 1024 * 1024};
 
   constexpr const char default_rpc_username[] = "monero";
 
@@ -331,7 +333,8 @@ namespace tools
       rng, std::move(bind_port), std::move(rpc_config->bind_ip),
       std::move(rpc_config->bind_ipv6_address), std::move(rpc_config->use_ipv6), std::move(rpc_config->require_ipv4),
       std::move(rpc_config->access_control_origins), std::move(http_login),
-      std::move(rpc_config->ssl_options)
+      std::move(rpc_config->ssl_options),
+      command_line::get_arg(vm, arg_rpc_max_ip_connections)
     );
   }
   //------------------------------------------------------------------------------------------------------------------------------
@@ -4974,6 +4977,7 @@ int main(int argc, char** argv) {
   command_line::add_arg(desc_params, arg_wallet_dir);
   command_line::add_arg(desc_params, arg_prompt_for_password);
   command_line::add_arg(desc_params, arg_no_initial_sync);
+  command_line::add_arg(desc_params, arg_rpc_max_ip_connections);
   command_line::add_arg(hidden_options, daemonizer::arg_non_interactive);
 
   daemonizer::init_options(hidden_options, desc_params);