Merge pull request #22322 from abyss7/issue-21907

Do not limit HTTP chunk size

src/IO/HTTPChunkedReadBuffer.cpp

@@ -14,7 +14,6 @@ namespace ErrorCodes
     extern const int ARGUMENT_OUT_OF_BOUND;
     extern const int UNEXPECTED_END_OF_FILE;
     extern const int CORRUPTED_DATA;
-    extern const int TOO_MANY_BYTES;
 }
 
 size_t HTTPChunkedReadBuffer::readChunkHeader()
@@ -40,9 +39,6 @@ size_t HTTPChunkedReadBuffer::readChunkHeader()
     if (in->eof())
         throw Exception("Unexpected end of file while reading chunk header of HTTP chunked data", ErrorCodes::UNEXPECTED_END_OF_FILE);
 
-    if (res > max_size)
-        throw Exception("Chunk size is too large", ErrorCodes::TOO_MANY_BYTES);
-
     assertString("\n", *in);
     return res;
 }

src/IO/HTTPChunkedReadBuffer.h

@@ -10,11 +10,10 @@ namespace DB
 class HTTPChunkedReadBuffer : public BufferWithOwnMemory<ReadBuffer>
 {
 public:
-    HTTPChunkedReadBuffer(std::unique_ptr<ReadBuffer> in_, size_t max_chunk_size) : in(std::move(in_)), max_size(max_chunk_size) {}
+    explicit HTTPChunkedReadBuffer(std::unique_ptr<ReadBuffer> in_) : in(std::move(in_)) {}
 
 private:
     std::unique_ptr<ReadBuffer> in;
-    const size_t max_size;
 
     size_t readChunkHeader();
     void readChunkFooter();

src/Server/HTTP/HTTPServerRequest.cpp

@@ -26,7 +26,6 @@ HTTPServerRequest::HTTPServerRequest(const Context & context, HTTPServerResponse
 
     auto receive_timeout = context.getSettingsRef().http_receive_timeout;
     auto send_timeout = context.getSettingsRef().http_send_timeout;
-    auto max_query_size = context.getSettingsRef().max_query_size;
 
     session.socket().setReceiveTimeout(receive_timeout);
     session.socket().setSendTimeout(send_timeout);
@@ -37,7 +36,7 @@ HTTPServerRequest::HTTPServerRequest(const Context & context, HTTPServerResponse
     readRequest(*in);  /// Try parse according to RFC7230
 
     if (getChunkedTransferEncoding())
-        stream = std::make_unique<HTTPChunkedReadBuffer>(std::move(in), max_query_size);
+        stream = std::make_unique<HTTPChunkedReadBuffer>(std::move(in));
     else if (hasContentLength())
         stream = std::make_unique<LimitReadBuffer>(std::move(in), getContentLength(), false);
     else if (getMethod() != HTTPRequest::HTTP_GET && getMethod() != HTTPRequest::HTTP_HEAD && getMethod() != HTTPRequest::HTTP_DELETE)

tests/queries/0_stateless/01783_http_chunk_size.reference

+1234567890 1234567890 1234567890 1234567890

tests/queries/0_stateless/01783_http_chunk_size.sh

+#!/usr/bin/env bash
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+URL="${CLICKHOUSE_URL}&session_id=id_${CLICKHOUSE_DATABASE}"
+
+echo "DROP TABLE IF EXISTS table" | ${CLICKHOUSE_CURL} -sSg "${URL}" -d @-
+echo "CREATE TABLE table (a String) ENGINE Memory()" | ${CLICKHOUSE_CURL} -sSg "${URL}" -d @-
+
+# NOTE: suppose that curl sends everything in a single chunk - there are no options to force the chunk-size.
+echo "SET max_query_size=44" | ${CLICKHOUSE_CURL} -sSg "${URL}" -d @-
+echo -ne "INSERT INTO TABLE table FORMAT TabSeparated 1234567890 1234567890 1234567890 1234567890\n" | ${CLICKHOUSE_CURL} -H "Transfer-Encoding: chunked" -sS "${URL}" --data-binary @-
+
+echo "SELECT * from table" | ${CLICKHOUSE_CURL} -sSg "${URL}" -d @-
+echo "DROP TABLE table" | ${CLICKHOUSE_CURL} -sSg "${URL}" -d @-

tests/queries/skip_list.json

@@ -164,7 +164,8 @@
         "00062_replicated_merge_tree_alter_zookeeper",
         /// Does not support renaming of multiple tables in single query
         "00634_rename_view",
-        "00140_rename"
+        "00140_rename",
+        "01783_http_chunk_size"
     ],
     "polymorphic-parts": [
         "01508_partition_pruning_long", /// bug, shoud be fixed