From c720c0d9e8db2a2911ddd797c0876d675f58ddfb Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 15 Mar 2021 01:36:20 +0100 Subject: [PATCH] Stream::send() (#6979) --- cores/esp8266/Client.h | 18 +- cores/esp8266/FS.cpp | 2 +- cores/esp8266/FS.h | 4 +- cores/esp8266/FSImpl.h | 2 +- cores/esp8266/HardwareSerial.h | 31 +- cores/esp8266/Print.cpp | 10 +- cores/esp8266/Print.h | 4 + cores/esp8266/Stream.cpp | 20 + cores/esp8266/Stream.h | 125 +++++- cores/esp8266/StreamDev.h | 248 ++++++++++++ cores/esp8266/StreamSend.cpp | 383 ++++++++++++++++++ cores/esp8266/StreamString.cpp | 68 ---- cores/esp8266/StreamString.h | 302 ++++++++++++-- cores/esp8266/debug.cpp | 7 + cores/esp8266/debug.h | 14 + cores/esp8266/esp_priv.h | 44 ++ cores/esp8266/spiffs_api.h | 4 +- cores/esp8266/uart.cpp | 35 ++ cores/esp8266/uart.h | 10 + doc/reference.rst | 193 +++++++++ .../src/ESP8266HTTPClient.cpp | 286 ++----------- .../ESP8266HTTPClient/src/ESP8266HTTPClient.h | 4 +- .../examples/HelloServer/HelloServer.ino | 4 +- .../src/ESP8266WebServer-impl.h | 107 +++-- .../ESP8266WebServer/src/ESP8266WebServer.h | 35 +- libraries/ESP8266WebServer/src/Parsing-impl.h | 18 +- .../examples/WiFiEcho/WiFiEcho.ino | 141 +++++++ .../examples/WiFiEcho/echo-client.py | 49 +++ .../ESP8266WiFi/src/CertStoreBearSSL.cpp | 4 +- libraries/ESP8266WiFi/src/WiFiClient.cpp | 35 +- libraries/ESP8266WiFi/src/WiFiClient.h | 20 +- .../src/WiFiClientSecureBearSSL.cpp | 16 + .../ESP8266WiFi/src/WiFiClientSecureBearSSL.h | 27 ++ .../ESP8266WiFi/src/include/ClientContext.h | 60 ++- .../ESP8266WiFi/src/include/DataSource.h | 154 ------- libraries/LittleFS/src/LittleFS.h | 2 +- libraries/Netdump/src/Netdump.h | 2 +- libraries/SDFS/src/SDFS.h | 2 +- .../examples/StreamString/StreamString.ino | 188 +++++++++ tests/device/Makefile | 2 +- .../test_sw_StreamString.ino | 25 ++ .../test_sw_WiFiServer/test_sw_WiFiServer.py | 1 + tests/host/Makefile | 2 +- tests/host/common/Arduino.cpp | 1 - tests/host/common/ClientContextSocket.cpp | 22 +- tests/host/common/MockUART.cpp | 6 + tests/host/common/include/ClientContext.h | 29 +- tests/restyle.sh | 4 +- 48 files changed, 2128 insertions(+), 642 deletions(-) create mode 100644 cores/esp8266/StreamDev.h create mode 100644 cores/esp8266/StreamSend.cpp delete mode 100644 cores/esp8266/StreamString.cpp create mode 100644 cores/esp8266/esp_priv.h create mode 100644 libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino create mode 100755 libraries/ESP8266WiFi/examples/WiFiEcho/echo-client.py delete mode 100644 libraries/ESP8266WiFi/src/include/DataSource.h create mode 100644 libraries/esp8266/examples/StreamString/StreamString.ino create mode 100644 tests/device/test_sw_StreamString/test_sw_StreamString.ino diff --git a/cores/esp8266/Client.h b/cores/esp8266/Client.h index ed0f0f80..7f8f8104 100644 --- a/cores/esp8266/Client.h +++ b/cores/esp8266/Client.h @@ -26,15 +26,15 @@ class Client: public Stream { public: - virtual int connect(IPAddress ip, uint16_t port) =0; - virtual int connect(const char *host, uint16_t port) =0; - virtual size_t write(uint8_t) =0; - virtual size_t write(const uint8_t *buf, size_t size) =0; - virtual int available() = 0; - virtual int read() = 0; - virtual int read(uint8_t *buf, size_t size) = 0; - virtual int peek() = 0; - virtual void flush() = 0; + virtual int connect(IPAddress ip, uint16_t port) = 0; + virtual int connect(const char *host, uint16_t port) = 0; + virtual size_t write(uint8_t) override = 0; + virtual size_t write(const uint8_t *buf, size_t size) override = 0; + virtual int available() override = 0; + virtual int read() override = 0; + virtual int read(uint8_t *buf, size_t size) override = 0; + virtual int peek() override = 0; + virtual void flush() override = 0; virtual void stop() = 0; virtual uint8_t connected() = 0; virtual operator bool() = 0; diff --git a/cores/esp8266/FS.cpp b/cores/esp8266/FS.cpp index 9d87d037..d9e4209c 100644 --- a/cores/esp8266/FS.cpp +++ b/cores/esp8266/FS.cpp @@ -66,7 +66,7 @@ int File::read() { return result; } -size_t File::read(uint8_t* buf, size_t size) { +int File::read(uint8_t* buf, size_t size) { if (!_p) return 0; diff --git a/cores/esp8266/FS.h b/cores/esp8266/FS.h index 5f420ec4..f8d68d56 100644 --- a/cores/esp8266/FS.h +++ b/cores/esp8266/FS.h @@ -67,13 +67,14 @@ public: size_t readBytes(char *buffer, size_t length) override { return read((uint8_t*)buffer, length); } - size_t read(uint8_t* buf, size_t size); + int read(uint8_t* buf, size_t size) override; bool seek(uint32_t pos, SeekMode mode); bool seek(uint32_t pos) { return seek(pos, SeekSet); } size_t position() const; size_t size() const; + virtual ssize_t streamRemaining() override { return (ssize_t)size() - (ssize_t)position(); } void close(); operator bool() const; const char* name() const; @@ -84,6 +85,7 @@ public: bool isDirectory() const; // Arduino "class SD" methods for compatibility + //TODO use stream::send / check read(buf,size) result template size_t write(T &src){ uint8_t obuf[256]; size_t doneLen = 0; diff --git a/cores/esp8266/FSImpl.h b/cores/esp8266/FSImpl.h index fdc857bc..22a058f7 100644 --- a/cores/esp8266/FSImpl.h +++ b/cores/esp8266/FSImpl.h @@ -30,7 +30,7 @@ class FileImpl { public: virtual ~FileImpl() { } virtual size_t write(const uint8_t *buf, size_t size) = 0; - virtual size_t read(uint8_t* buf, size_t size) = 0; + virtual int read(uint8_t* buf, size_t size) = 0; virtual void flush() = 0; virtual bool seek(uint32_t pos, SeekMode mode) = 0; virtual size_t position() const = 0; diff --git a/cores/esp8266/HardwareSerial.h b/cores/esp8266/HardwareSerial.h index 940bde17..b8747f09 100644 --- a/cores/esp8266/HardwareSerial.h +++ b/cores/esp8266/HardwareSerial.h @@ -135,16 +135,45 @@ public: // return -1 when data is unvailable (arduino api) return uart_peek_char(_uart); } + + virtual bool hasPeekBufferAPI () const override + { + return true; + } + + // return a pointer to available data buffer (size = available()) + // semantic forbids any kind of read() before calling peekConsume() + const char* peekBuffer () override + { + return uart_peek_buffer(_uart); + } + + // return number of byte accessible by peekBuffer() + size_t peekAvailable () override + { + return uart_peek_available(_uart); + } + + // consume bytes after use (see peekBuffer) + void peekConsume (size_t consume) override + { + return uart_peek_consume(_uart, consume); + } + int read(void) override { // return -1 when data is unvailable (arduino api) return uart_read_char(_uart); } // ::read(buffer, size): same as readBytes without timeout - size_t read(char* buffer, size_t size) + int read(char* buffer, size_t size) { return uart_read(_uart, buffer, size); } + int read(uint8_t* buffer, size_t size) override + { + return uart_read(_uart, (char*)buffer, size); + } size_t readBytes(char* buffer, size_t size) override; size_t readBytes(uint8_t* buffer, size_t size) override { diff --git a/cores/esp8266/Print.cpp b/cores/esp8266/Print.cpp index 186d6e1c..c2075fa7 100644 --- a/cores/esp8266/Print.cpp +++ b/cores/esp8266/Print.cpp @@ -33,15 +33,7 @@ /* default implementation: may be overridden */ size_t Print::write(const uint8_t *buffer, size_t size) { - -#ifdef DEBUG_ESP_CORE - static char not_the_best_way [] PROGMEM STORE_ATTR = "Print::write(data,len) should be overridden for better efficiency\r\n"; - static bool once = false; - if (!once) { - once = true; - os_printf_plus(not_the_best_way); - } -#endif + IAMSLOW(); size_t n = 0; while (size--) { diff --git a/cores/esp8266/Print.h b/cores/esp8266/Print.h index 6b2faed6..746ebc1e 100644 --- a/cores/esp8266/Print.h +++ b/cores/esp8266/Print.h @@ -111,6 +111,10 @@ class Print { size_t println(void); virtual void flush() { /* Empty implementation for backward compatibility */ } + + // by default write timeout is possible (outgoing data from network,serial..) + // (children can override to false (like String)) + virtual bool outputCanTimeout () { return true; } }; template<> size_t Print::printNumber(double number, uint8_t digits); diff --git a/cores/esp8266/Stream.cpp b/cores/esp8266/Stream.cpp index c4d5fc87..0bb2113f 100644 --- a/cores/esp8266/Stream.cpp +++ b/cores/esp8266/Stream.cpp @@ -22,6 +22,7 @@ #include #include + #define PARSE_TIMEOUT 1000 // default number of milli-seconds to wait #define NO_SKIP_CHAR 1 // a magic char not found in a valid ASCII numeric field @@ -210,6 +211,8 @@ float Stream::parseFloat(char skipChar) { // the buffer is NOT null terminated. // size_t Stream::readBytes(char *buffer, size_t length) { + IAMSLOW(); + size_t count = 0; while(count < length) { int c = timedRead(); @@ -258,3 +261,20 @@ String Stream::readStringUntil(char terminator) { } return ret; } + +// read what can be read, immediate exit on unavailable data +// prototype similar to Arduino's `int Client::read(buf, len)` +int Stream::read (uint8_t* buffer, size_t maxLen) +{ + IAMSLOW(); + + size_t nbread = 0; + while (nbread < maxLen && available()) + { + int c = read(); + if (c == -1) + break; + buffer[nbread++] = read(); + } + return nbread; +} diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index 6dcb508d..340d4f9c 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -22,10 +22,13 @@ #ifndef Stream_h #define Stream_h +#include #include -#include "Print.h" +#include +#include +#include // ssize_t -// compatability macros for testing +// compatibility macros for testing /* #define getInt() parseInt() #define getInt(skipChar) parseInt(skipchar) @@ -35,6 +38,15 @@ readBytesBetween( pre_string, terminator, buffer, length) */ +// Arduino `Client: public Stream` class defines `virtual int read(uint8_t *buf, size_t size) = 0;` +// This function is now imported into `Stream::` for `Stream::send*()`. +// Other classes inheriting from `Stream::` and implementing `read(uint8_t *buf, size_t size)` +// must consequently use `int` as return type, namely Hardware/SoftwareSerial, FileSystems... +#define STREAM_READ_RETURNS_INT 1 + +// Stream::send API is present +#define STREAMSEND_API 1 + class Stream: public Print { protected: unsigned long _timeout = 1000; // number of milliseconds to wait for the next char before aborting timed read @@ -53,6 +65,7 @@ class Stream: public Print { // parsing methods void setTimeout(unsigned long timeout); // sets maximum milliseconds to wait for stream data, default is 1 second + unsigned long getTimeout () const { return _timeout; } bool find(const char *target); // reads data from the stream until the target string is found bool find(uint8_t *target) { @@ -102,12 +115,114 @@ class Stream: public Print { virtual String readString(); String readStringUntil(char terminator); + virtual int read (uint8_t* buffer, size_t len); + int read (char* buffer, size_t len) { return read((uint8_t*)buffer, len); } + + //////////////////// extension: direct access to input buffer + // to provide when possible a pointer to available data for read + + // informs user and ::to*() on effective buffered peek API implementation + // by default: not available + virtual bool hasPeekBufferAPI () const { return false; } + + // returns number of byte accessible by peekBuffer() + virtual size_t peekAvailable () { return 0; } + + // returns a pointer to available data buffer (size = peekAvailable()) + // semantic forbids any kind of ::read() + // - after calling peekBuffer() + // - and before calling peekConsume() + virtual const char* peekBuffer () { return nullptr; } + + // consumes bytes after peekBuffer() use + // (then ::read() is allowed) + virtual void peekConsume (size_t consume) { (void)consume; } + + // by default read timeout is possible (incoming data from network,serial..) + // children can override to false (like String::) + virtual bool inputCanTimeout () { return true; } + + // (outputCanTimeout() is defined in Print::) + + //////////////////////// + //////////////////// extensions: Streaming streams to streams + // Stream::send*() + // + // Stream::send*() uses 1-copy transfers when peekBuffer API is + // available, or makes a regular transfer through a temporary buffer. + // + // - for efficiency, Stream classes should implement peekAPI when + // possible + // - for an efficient timeout management, Print/Stream classes + // should implement {output,input}CanTimeout() + + using oneShotMs = esp8266::polledTimeout::oneShotFastMs; + static constexpr int temporaryStackBufferSize = 64; + + // ::send*() methods: + // - always stop before timeout when "no-more-input-possible-data" + // or "no-more-output-possible-data" condition is met + // - always return number of transfered bytes + // When result is 0 or less than requested maxLen, Print::getLastSend() + // contains an error reason. + + // transfers already buffered / immediately available data (no timeout) + // returns number of transfered bytes + size_t sendAvailable (Print* to) { return sendGeneric(to, -1, -1, oneShotMs::alwaysExpired); } + size_t sendAvailable (Print& to) { return sendAvailable(&to); } + + // transfers data until timeout + // returns number of transfered bytes + size_t sendAll (Print* to, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return sendGeneric(to, -1, -1, timeoutMs); } + size_t sendAll (Print& to, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return sendAll(&to, timeoutMs); } + + // transfers data until a char is encountered (the char is swallowed but not transfered) with timeout + // returns number of transfered bytes + size_t sendUntil (Print* to, const int readUntilChar, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return sendGeneric(to, -1, readUntilChar, timeoutMs); } + size_t sendUntil (Print& to, const int readUntilChar, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return sendUntil(&to, readUntilChar, timeoutMs); } + + // transfers data until requested size or timeout + // returns number of transfered bytes + size_t sendSize (Print* to, const ssize_t maxLen, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return sendGeneric(to, maxLen, -1, timeoutMs); } + size_t sendSize (Print& to, const ssize_t maxLen, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return sendSize(&to, maxLen, timeoutMs); } + + // remaining size (-1 by default = unknown) + virtual ssize_t streamRemaining () { return -1; } + + enum class Report + { + Success = 0, + TimedOut, + ReadError, + WriteError, + ShortOperation, + }; + + Report getLastSendReport () const { return _sendReport; } + + protected: + size_t sendGeneric (Print* to, + const ssize_t len = -1, + const int readUntilChar = -1, + oneShotMs::timeType timeoutMs = oneShotMs::neverExpires /* neverExpires=>getTimeout() */); + + size_t SendGenericPeekBuffer(Print* to, const ssize_t len, const int readUntilChar, const oneShotMs::timeType timeoutMs); + size_t SendGenericRegularUntil(Print* to, const ssize_t len, const int readUntilChar, const oneShotMs::timeType timeoutMs); + size_t SendGenericRegular(Print* to, const ssize_t len, const oneShotMs::timeType timeoutMs); + + void setReport (Report report) { _sendReport = report; } + + private: + + Report _sendReport = Report::Success; + + //////////////////// end of extensions + protected: - long parseInt(char skipChar); // as above but the given skipChar is ignored - // as above but the given skipChar is ignored + long parseInt(char skipChar); // as parseInt() but the given skipChar is ignored // this allows format characters (typically commas) in values to be ignored - float parseFloat(char skipChar); // as above but the given skipChar is ignored + float parseFloat(char skipChar); // as parseFloat() but the given skipChar is ignored }; #endif diff --git a/cores/esp8266/StreamDev.h b/cores/esp8266/StreamDev.h new file mode 100644 index 00000000..1112ce73 --- /dev/null +++ b/cores/esp8266/StreamDev.h @@ -0,0 +1,248 @@ +/* + StreamDev.h - Stream helpers + Copyright (c) 2019 David Gauchard. All right reserved. + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef __STREAMDEV_H +#define __STREAMDEV_H + +#include +#include +#include + +/////////////////////////////////////////////// +// /dev/null +// - black hole as output, swallow everything, availableForWrite = infinite +// - black hole as input, nothing to read, available = 0 + +class StreamNull: public Stream +{ +public: + + // Print + virtual size_t write(uint8_t) override + { + return 1; + } + + virtual size_t write(const uint8_t* buffer, size_t size) override + { + (void)buffer; + (void)size; + return size; + } + + virtual int availableForWrite() override + { + return std::numeric_limits::max(); + } + + // Stream + virtual int available() override + { + return 0; + } + + virtual int read() override + { + return -1; + } + + virtual int peek() override + { + return -1; + } + + virtual size_t readBytes(char* buffer, size_t len) override + { + (void)buffer; + (void)len; + return 0; + } + + virtual int read(uint8_t* buffer, size_t len) override + { + (void)buffer; + (void)len; + return 0; + } + + virtual bool outputCanTimeout() override + { + return false; + } + + virtual bool inputCanTimeout() override + { + return false; + } + + virtual ssize_t streamRemaining() override + { + return 0; + } +}; + +/////////////////////////////////////////////// +// /dev/zero +// - black hole as output, swallow everything, availableForWrite = infinite +// - big bang as input, gives infinity to read, available = infinite + +class StreamZero: public StreamNull +{ +protected: + + char _zero; + +public: + + StreamZero(char zero = 0): _zero(zero) { } + + // Stream + virtual int available() override + { + return std::numeric_limits::max(); + } + + virtual int read() override + { + return _zero; + } + + virtual int peek() override + { + return _zero; + } + + virtual size_t readBytes(char* buffer, size_t len) override + { + memset(buffer, _zero, len); + return len; + } + + virtual int read(uint8_t* buffer, size_t len) override + { + memset((char*)buffer, _zero, len); + return len; + } + + virtual ssize_t streamRemaining() override + { + return std::numeric_limits::max(); + } +}; + +/////////////////////////////////////////////// +// static buffer (in flash or ram) +// - black hole as output, swallow everything, availableForWrite = infinite +// - Stream buffer out as input, resettable + +class StreamConstPtr: public StreamNull +{ +protected: + const char* _buffer; + size_t _size; + bool _byteAddressable; + size_t _peekPointer = 0; + +public: + StreamConstPtr(const String& string): _buffer(string.c_str()), _size(string.length()), _byteAddressable(true) { } + StreamConstPtr(const char* buffer, size_t size): _buffer(buffer), _size(size), _byteAddressable(__byteAddressable(buffer)) { } + StreamConstPtr(const uint8_t* buffer, size_t size): _buffer((const char*)buffer), _size(size), _byteAddressable(__byteAddressable(buffer)) { } + StreamConstPtr(const __FlashStringHelper* buffer, size_t size): _buffer(reinterpret_cast(buffer)), _size(size), _byteAddressable(false) { } + StreamConstPtr(const __FlashStringHelper* text): _buffer(reinterpret_cast(text)), _size(strlen_P((PGM_P)text)), _byteAddressable(false) { } + + void resetPointer(int pointer = 0) + { + _peekPointer = pointer; + } + + // Stream + virtual int available() override + { + return peekAvailable(); + } + + virtual int read() override + { + return _peekPointer < _size ? _buffer[_peekPointer++] : -1; + } + + virtual int peek() override + { + return _peekPointer < _size ? _buffer[_peekPointer] : -1; + } + + virtual size_t readBytes(char* buffer, size_t len) override + { + if (_peekPointer >= _size) + { + return 0; + } + size_t cpylen = std::min(_size - _peekPointer, len); + memcpy_P(buffer, _buffer + _peekPointer, cpylen); // whether byte adressible is true + _peekPointer += cpylen; + return cpylen; + } + + virtual int read(uint8_t* buffer, size_t len) override + { + return readBytes((char*)buffer, len); + } + + virtual ssize_t streamRemaining() override + { + return _size; + } + + // peekBuffer + virtual bool hasPeekBufferAPI() const override + { + return _byteAddressable; + } + + virtual size_t peekAvailable() override + { + return _peekPointer < _size ? _size - _peekPointer : 0; + } + + virtual const char* peekBuffer() override + { + return _peekPointer < _size ? _buffer + _peekPointer : nullptr; + } + + virtual void peekConsume(size_t consume) override + { + _peekPointer += consume; + } +}; + +/////////////////////////////////////////////// + +Stream& operator << (Stream& out, String& string); +Stream& operator << (Stream& out, Stream& stream); +Stream& operator << (Stream& out, StreamString& stream); +Stream& operator << (Stream& out, const char* text); +Stream& operator << (Stream& out, const __FlashStringHelper* text); + +/////////////////////////////////////////////// + +#if !defined(NO_GLOBAL_INSTANCES) && !defined(NO_GLOBAL_STREAMDEV) +extern StreamNull devnull; +#endif + +#endif // __STREAMDEV_H diff --git a/cores/esp8266/StreamSend.cpp b/cores/esp8266/StreamSend.cpp new file mode 100644 index 00000000..f743889b --- /dev/null +++ b/cores/esp8266/StreamSend.cpp @@ -0,0 +1,383 @@ +/* + StreamDev.cpp - 1-copy transfer related methods + Copyright (c) 2019 David Gauchard. All right reserved. + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + + parsing functions based on TextFinder library by Michael Margolis +*/ + + +#include +#include + +size_t Stream::sendGeneric(Print* to, + const ssize_t len, + const int readUntilChar, + const esp8266::polledTimeout::oneShotFastMs::timeType timeoutMs) +{ + setReport(Report::Success); + + if (len == 0) + { + return 0; // conveniently avoids timeout for no requested data + } + + // There are two timeouts: + // - read (network, serial, ...) + // - write (network, serial, ...) + // However + // - getTimeout() is for reading only + // - there is no getOutputTimeout() api + // So we use getTimeout() for both, + // (also when inputCanTimeout() is false) + + if (hasPeekBufferAPI()) + { + return SendGenericPeekBuffer(to, len, readUntilChar, timeoutMs); + } + + if (readUntilChar >= 0) + { + return SendGenericRegularUntil(to, len, readUntilChar, timeoutMs); + } + + return SendGenericRegular(to, len, timeoutMs); +} + + +size_t Stream::SendGenericPeekBuffer(Print* to, const ssize_t len, const int readUntilChar, const esp8266::polledTimeout::oneShotFastMs::timeType timeoutMs) +{ + // "neverExpires (default, impossible)" is translated to default timeout + esp8266::polledTimeout::oneShotFastMs timedOut(timeoutMs >= esp8266::polledTimeout::oneShotFastMs::neverExpires ? getTimeout() : timeoutMs); + // len==-1 => maxLen=0 <=> until starvation + const size_t maxLen = std::max((ssize_t)0, len); + size_t written = 0; + + while (!maxLen || written < maxLen) + { + size_t avpk = peekAvailable(); + if (avpk == 0 && !inputCanTimeout()) + { + // no more data to read, ever + break; + } + + size_t w = to->availableForWrite(); + if (w == 0 && !to->outputCanTimeout()) + { + // no more data can be written, ever + break; + } + + w = std::min(w, avpk); + if (maxLen) + { + w = std::min(w, maxLen - written); + } + if (w) + { + const char* directbuf = peekBuffer(); + bool foundChar = false; + if (readUntilChar >= 0) + { + const char* last = (const char*)memchr(directbuf, readUntilChar, w); + if (last) + { + w = std::min((size_t)(last - directbuf), w); + foundChar = true; + } + } + if (w && ((w = to->write(directbuf, w)))) + { + peekConsume(w); + written += w; + if (maxLen) + { + timedOut.reset(); + } + } + if (foundChar) + { + peekConsume(1); + break; + } + } + + if (!w && !maxLen && readUntilChar < 0) + { + // nothing has been transferred and no specific condition is requested + break; + } + + if (timedOut) + { + // either (maxLen>0) nothing has been transferred for too long + // or readUntilChar >= 0 but char is not encountered for too long + // or (maxLen=0) too much time has been spent here + break; + } + + optimistic_yield(1000); + } + + if (getLastSendReport() == Report::Success && maxLen > 0) + { + if (timeoutMs && timedOut) + { + setReport(Report::TimedOut); + } + else if ((ssize_t)written != len) + { + // This is happening when source cannot timeout (ex: a String) + // but has not enough data, or a dest has closed or cannot + // timeout but is too small (String, buffer...) + // + // Mark it as an error because user usually wants to get what is + // asked for. + setReport(Report::ShortOperation); + } + } + + return written; +} + +size_t Stream::SendGenericRegularUntil(Print* to, const ssize_t len, const int readUntilChar, const esp8266::polledTimeout::oneShotFastMs::timeType timeoutMs) +{ + // regular Stream API + // no other choice than reading byte by byte + + // "neverExpires (default, impossible)" is translated to default timeout + esp8266::polledTimeout::oneShotFastMs timedOut(timeoutMs >= esp8266::polledTimeout::oneShotFastMs::neverExpires ? getTimeout() : timeoutMs); + // len==-1 => maxLen=0 <=> until starvation + const size_t maxLen = std::max((ssize_t)0, len); + size_t written = 0; + + while (!maxLen || written < maxLen) + { + size_t avr = available(); + if (avr == 0 && !inputCanTimeout()) + { + // no more data to read, ever + break; + } + + size_t w = to->availableForWrite(); + if (w == 0 && !to->outputCanTimeout()) + { + // no more data can be written, ever + break; + } + + int c = read(); + if (c != -1) + { + if (c == readUntilChar) + { + break; + } + w = to->write(c); + if (w != 1) + { + setReport(Report::WriteError); + break; + } + written += 1; + if (maxLen) + { + timedOut.reset(); + } + } + + if (!w && !maxLen && readUntilChar < 0) + { + // nothing has been transferred and no specific condition is requested + break; + } + + if (timedOut) + { + // either (maxLen>0) nothing has been transferred for too long + // or readUntilChar >= 0 but char is not encountered for too long + // or (maxLen=0) too much time has been spent here + break; + } + + optimistic_yield(1000); + } + + if (getLastSendReport() == Report::Success && maxLen > 0) + { + if (timeoutMs && timedOut) + { + setReport(Report::TimedOut); + } + else if ((ssize_t)written != len) + { + // This is happening when source cannot timeout (ex: a String) + // but has not enough data, or a dest has closed or cannot + // timeout but is too small (String, buffer...) + // + // Mark it as an error because user usually wants to get what is + // asked for. + setReport(Report::ShortOperation); + } + } + + return written; +} + +size_t Stream::SendGenericRegular(Print* to, const ssize_t len, const esp8266::polledTimeout::oneShotFastMs::timeType timeoutMs) +{ + // regular Stream API + // use an intermediary buffer + + // "neverExpires (default, impossible)" is translated to default timeout + esp8266::polledTimeout::oneShotFastMs timedOut(timeoutMs >= esp8266::polledTimeout::oneShotFastMs::neverExpires ? getTimeout() : timeoutMs); + // len==-1 => maxLen=0 <=> until starvation + const size_t maxLen = std::max((ssize_t)0, len); + size_t written = 0; + + while (!maxLen || written < maxLen) + { + size_t avr = available(); + if (avr == 0 && !inputCanTimeout()) + { + // no more data to read, ever + break; + } + + size_t w = to->availableForWrite(); + if (w == 0 && !to->outputCanTimeout()) + // no more data can be written, ever + { + break; + } + + w = std::min(w, avr); + if (maxLen) + { + w = std::min(w, maxLen - written); + } + w = std::min(w, (decltype(w))temporaryStackBufferSize); + if (w) + { + char temp[w]; + ssize_t r = read(temp, w); + if (r < 0) + { + setReport(Report::ReadError); + break; + } + w = to->write(temp, r); + written += w; + if ((size_t)r != w) + { + setReport(Report::WriteError); + break; + } + if (maxLen && w) + { + timedOut.reset(); + } + } + + if (!w && !maxLen) + { + // nothing has been transferred and no specific condition is requested + break; + } + + if (timedOut) + { + // either (maxLen>0) nothing has been transferred for too long + // or readUntilChar >= 0 but char is not encountered for too long + // or (maxLen=0) too much time has been spent here + break; + } + + optimistic_yield(1000); + } + + if (getLastSendReport() == Report::Success && maxLen > 0) + { + if (timeoutMs && timedOut) + { + setReport(Report::TimedOut); + } + else if ((ssize_t)written != len) + { + // This is happening when source cannot timeout (ex: a String) + // but has not enough data, or a dest has closed or cannot + // timeout but is too small (String, buffer...) + // + // Mark it as an error because user usually wants to get what is + // asked for. + setReport(Report::ShortOperation); + } + } + + return written; +} + +Stream& operator << (Stream& out, String& string) +{ + StreamConstPtr(string).sendAll(out); + return out; +} + +Stream& operator << (Stream& out, StreamString& stream) +{ + stream.sendAll(out); + return out; +} + +Stream& operator << (Stream& out, Stream& stream) +{ + if (stream.streamRemaining() < 0) + { + if (stream.inputCanTimeout()) + { + // restrict with only what's buffered on input + stream.sendAvailable(out); + } + else + { + // take all what is in input + stream.sendAll(out); + } + } + else + { + stream.sendSize(out, stream.streamRemaining()); + } + return out; +} + +Stream& operator << (Stream& out, const char* text) +{ + StreamConstPtr(text).sendAll(out); + return out; +} + +Stream& operator << (Stream& out, const __FlashStringHelper* text) +{ + StreamConstPtr(text).sendAll(out); + return out; +} + +#if !defined(NO_GLOBAL_INSTANCES) && !defined(NO_GLOBAL_STREAMDEV) +StreamNull devnull; +#endif diff --git a/cores/esp8266/StreamString.cpp b/cores/esp8266/StreamString.cpp deleted file mode 100644 index 24cfe0dd..00000000 --- a/cores/esp8266/StreamString.cpp +++ /dev/null @@ -1,68 +0,0 @@ -/** - StreamString.cpp - - Copyright (c) 2015 Markus Sattler. All rights reserved. - This file is part of the esp8266 core for Arduino environment. - - This library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - This library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with this library; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - - */ - -#include -#include "StreamString.h" - -size_t StreamString::write(const uint8_t *data, size_t size) { - if(size && data) { - const unsigned int newlen = length() + size; - if(reserve(newlen + 1)) { - memcpy((void *) (wbuffer() + len()), (const void *) data, size); - setLen(newlen); - *(wbuffer() + newlen) = 0x00; // add null for string end - return size; - } - DEBUGV(":stream2string: OOM (%d->%d)\n", length(), newlen+1); - } - return 0; -} - -size_t StreamString::write(uint8_t data) { - return concat((char) data); -} - -int StreamString::available() { - return length(); -} - -int StreamString::read() { - if(length()) { - char c = charAt(0); - remove(0, 1); - return c; - - } - return -1; -} - -int StreamString::peek() { - if(length()) { - char c = charAt(0); - return c; - } - return -1; -} - -void StreamString::flush() { -} - diff --git a/cores/esp8266/StreamString.h b/cores/esp8266/StreamString.h index 2e81fa14..a868f28d 100644 --- a/cores/esp8266/StreamString.h +++ b/cores/esp8266/StreamString.h @@ -1,39 +1,293 @@ /** - StreamString.h + StreamString.h - Copyright (c) 2015 Markus Sattler. All rights reserved. - This file is part of the esp8266 core for Arduino environment. + Copyright (c) 2020 D. Gauchard. All rights reserved. + This file is part of the esp8266 core for Arduino environment. - This library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. - This library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. - You should have received a copy of the GNU Lesser General Public - License along with this library; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ -#ifndef STREAMSTRING_H_ -#define STREAMSTRING_H_ +#ifndef __STREAMSTRING_H +#define __STREAMSTRING_H +#include +#include "WString.h" -class StreamString: public Stream, public String { +/////////////////////////////////////////////////////////////// +// S2Stream points to a String and makes it a Stream +// (it is also the helper for StreamString) + +class S2Stream: public Stream +{ public: - size_t write(const uint8_t *buffer, size_t size) override; - size_t write(uint8_t data) override; - int available() override; - int read() override; - int peek() override; - void flush() override; + S2Stream(String& string, int peekPointer = -1): + string(&string), peekPointer(peekPointer) + { + } + + S2Stream(String* string, int peekPointer = -1): + string(string), peekPointer(peekPointer) + { + } + + virtual int available() override + { + return string->length(); + } + + virtual int availableForWrite() override + { + return std::numeric_limits::max(); + } + + virtual int read() override + { + if (peekPointer < 0) + { + // consume chars + if (string->length()) + { + char c = string->charAt(0); + string->remove(0, 1); + return c; + } + } + else if (peekPointer < (int)string->length()) + { + // return pointed and move pointer + return string->charAt(peekPointer++); + } + + // everything is read + return -1; + } + + virtual size_t write(uint8_t data) override + { + return string->concat((char)data); + } + + virtual int read(uint8_t* buffer, size_t len) override + { + if (peekPointer < 0) + { + // string will be consumed + size_t l = std::min(len, (size_t)string->length()); + memcpy(buffer, string->c_str(), l); + string->remove(0, l); + return l; + } + + if (peekPointer >= (int)string->length()) + { + return 0; + } + + // only the pointer is moved + size_t l = std::min(len, (size_t)(string->length() - peekPointer)); + memcpy(buffer, string->c_str() + peekPointer, l); + peekPointer += l; + return l; + } + + virtual size_t write(const uint8_t* buffer, size_t len) override + { + return string->concat((const char*)buffer, len) ? len : 0; + } + + virtual int peek() override + { + if (peekPointer < 0) + { + if (string->length()) + { + return string->charAt(0); + } + } + else if (peekPointer < (int)string->length()) + { + return string->charAt(peekPointer); + } + + return -1; + } + + virtual void flush() override + { + // nothing to do + } + + virtual bool inputCanTimeout() override + { + return false; + } + + virtual bool outputCanTimeout() override + { + return false; + } + + //// Stream's peekBufferAPI + + virtual bool hasPeekBufferAPI() const override + { + return true; + } + + virtual size_t peekAvailable() + { + if (peekPointer < 0) + { + return string->length(); + } + return string->length() - peekPointer; + } + + virtual const char* peekBuffer() override + { + if (peekPointer < 0) + { + return string->c_str(); + } + if (peekPointer < (int)string->length()) + { + return string->c_str() + peekPointer; + } + return nullptr; + } + + virtual void peekConsume(size_t consume) override + { + if (peekPointer < 0) + { + // string is really consumed + string->remove(0, consume); + } + else + { + // only the pointer is moved + peekPointer = std::min((size_t)string->length(), peekPointer + consume); + } + } + + virtual ssize_t streamRemaining() override + { + return peekPointer < 0 ? string->length() : string->length() - peekPointer; + } + + // calling setConsume() will consume bytes as the stream is read + // (enabled by default) + void setConsume() + { + peekPointer = -1; + } + + // Reading this stream will mark the string as read without consuming + // (not enabled by default) + // Calling resetPointer() resets the read state and allows rereading. + void resetPointer(int pointer = 0) + { + peekPointer = pointer; + } + +protected: + + String* string; + int peekPointer; // -1:String is consumed / >=0:resettable pointer }; -#endif /* STREAMSTRING_H_ */ +// StreamString is a S2Stream holding the String + +class StreamString: public String, public S2Stream +{ +protected: + + void resetpp() + { + if (peekPointer > 0) + { + peekPointer = 0; + } + } + +public: + + StreamString(StreamString&& bro): String(bro), S2Stream(this) { } + StreamString(const StreamString& bro): String(bro), S2Stream(this) { } + + // duplicate String contructors and operator=: + + StreamString(const char* text = nullptr): String(text), S2Stream(this) { } + StreamString(const String& string): String(string), S2Stream(this) { } + StreamString(const __FlashStringHelper *str): String(str), S2Stream(this) { } + StreamString(String&& string): String(string), S2Stream(this) { } + StreamString(StringSumHelper&& sum): String(sum), S2Stream(this) { } + + explicit StreamString(char c): String(c), S2Stream(this) { } + explicit StreamString(unsigned char c, unsigned char base = 10): String(c, base), S2Stream(this) { } + explicit StreamString(int i, unsigned char base = 10): String(i, base), S2Stream(this) { } + explicit StreamString(unsigned int i, unsigned char base = 10): String(i, base), S2Stream(this) { } + explicit StreamString(long l, unsigned char base = 10): String(l, base), S2Stream(this) { } + explicit StreamString(unsigned long l, unsigned char base = 10): String(l, base), S2Stream(this) { } + explicit StreamString(float f, unsigned char decimalPlaces = 2): String(f, decimalPlaces), S2Stream(this) { } + explicit StreamString(double d, unsigned char decimalPlaces = 2): String(d, decimalPlaces), S2Stream(this) { } + + StreamString& operator= (const StreamString& rhs) + { + String::operator=(rhs); + resetpp(); + return *this; + } + + StreamString& operator= (const String& rhs) + { + String::operator=(rhs); + resetpp(); + return *this; + } + + StreamString& operator= (const char* cstr) + { + String::operator=(cstr); + resetpp(); + return *this; + } + + StreamString& operator= (const __FlashStringHelper* str) + { + String::operator=(str); + resetpp(); + return *this; + } + + StreamString& operator= (String&& rval) + { + String::operator=(rval); + resetpp(); + return *this; + } + + StreamString& operator= (StringSumHelper&& rval) + { + String::operator=(rval); + resetpp(); + return *this; + } +}; + +#endif // __STREAMSTRING_H diff --git a/cores/esp8266/debug.cpp b/cores/esp8266/debug.cpp index 858f5e11..f26316f0 100644 --- a/cores/esp8266/debug.cpp +++ b/cores/esp8266/debug.cpp @@ -22,6 +22,13 @@ #include "debug.h" #include "osapi.h" +#ifdef DEBUG_ESP_CORE +void __iamslow(const char* what) +{ + DEBUGV("%s should be overridden for better efficiency\r\n", what); +} +#endif + IRAM_ATTR void hexdump(const void *mem, uint32_t len, uint8_t cols) { diff --git a/cores/esp8266/debug.h b/cores/esp8266/debug.h index 6fdda438..263d3e91 100644 --- a/cores/esp8266/debug.h +++ b/cores/esp8266/debug.h @@ -26,6 +26,20 @@ void __unhandled_exception(const char *str) __attribute__((noreturn)); void __panic_func(const char* file, int line, const char* func) __attribute__((noreturn)); #define panic() __panic_func(PSTR(__FILE__), __LINE__, __func__) +#ifdef DEBUG_ESP_CORE +extern void __iamslow(const char* what); +#define IAMSLOW() \ + do { \ + static bool once = false; \ + if (!once) { \ + once = true; \ + __iamslow((PGM_P)FPSTR(__FUNCTION__)); \ + } \ + } while (0) +#else +#define IAMSLOW() do { (void)0; } while (0) +#endif + #ifdef __cplusplus } #endif diff --git a/cores/esp8266/esp_priv.h b/cores/esp8266/esp_priv.h new file mode 100644 index 00000000..305197e1 --- /dev/null +++ b/cores/esp8266/esp_priv.h @@ -0,0 +1,44 @@ +/* + esp_priv.h - private esp8266 helpers + Copyright (c) 2020 esp8266/Arduino community. All right reserved. + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +*/ + + +#ifndef __ESP_PRIV +#define __ESP_PRIV + +#if defined(CORE_MOCK) + +constexpr bool __byteAddressable(const void* addr) +{ + (void)addr; + return true; +} + +#else // on hardware + +#include + +// returns true when addr can be used without "pgm_" functions or non32xfer service +constexpr bool __byteAddressable(const void* addr) +{ + return addr < (const void*)(XCHAL_DATARAM0_VADDR + XCHAL_DATARAM0_SIZE); +} + +#endif // on hardware + +#endif // __ESP_PRIV diff --git a/cores/esp8266/spiffs_api.h b/cores/esp8266/spiffs_api.h index 26da2513..96b01358 100644 --- a/cores/esp8266/spiffs_api.h +++ b/cores/esp8266/spiffs_api.h @@ -388,10 +388,10 @@ public: return result; } - size_t read(uint8_t* buf, size_t size) override + int read(uint8_t* buf, size_t size) override { CHECKFD(); - auto result = SPIFFS_read(_fs->getFs(), _fd, (void*) buf, size); + int result = SPIFFS_read(_fs->getFs(), _fd, (void*) buf, size); if (result < 0) { DEBUGV("SPIFFS_read rc=%d\r\n", result); return 0; diff --git a/cores/esp8266/uart.cpp b/cores/esp8266/uart.cpp index 736dbae2..3359ae6f 100644 --- a/cores/esp8266/uart.cpp +++ b/cores/esp8266/uart.cpp @@ -240,6 +240,41 @@ uart_peek_char(uart_t* uart) return ret; } +// return number of byte accessible by uart_peek_buffer() +size_t uart_peek_available (uart_t* uart) +{ + // path for further optimization: + // - return already copied buffer pointer (= older data) + // - or return fifo when buffer is empty but then any move from fifo to + // buffer should be blocked until peek_consume is called + + ETS_UART_INTR_DISABLE(); + uart_rx_copy_fifo_to_buffer_unsafe(uart); + auto rpos = uart->rx_buffer->rpos; + auto wpos = uart->rx_buffer->wpos; + ETS_UART_INTR_ENABLE(); + if(wpos < rpos) + return uart->rx_buffer->size - rpos; + return wpos - rpos; +} + +// return a pointer to available data buffer (size = available()) +// semantic forbids any kind of read() between peekBuffer() and peekConsume() +const char* uart_peek_buffer (uart_t* uart) +{ + return (const char*)&uart->rx_buffer->buffer[uart->rx_buffer->rpos]; +} + +// consume bytes after use (see uart_peek_buffer) +void uart_peek_consume (uart_t* uart, size_t consume) +{ + ETS_UART_INTR_DISABLE(); + uart->rx_buffer->rpos += consume; + if (uart->rx_buffer->rpos >= uart->rx_buffer->size) + uart->rx_buffer->rpos -= uart->rx_buffer->size; + ETS_UART_INTR_ENABLE(); +} + int uart_read_char(uart_t* uart) { diff --git a/cores/esp8266/uart.h b/cores/esp8266/uart.h index 4871b5e9..d792de66 100644 --- a/cores/esp8266/uart.h +++ b/cores/esp8266/uart.h @@ -147,6 +147,16 @@ int uart_get_debug(); void uart_start_detect_baudrate(int uart_nr); int uart_detect_baudrate(int uart_nr); +// return number of byte accessible by peekBuffer() +size_t uart_peek_available (uart_t* uart); + +// return a pointer to available data buffer (size = available()) +// semantic forbids any kind of read() before calling peekConsume() +const char* uart_peek_buffer (uart_t* uart); + +// consume bytes after use (see peekBuffer) +void uart_peek_consume (uart_t* uart, size_t consume); + uint8_t uart_get_bit_length(const int uart_nr); #if defined (__cplusplus) diff --git a/doc/reference.rst b/doc/reference.rst index 666ccd57..236d4ff7 100644 --- a/doc/reference.rst +++ b/doc/reference.rst @@ -326,3 +326,196 @@ C++ This assures correct behavior, including handling of all subobjects, which guarantees stability. History: `#6269 `__ `#6309 `__ `#6312 `__ + +Streams +------- + +Arduino API + + Stream is one of the core classes in the Arduino API. Wire, serial, network and + filesystems are streams, from which data are read or written. + + Making a transfer with streams is quite common, like for example the + historical WiFiSerial sketch: + + .. code:: cpp + + //check clients for data + //get data from the telnet client and push it to the UART + while (serverClient.available()) { + Serial.write(serverClient.read()); + } + + //check UART for data + if (Serial.available()) { + size_t len = Serial.available(); + uint8_t sbuf[len]; + Serial.readBytes(sbuf, len); + //push UART data to all connected telnet clients + if (serverClient && serverClient.connected()) { + serverClient.write(sbuf, len); + } + } + + One will notice that in the network to serial direction, data are transfered + byte by byte while data are available. In the other direction, a temporary + buffer is created on stack, filled with available serial data, then + transferred to network. + + The ``readBytes(buffer, length)`` method includes a timeout to ensure that + all required bytes are received. The ``write(buffer, length)`` (inherited + from ``Print::``) function is also usually blocking until the full buffer is + transmitted. Both functions return the number of transmitted bytes. + + That's the way the Stream class works and is commonly used. + + Classes derived from ``Stream::`` also usually introduce the ``read(buffer, + len)`` method, which is similar to ``readBytes(buffer, len)`` without + timeout: the returned value can be less than the requested size, so special + care must be taken with this function, introduced in the Arduino + ``Client::`` class (cf. AVR reference implementation). + This function has also been introduced in other classes + that don't derive from ``Client::``, e.g. ``HardwareSerial::``. + +Stream extensions + + Stream extensions are designed to be compatible with Arduino API, and + offer additional methods to make transfers more efficient and easier to + use. + + The serial to network transfer above can be written like this: + + .. code:: cpp + + serverClient.sendAvailable(Serial); // chunk by chunk + Serial.sendAvailable(serverClient); // chunk by chunk + + An echo service can be written like this: + + .. code:: cpp + + serverClient.sendAvailable(serverClient); // tcp echo service + + Serial.sendAvailable(Serial); // serial software loopback + + Beside reducing coding time, these methods optimize transfers by avoiding + buffer copies when possible. + + - User facing API: ``Stream::send()`` + + The goal of streams is to transfer data between producers and consumers, + like the telnet/serial example above. Four methods are provided, all of + them return the number of transmitted bytes: + + - ``Stream::sendSize(dest, size [, timeout])`` + + This method waits up to the given or default timeout to transfer + ``size`` bytes to the the ``dest`` Stream. + + - ``Stream::sendUntil(dest, delim [, timeout])`` + + This method waits up to the given or default timeout to transfer data + until the character ``delim`` is met. + Note: The delimiter is read but not transferred (like ``readBytesUntil``) + + - ``Stream::sendAvailable(dest)`` + + This method transfers all already available data to the destination. + There is no timeout and the returned value is 0 when there is nothing + to transfer or no room in the destination. + + - ``Stream::sendAll(dest [, timeout])`` + + This method waits up to the given or default timeout to transfer all + available data. It is useful when source is able to tell that no more + data will be available for this call, or when destination can tell + that it will no be able to receive anymore. + + For example, a source String will not grow during the transfer, or a + particular network connection supposed to send a fixed amount of data + before closing. ``::sendAll()`` will receive all bytes. Timeout is + useful when destination needs processing time (e.g. network or serial + input buffer full = please wait a bit). + + - String, flash strings helpers + + Two additional classes are provided. + + - ``StreamPtr::`` is designed to hold a constant buffer (in ram or flash). + + With this class, a ``Stream::`` can be made from ``const char*``, + ``F("some words in flash")`` or ``PROGMEM`` strings. This class makes + no copy, even with data in flash. For flash content, byte-by-byte + transfers is a consequence when "memcpy_P" cannot be used. Other + contents can be transferred at once when possible. + + .. code:: cpp + + StreamPtr css(F("my long css data")); // CSS data not copied to RAM + server.sendAll(css); + + - ``S2Stream::`` is designed to make a ``Stream::`` out of a ``String::`` without copy. + + .. code:: cpp + + String helloString("hello"); + S2Stream hello(helloString); + hello.reset(0); // prevents ::read() to consume the string + + hello.sendAll(Serial); // shows "hello" + hello.sendAll(Serial); // shows nothing, content has already been read + hello.reset(); // reset content pointer + hello.sendAll(Serial); // shows "hello" + hello.reset(3); // reset content pointer to a specific position + hello.sendAll(Serial); // shows "lo" + + hello.setConsume(); // ::read() will consume, this is the default + Serial.println(helloString.length()); // shows 5 + hello.sendAll(Serial); // shows "hello" + Serial.println(helloString.length()); // shows 0, string is consumed + + ``StreamString::`` derives from ``S2Stream`` + + .. code:: cpp + + StreamString contentStream; + client.sendSize(contentStream, SOME_SIZE); // receives at most SOME_SIZE bytes + + // equivalent to: + + String content; + S2Stream contentStream(content); + client.sendSize(contentStream, SOME_SIZE); // receives at most SOME_SIZE bytes + // content has the data + + - Internal Stream API: ``peekBuffer`` + + Here is the method list and their significations. They are currently + implemented in ``HardwareSerial``, ``WiFiClient`` and + ``WiFiClientSecure``. + + - ``virtual bool hasPeekBufferAPI ()`` returns ``true`` when the API is present in the class + + - ``virtual size_t peekAvailable ()`` returns the number of reachable bytes + + - ``virtual const char* peekBuffer ()`` returns the pointer to these bytes + + This API requires that any kind of ``"read"`` function must not be called after ``peekBuffer()`` + and until ``peekConsume()`` is called. + + - ``virtual void peekConsume (size_t consume)`` tells to discard that number of bytes + + - ``virtual bool inputCanTimeout ()`` + + A ``StringStream`` will return false. A closed network connection returns false. + This function allows ``Stream::sendAll()`` to return earlier. + + - ``virtual bool outputCanTimeout ()`` + + A closed network connection returns false. + This function allows ``Stream::sendAll()`` to return earlier. + + - ``virtual ssize_t streamRemaining()`` + + It returns -1 when stream remaining size is unknown, depending on implementation + (string size, file size..). diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index e4162128..592a14d0 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -25,9 +25,22 @@ #include "ESP8266HTTPClient.h" #include -#include +#include #include +static int StreamReportToHttpClientReport (Stream::Report streamSendError) +{ + switch (streamSendError) + { + case Stream::Report::TimedOut: return HTTPC_ERROR_READ_TIMEOUT; + case Stream::Report::ReadError: return HTTPC_ERROR_NO_STREAM; + case Stream::Report::WriteError: return HTTPC_ERROR_STREAM_WRITE; + case Stream::Report::ShortOperation: return HTTPC_ERROR_STREAM_WRITE; + case Stream::Report::Success: return 0; + } + return 0; // never reached, keep gcc quiet +} + /** * constructor */ @@ -429,24 +442,9 @@ int HTTPClient::sendRequest(const char * type, const uint8_t * payload, size_t s return returnError(HTTPC_ERROR_SEND_HEADER_FAILED); } - // send Payload if needed - if (payload && size > 0) { - size_t bytesWritten = 0; - const uint8_t *p = payload; - size_t originalSize = size; - while (bytesWritten < originalSize) { - int written; - int towrite = std::min((int)size, (int)HTTP_TCP_BUFFER_SIZE); - written = _client->write(p + bytesWritten, towrite); - if (written < 0) { - return returnError(HTTPC_ERROR_SEND_PAYLOAD_FAILED); - } else if (written == 0) { - return returnError(HTTPC_ERROR_CONNECTION_LOST); - } - bytesWritten += written; - size -= written; - } - } + // transfer all of it, with send-timeout + if (size && StreamConstPtr(payload, size).sendAll(_client) != size) + return returnError(HTTPC_ERROR_SEND_PAYLOAD_FAILED); // handle Server Response (Header) code = handleHeaderResponse(); @@ -545,111 +543,12 @@ int HTTPClient::sendRequest(const char * type, Stream * stream, size_t size) return returnError(HTTPC_ERROR_SEND_HEADER_FAILED); } - int buff_size = HTTP_TCP_BUFFER_SIZE; - - int len = size; - int bytesWritten = 0; - - if(len == 0) { - len = -1; - } - - // if possible create smaller buffer then HTTP_TCP_BUFFER_SIZE - if((len > 0) && (len < HTTP_TCP_BUFFER_SIZE)) { - buff_size = len; - } - - // create buffer for read - uint8_t * buff = (uint8_t *) malloc(buff_size); - - if(buff) { - // read all data from stream and send it to server - while(connected() && (stream->available() > 0) && (len > 0 || len == -1)) { - - // get available data size - int sizeAvailable = stream->available(); - - if(sizeAvailable) { - - int readBytes = sizeAvailable; - - // read only the asked bytes - if(len > 0 && readBytes > len) { - readBytes = len; - } - - // not read more the buffer can handle - if(readBytes > buff_size) { - readBytes = buff_size; - } - - // read data - int bytesRead = stream->readBytes(buff, readBytes); - - // write it to Stream - int bytesWrite = _client->write((const uint8_t *) buff, bytesRead); - bytesWritten += bytesWrite; - - // are all Bytes a writen to stream ? - if(bytesWrite != bytesRead) { - DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] short write, asked for %d but got %d retry...\n", bytesRead, bytesWrite); - - // check for write error - if(_client->getWriteError()) { - DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] stream write error %d\n", _client->getWriteError()); - - //reset write error for retry - _client->clearWriteError(); - } - - // some time for the stream - delay(1); - - int leftBytes = (readBytes - bytesWrite); - - // retry to send the missed bytes - bytesWrite = _client->write((const uint8_t *) (buff + bytesWrite), leftBytes); - bytesWritten += bytesWrite; - - if(bytesWrite != leftBytes) { - // failed again - DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] short write, asked for %d but got %d failed.\n", leftBytes, bytesWrite); - free(buff); - return returnError(HTTPC_ERROR_SEND_PAYLOAD_FAILED); - } - } - - // check for write error - if(_client->getWriteError()) { - DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] stream write error %d\n", _client->getWriteError()); - free(buff); - return returnError(HTTPC_ERROR_SEND_PAYLOAD_FAILED); - } - - // count bytes to read left - if(len > 0) { - len -= readBytes; - } - - delay(0); - } else { - delay(1); - } - } - - free(buff); - - if(size && (int) size != bytesWritten) { - DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] Stream payload bytesWritten %d and size %zd mismatch!.\n", bytesWritten, size); - DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] ERROR SEND PAYLOAD FAILED!"); - return returnError(HTTPC_ERROR_SEND_PAYLOAD_FAILED); - } else { - DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] Stream payload written: %d\n", bytesWritten); - } - - } else { - DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] not enough ram! need %d\n", HTTP_TCP_BUFFER_SIZE); - return returnError(HTTPC_ERROR_TOO_LESS_RAM); + // transfer all of it, with timeout + size_t transferred = stream->sendSize(_client, size); + if (transferred != size) + { + DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] short write, asked for %d but got %d failed.\n", size, transferred); + return returnError(HTTPC_ERROR_SEND_PAYLOAD_FAILED); } // handle Server Response (Header) @@ -725,13 +624,13 @@ int HTTPClient::writeToStream(Stream * stream) int ret = 0; if(_transferEncoding == HTTPC_TE_IDENTITY) { - if(len > 0 || len == -1) { - ret = writeToStreamDataBlock(stream, len); + // len < 0: transfer all of it, with timeout + // len >= 0: max:len, with timeout + ret = _client->sendSize(stream, len); - // have we an error? - if(ret < 0) { - return returnError(ret); - } + // do we have an error? + if(_client->getLastSendReport() != Stream::Report::Success) { + return returnError(StreamReportToHttpClientReport(_client->getLastSendReport())); } } else if(_transferEncoding == HTTPC_TE_CHUNKED) { int size = 0; @@ -754,11 +653,11 @@ int HTTPClient::writeToStream(Stream * stream) // data left? if(len > 0) { - int r = writeToStreamDataBlock(stream, len); - if(r < 0) { - // error in writeToStreamDataBlock - return returnError(r); - } + // read len bytes with timeout + int r = _client->sendSize(stream, len); + if (_client->getLastSendReport() != Stream::Report::Success) + // not all data transferred + return returnError(StreamReportToHttpClientReport(_client->getLastSendReport())); ret += r; } else { @@ -948,9 +847,7 @@ bool HTTPClient::connect(void) { if(_reuse && _canReuse && connected()) { DEBUG_HTTPCLIENT("[HTTP-Client] connect: already connected, reusing connection\n"); - while(_client->available() > 0) { - _client->read(); - } + _client->sendAvailable(devnull); // clear _client's output (all of it, no timeout) return true; } @@ -1032,7 +929,8 @@ bool HTTPClient::sendHeader(const char * type) DEBUG_HTTPCLIENT("[HTTP-Client] sending request header\n-----\n%s-----\n", header.c_str()); - return (_client->write((const uint8_t *) header.c_str(), header.length()) == header.length()); + // transfer all of it, with timeout + return StreamConstPtr(header).sendAll(_client) == header.length(); } /** @@ -1150,116 +1048,6 @@ int HTTPClient::handleHeaderResponse() return HTTPC_ERROR_CONNECTION_LOST; } -/** - * write one Data Block to Stream - * @param stream Stream * - * @param size int - * @return < 0 = error >= 0 = size written - */ -int HTTPClient::writeToStreamDataBlock(Stream * stream, int size) -{ - int buff_size = HTTP_TCP_BUFFER_SIZE; - int len = size; // left size to read - int bytesWritten = 0; - - // if possible create smaller buffer then HTTP_TCP_BUFFER_SIZE - if((len > 0) && (len < HTTP_TCP_BUFFER_SIZE)) { - buff_size = len; - } - - // create buffer for read - uint8_t * buff = (uint8_t *) malloc(buff_size); - - if(!buff) { - DEBUG_HTTPCLIENT("[HTTP-Client][writeToStreamDataBlock] not enough ram! need %d\n", HTTP_TCP_BUFFER_SIZE); - return HTTPC_ERROR_TOO_LESS_RAM; - } - - // read all data from server - while(connected() && (len > 0 || len == -1)) - { - int readBytes = len; - - // not read more the buffer can handle - if(readBytes > buff_size) { - readBytes = buff_size; - } - - // len == -1 or len > what is available, read only what is available - int av = _client->available(); - if (readBytes < 0 || readBytes > av) { - readBytes = av; - } - - // read data - int bytesRead = _client->readBytes(buff, readBytes); - if (!bytesRead) - { - DEBUG_HTTPCLIENT("[HTTP-Client][writeToStreamDataBlock] input stream timeout\n"); - free(buff); - return HTTPC_ERROR_READ_TIMEOUT; - } - - // write it to Stream - int bytesWrite = stream->write(buff, bytesRead); - bytesWritten += bytesWrite; - - // are all Bytes a writen to stream ? - if(bytesWrite != bytesRead) { - DEBUG_HTTPCLIENT("[HTTP-Client][writeToStream] short write asked for %d but got %d retry...\n", bytesRead, bytesWrite); - - // check for write error - if(stream->getWriteError()) { - DEBUG_HTTPCLIENT("[HTTP-Client][writeToStreamDataBlock] stream write error %d\n", stream->getWriteError()); - - //reset write error for retry - stream->clearWriteError(); - } - - // some time for the stream - delay(1); - - int leftBytes = (bytesRead - bytesWrite); - - // retry to send the missed bytes - bytesWrite = stream->write((buff + bytesWrite), leftBytes); - bytesWritten += bytesWrite; - - if(bytesWrite != leftBytes) { - // failed again - DEBUG_HTTPCLIENT("[HTTP-Client][writeToStream] short write asked for %d but got %d failed.\n", leftBytes, bytesWrite); - free(buff); - return HTTPC_ERROR_STREAM_WRITE; - } - } - - // check for write error - if(stream->getWriteError()) { - DEBUG_HTTPCLIENT("[HTTP-Client][writeToStreamDataBlock] stream write error %d\n", stream->getWriteError()); - free(buff); - return HTTPC_ERROR_STREAM_WRITE; - } - - // count bytes to read left - if(len > 0) { - len -= bytesRead; - } - - delay(0); - } - - free(buff); - - DEBUG_HTTPCLIENT("[HTTP-Client][writeToStreamDataBlock] end of chunk or data (transferred: %d).\n", bytesWritten); - - if((size > 0) && (size != bytesWritten)) { - DEBUG_HTTPCLIENT("[HTTP-Client][writeToStreamDataBlock] transferred size %d and request size %d mismatch!.\n", bytesWritten, size); - return HTTPC_ERROR_STREAM_WRITE; - } - - return bytesWritten; -} - /** * called to handle error return, may disconnect the connection if still exists * @param error diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h index 9699b92b..44699488 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h @@ -28,7 +28,7 @@ #include #include - +#include #include #ifdef DEBUG_ESP_HTTP_CLIENT @@ -148,8 +148,6 @@ typedef enum { class TransportTraits; typedef std::unique_ptr TransportTraitsPtr; -class StreamString; - class HTTPClient { public: diff --git a/libraries/ESP8266WebServer/examples/HelloServer/HelloServer.ino b/libraries/ESP8266WebServer/examples/HelloServer/HelloServer.ino index 6715f9ee..7fdee166 100644 --- a/libraries/ESP8266WebServer/examples/HelloServer/HelloServer.ino +++ b/libraries/ESP8266WebServer/examples/HelloServer/HelloServer.ino @@ -115,9 +115,9 @@ void setup(void) { // swallow the exact amount matching the full request+content, // hence the tcp connection cannot be handled anymore by the // webserver. -#ifdef STREAMTO_API +#ifdef STREAMSEND_API // we are lucky - client->toWithTimeout(Serial, 500); + client->sendAll(Serial, 500); #else auto last = millis(); while ((millis() - last) < 500) { diff --git a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h index 9ea77718..c48aa019 100644 --- a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h +++ b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h @@ -28,6 +28,7 @@ #include "FS.h" #include "base64.h" #include "detail/RequestHandlersImpl.h" +#include static const char AUTHORIZATION_HEADER[] PROGMEM = "Authorization"; static const char qop_auth[] PROGMEM = "qop=auth"; @@ -441,71 +442,68 @@ void ESP8266WebServerTemplate::_prepareHeader(String& response, int } template -void ESP8266WebServerTemplate::send(int code, const char* content_type, const String& content) { - String header; - // Can we asume the following? - //if(code == 200 && content.length() == 0 && _contentLength == CONTENT_LENGTH_NOT_SET) - // _contentLength = CONTENT_LENGTH_UNKNOWN; - _prepareHeader(header, code, content_type, content.length()); - _currentClient.write((const uint8_t *)header.c_str(), header.length()); - if(content.length()) - sendContent(content); +void ESP8266WebServerTemplate::send(int code, char* content_type, const String& content) { + return send(code, (const char*)content_type, content); } template -void ESP8266WebServerTemplate::send_P(int code, PGM_P content_type, PGM_P content) { - size_t contentLength = 0; +void ESP8266WebServerTemplate::send(int code, const char* content_type, const String& content) { + return send(code, content_type, content.c_str(), content.length()); +} - if (content != NULL) { - contentLength = strlen_P(content); - } +template +void ESP8266WebServerTemplate::send(int code, const String& content_type, const String& content) { + return send(code, (const char*)content_type.c_str(), content); +} - String header; - char type[64]; - memccpy_P((void*)type, (PGM_VOID_P)content_type, 0, sizeof(type)); - _prepareHeader(header, code, (const char* )type, contentLength); - _currentClient.write((const uint8_t *)header.c_str(), header.length()); - if (contentLength) { - sendContent_P(content); - } +template +void ESP8266WebServerTemplate::sendContent(const String& content) { + StreamConstPtr ref(content.c_str(), content.length()); + sendContent(&ref); } template -void ESP8266WebServerTemplate::send_P(int code, PGM_P content_type, PGM_P content, size_t contentLength) { - String header; - char type[64]; - memccpy_P((void*)type, (PGM_VOID_P)content_type, 0, sizeof(type)); - _prepareHeader(header, code, (const char* )type, contentLength); - _currentClient.write((const uint8_t *)header.c_str(), header.length()); - if (contentLength) { - sendContent_P(content, contentLength); - } +void ESP8266WebServerTemplate::send(int code, const char* content_type, Stream* stream, size_t content_length /*= 0*/) { + String header; + if (content_length == 0) + content_length = std::max((ssize_t)0, stream->streamRemaining()); + _prepareHeader(header, code, content_type, content_length); + size_t sent = StreamConstPtr(header).sendAll(&_currentClient); + if (sent != header.length()) + DBGWS("HTTPServer: error: sent %zd on %u bytes\n", sent, header.length()); + if (content_length) + return sendContent(stream, content_length); } template -void ESP8266WebServerTemplate::send(int code, char* content_type, const String& content) { - send(code, (const char*)content_type, content); +void ESP8266WebServerTemplate::send_P(int code, PGM_P content_type, PGM_P content) { + StreamConstPtr ref(content, strlen_P(content)); + return send(code, String(content_type).c_str(), &ref); } template -void ESP8266WebServerTemplate::send(int code, const String& content_type, const String& content) { - send(code, (const char*)content_type.c_str(), content); +void ESP8266WebServerTemplate::send_P(int code, PGM_P content_type, PGM_P content, size_t contentLength) { + StreamConstPtr ref(content, contentLength); + return send(code, String(content_type).c_str(), &ref); } template -void ESP8266WebServerTemplate::sendContent(const String& content) { - if (_currentMethod == HTTP_HEAD) return; - const char * footer = "\r\n"; - size_t len = content.length(); +void ESP8266WebServerTemplate::sendContent(Stream* content, ssize_t content_length /* = 0*/) { + if (_currentMethod == HTTP_HEAD) + return; + if (content_length <= 0) + content_length = std::max((ssize_t)0, content->streamRemaining()); if(_chunked) { - char chunkSize[11]; - sprintf(chunkSize, "%zx\r\n", len); - _currentClient.write((const uint8_t *)chunkSize, strlen(chunkSize)); + _currentClient.printf("%zx\r\n", content_length); + } + ssize_t sent = content->sendSize(&_currentClient, content_length); + if (sent != content_length) + { + DBGWS("HTTPServer: error: short send after timeout (%d<%d)\n", sent, content_length); } - _currentClient.write((const uint8_t *)content.c_str(), len); - if(_chunked){ - _currentClient.write((const uint8_t *)footer, 2); - if (len == 0) { + if(_chunked) { + _currentClient.printf_P(PSTR("\r\n")); + if (content_length == 0) { _chunked = false; } } @@ -518,19 +516,8 @@ void ESP8266WebServerTemplate::sendContent_P(PGM_P content) { template void ESP8266WebServerTemplate::sendContent_P(PGM_P content, size_t size) { - const char * footer = "\r\n"; - if(_chunked) { - char chunkSize[11]; - sprintf(chunkSize, "%zx\r\n", size); - _currentClient.write((const uint8_t *)chunkSize, strlen(chunkSize)); - } - _currentClient.write_P(content, size); - if(_chunked){ - _currentClient.write((const uint8_t *)footer, 2); - if (size == 0) { - _chunked = false; - } - } + StreamConstPtr ptr(content, size); + return sendContent(&ptr, size); } template @@ -694,7 +681,7 @@ void ESP8266WebServerTemplate::_handleRequest() { } if (!handled) { using namespace mime; - send(404, String(FPSTR(mimeTable[html].mimeType)), String(F("Not found: ")) + _currentUri); + send(404, FPSTR(mimeTable[html].mimeType), String(F("Not found: ")) + _currentUri); handled = true; } if (handled) { diff --git a/libraries/ESP8266WebServer/src/ESP8266WebServer.h b/libraries/ESP8266WebServer/src/ESP8266WebServer.h index a8453b42..76c43cc8 100644 --- a/libraries/ESP8266WebServer/src/ESP8266WebServer.h +++ b/libraries/ESP8266WebServer/src/ESP8266WebServer.h @@ -149,7 +149,7 @@ public: // code - HTTP response code, can be 200 or 404 // content_type - HTTP content type, like "text/plain" or "image/png" // content - actual content body - void send(int code, const char* content_type = NULL, const String& content = String("")); + void send(int code, const char* content_type = NULL, const String& content = emptyString); void send(int code, char* content_type, const String& content); void send(int code, const String& content_type, const String& content); void send(int code, const char *content_type, const char *content) { @@ -164,14 +164,23 @@ public: void send_P(int code, PGM_P content_type, PGM_P content); void send_P(int code, PGM_P content_type, PGM_P content, size_t contentLength); + void send(int code, const char* content_type, Stream* stream, size_t content_length = 0); + void send(int code, const char* content_type, Stream& stream, size_t content_length = 0); + void setContentLength(const size_t contentLength); void sendHeader(const String& name, const String& value, bool first = false); void sendContent(const String& content); + void sendContent(String& content) { + sendContent((const String&)content); + } void sendContent_P(PGM_P content); void sendContent_P(PGM_P content, size_t size); void sendContent(const char *content) { sendContent_P(content); } void sendContent(const char *content, size_t size) { sendContent_P(content, size); } + void sendContent(Stream* content, ssize_t content_length = 0); + void sendContent(Stream& content, ssize_t content_length = 0) { sendContent(&content, content_length); } + bool chunkedResponseModeStart_P (int code, PGM_P content_type) { if (_currentVersion == 0) // no chunk mode in HTTP/1.0 @@ -220,6 +229,30 @@ public: return contentLength; } + // Implement GET and HEAD requests for stream + // Stream body on HTTP_GET but not on HTTP_HEAD requests. + template + size_t stream(T &aStream, const String& contentType, HTTPMethod requestMethod, ssize_t size) { + setContentLength(size); + send(200, contentType, emptyString); + if (requestMethod == HTTP_GET) + size = aStream.sendSize(_currentClient, size); + return size; + } + + // Implement GET and HEAD requests for stream + // Stream body on HTTP_GET but not on HTTP_HEAD requests. + template + size_t stream(T& aStream, const String& contentType, HTTPMethod requestMethod = HTTP_GET) { + ssize_t size = aStream.size(); + if (size < 0) + { + send(500, F("text/html"), F("input stream: undetermined size")); + return 0; + } + return stream(aStream, contentType, requestMethod, size); + } + static String responseCodeToString(const int code); void addHook (HookFunction hook) { diff --git a/libraries/ESP8266WebServer/src/Parsing-impl.h b/libraries/ESP8266WebServer/src/Parsing-impl.h index e50ea727..8e4a6d1a 100644 --- a/libraries/ESP8266WebServer/src/Parsing-impl.h +++ b/libraries/ESP8266WebServer/src/Parsing-impl.h @@ -37,22 +37,8 @@ namespace esp8266webserver { template static bool readBytesWithTimeout(typename ServerType::ClientType& client, size_t maxLength, String& data, int timeout_ms) { - if (!data.reserve(maxLength + 1)) - return false; - data[0] = 0; // data.clear()?? - while (data.length() < maxLength) { - int tries = timeout_ms; - size_t avail; - while (!(avail = client.available()) && tries--) - delay(1); - if (!avail) - break; - if (data.length() + avail > maxLength) - avail = maxLength - data.length(); - while (avail--) - data += (char)client.read(); - } - return data.length() == maxLength; + S2Stream dataStream(data); + return client.sendSize(dataStream, maxLength, timeout_ms) == maxLength; } template diff --git a/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino b/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino new file mode 100644 index 00000000..0962b083 --- /dev/null +++ b/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino @@ -0,0 +1,141 @@ +/* + WiFiEcho - Echo server + + released to public domain +*/ + +#include +#include +#include +#include // std::min + +#ifndef STASSID +#define STASSID "your-ssid" +#define STAPSK "your-password" +#endif + +constexpr int port = 23; + +WiFiServer server(port); +WiFiClient client; + +constexpr size_t sizes [] = { 0, 512, 384, 256, 128, 64, 16, 8, 4 }; +constexpr uint32_t breathMs = 200; +esp8266::polledTimeout::oneShotFastMs enoughMs(breathMs); +esp8266::polledTimeout::periodicFastMs test(2000); +int t = 1; // test (1, 2 or 3, see below) +int s = 0; // sizes[] index + +void setup() { + + Serial.begin(115200); + Serial.println(ESP.getFullVersion()); + + WiFi.mode(WIFI_STA); + WiFi.begin(STASSID, STAPSK); + Serial.print("\nConnecting to "); + Serial.println(STASSID); + while (WiFi.status() != WL_CONNECTED) { + Serial.print('.'); + delay(500); + } + Serial.println(); + Serial.print("connected, address="); + Serial.println(WiFi.localIP()); + + server.begin(); + + MDNS.begin("echo23"); + + Serial.printf("Ready!\n" + "- Use 'telnet/nc echo23.local %d' to try echo\n\n" + "- Use 'python3 echo-client.py' bandwidth meter to compare transfer APIs\n\n" + " and try typing 1, 1, 1, 2, 2, 2, 3, 3, 3 on console during transfers\n\n", + port); +} + + +void loop() { + + MDNS.update(); + + static uint32_t tot = 0; + static uint32_t cnt = 0; + if (test && cnt) { + Serial.printf("measured-block-size=%u min-free-stack=%u", tot / cnt, ESP.getFreeContStack()); + if (t == 2 && sizes[s]) { + Serial.printf(" (blocks: at most %d bytes)", sizes[s]); + } + if (t == 3 && sizes[s]) { + Serial.printf(" (blocks: exactly %d bytes)", sizes[s]); + } + if (t == 3 && !sizes[s]) { + Serial.printf(" (blocks: any size)"); + } + Serial.printf("\n"); + } + + //check if there are any new clients + if (server.hasClient()) { + client = server.available(); + Serial.println("New client"); + } + + if (Serial.available()) { + s = (s + 1) % (sizeof(sizes) / sizeof(sizes[0])); + switch (Serial.read()) { + case '1': if (t != 1) s = 0; t = 1; Serial.println("byte-by-byte (watch then press 2 or 3)"); break; + case '2': if (t != 2) s = 1; t = 2; Serial.printf("through buffer (watch then press 2 again, or 1 or 3)\n"); break; + case '3': if (t != 3) s = 0; t = 3; Serial.printf("direct access (watch then press 3 again, or 1 or 2)\n"); break; + } + tot = cnt = 0; + ESP.resetFreeContStack(); + } + + enoughMs.reset(breathMs); + + if (t == 1) { + // byte by byte + while (client.available() && client.availableForWrite() && !enoughMs) { + // working char by char is not efficient + client.write(client.read()); + cnt++; + tot += 1; + } + } + + else if (t == 2) { + // block by block through a local buffer (2 copies) + while (client.available() && client.availableForWrite() && !enoughMs) { + size_t maxTo = std::min(client.available(), client.availableForWrite()); + maxTo = std::min(maxTo, sizes[s]); + uint8_t buf[maxTo]; + size_t tcp_got = client.read(buf, maxTo); + size_t tcp_sent = client.write(buf, tcp_got); + if (tcp_sent != maxTo) { + Serial.printf("len mismatch: available:%zd tcp-read:%zd serial-write:%zd\n", maxTo, tcp_got, tcp_sent); + } + tot += tcp_sent; + cnt++; + } + } + + else if (t == 3) { + // stream to print, possibly with only one copy + if (sizes[s]) { + tot += client.sendSize(&client, sizes[s]); + } else { + tot += client.sendAll(&client); + } + cnt++; + + switch (client.getLastSendReport()) { + case Stream::Report::Success: break; + case Stream::Report::TimedOut: Serial.println("Stream::send: timeout"); break; + case Stream::Report::ReadError: Serial.println("Stream::send: read error"); break; + case Stream::Report::WriteError: Serial.println("Stream::send: write error"); break; + case Stream::Report::ShortOperation: Serial.println("Stream::send: short transfer"); break; + } + } + +} diff --git a/libraries/ESP8266WiFi/examples/WiFiEcho/echo-client.py b/libraries/ESP8266WiFi/examples/WiFiEcho/echo-client.py new file mode 100755 index 00000000..55c90073 --- /dev/null +++ b/libraries/ESP8266WiFi/examples/WiFiEcho/echo-client.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 + +import os +import asyncio + +# 512 bytes +message = bytearray(512); +bufsize=len(message) +print('message len=', bufsize) + +global recv +recv = 0 + +async def tcp_echo_open (ip, port): + return await asyncio.open_connection(ip, port) + +async def tcp_echo_sender(message, writer): + print('Writer started') + while True: + writer.write(message) + await writer.drain() + +async def tcp_echo_receiver(message, reader): + global recv + print('Reader started') + while True: + data = ''.encode('utf8') + while len(data) < bufsize: + data += await reader.read(bufsize - len(data)) + recv += len(data); + if data != message: + print('error') + +async def tcp_stat(): + global recv + dur = 0 + loopsec = 2 + while True: + last = recv + await asyncio.sleep(loopsec) # drifting + dur += loopsec + print('BW=', (recv - last) * 2 * 8 / 1024 / loopsec, 'Kibits/s avg=', recv * 2 * 8 / 1024 / dur) + +loop = asyncio.get_event_loop() +reader, writer = loop.run_until_complete(tcp_echo_open('echo23.local', 23)) +loop.create_task(tcp_echo_receiver(message, reader)) +loop.create_task(tcp_echo_sender(message, writer)) +loop.create_task(tcp_stat()) +loop.run_forever() diff --git a/libraries/ESP8266WiFi/src/CertStoreBearSSL.cpp b/libraries/ESP8266WiFi/src/CertStoreBearSSL.cpp index 8af9cb8b..b16af42a 100644 --- a/libraries/ESP8266WiFi/src/CertStoreBearSSL.cpp +++ b/libraries/ESP8266WiFi/src/CertStoreBearSSL.cpp @@ -125,7 +125,7 @@ int CertStore::initCertStore(fs::FS &fs, const char *indexFileName, const char * uint8_t fileHeader[60]; // 0..15 = filename in ASCII // 48...57 = length in decimal ASCII - uint32_t length; + int32_t length; if (data.read(fileHeader, sizeof(fileHeader)) != sizeof(fileHeader)) { break; } @@ -201,7 +201,7 @@ const br_x509_trust_anchor *CertStore::findHashedTA(void *ctx, void *hashed_dn, free(der); return nullptr; } - if (data.read((uint8_t *)der, ci.length) != ci.length) { + if (data.read(der, ci.length) != (int)ci.length) { free(der); return nullptr; } diff --git a/libraries/ESP8266WiFi/src/WiFiClient.cpp b/libraries/ESP8266WiFi/src/WiFiClient.cpp index 5aa09b88..07e4d995 100644 --- a/libraries/ESP8266WiFi/src/WiFiClient.cpp +++ b/libraries/ESP8266WiFi/src/WiFiClient.cpp @@ -244,7 +244,7 @@ size_t WiFiClient::write_P(PGM_P buf, size_t size) int WiFiClient::available() { if (!_client) - return false; + return 0; int result = _client->getSize(); @@ -262,10 +262,14 @@ int WiFiClient::read() return _client->read(); } - int WiFiClient::read(uint8_t* buf, size_t size) { - return (int) _client->read(reinterpret_cast(buf), size); + return (int)_client->read((char*)buf, size); +} + +int WiFiClient::read(char* buf, size_t size) +{ + return (int)_client->read(buf, size); } int WiFiClient::peek() @@ -412,3 +416,28 @@ uint8_t WiFiClient::getKeepAliveCount () const { return _client->getKeepAliveCount(); } + +bool WiFiClient::hasPeekBufferAPI () const +{ + return true; +} + +// return a pointer to available data buffer (size = peekAvailable()) +// semantic forbids any kind of read() before calling peekConsume() +const char* WiFiClient::peekBuffer () +{ + return _client? _client->peekBuffer(): nullptr; +} + +// return number of byte accessible by peekBuffer() +size_t WiFiClient::peekAvailable () +{ + return _client? _client->peekAvailable(): 0; +} + +// consume bytes after use (see peekBuffer) +void WiFiClient::peekConsume (size_t consume) +{ + if (_client) + _client->peekConsume(consume); +} diff --git a/libraries/ESP8266WiFi/src/WiFiClient.h b/libraries/ESP8266WiFi/src/WiFiClient.h index 41c099bf..d59ae6ca 100644 --- a/libraries/ESP8266WiFi/src/WiFiClient.h +++ b/libraries/ESP8266WiFi/src/WiFiClient.h @@ -66,7 +66,9 @@ public: virtual int available() override; virtual int read() override; - virtual int read(uint8_t *buf, size_t size) override; + virtual int read(uint8_t* buf, size_t size) override; + int read(char* buf, size_t size); + virtual int peek() override; virtual size_t peekBytes(uint8_t *buffer, size_t length); size_t peekBytes(char *buffer, size_t length) { @@ -120,6 +122,22 @@ public: bool getSync() const; void setSync(bool sync); + // peek buffer API is present + virtual bool hasPeekBufferAPI () const override; + + // return number of byte accessible by peekBuffer() + virtual size_t peekAvailable () override; + + // return a pointer to available data buffer (size = peekAvailable()) + // semantic forbids any kind of read() before calling peekConsume() + virtual const char* peekBuffer () override; + + // consume bytes after use (see peekBuffer) + virtual void peekConsume (size_t consume) override; + + virtual bool outputCanTimeout () override { return connected(); } + virtual bool inputCanTimeout () override { return connected(); } + protected: static int8_t _s_connected(void* arg, void* tpcb, int8_t err); diff --git a/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.cpp b/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.cpp index 48dc531a..9583d14b 100644 --- a/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.cpp +++ b/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.cpp @@ -362,6 +362,22 @@ int WiFiClientSecureCtx::read(uint8_t *buf, size_t size) { return 0; // If we're connected, no error but no read. } +// return a pointer to available data buffer (size = peekAvailable()) +// semantic forbids any kind of read() before calling peekConsume() +const char* WiFiClientSecureCtx::peekBuffer () +{ + return (const char*)_recvapp_buf; +} + +// consume bytes after use (see peekBuffer) +void WiFiClientSecureCtx::peekConsume (size_t consume) +{ + // according to WiFiClientSecureCtx::read: + br_ssl_engine_recvapp_ack(_eng, consume); + _recvapp_buf = nullptr; + _recvapp_len = 0; +} + int WiFiClientSecureCtx::read() { uint8_t c; if (1 == read(&c, 1)) { diff --git a/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h b/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h index 55c36607..52dd8787 100644 --- a/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h +++ b/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h @@ -48,6 +48,7 @@ class WiFiClientSecureCtx : public WiFiClient { size_t write_P(PGM_P buf, size_t size) override; size_t write(Stream& stream); // Note this is not virtual int read(uint8_t *buf, size_t size) override; + int read(char *buf, size_t size) { return read((uint8_t*)buf, size); } int available() override; int read() override; int peek() override; @@ -120,6 +121,19 @@ class WiFiClientSecureCtx : public WiFiClient { bool setCiphers(const std::vector& list); bool setCiphersLessSecure(); // Only use the limited set of RSA ciphers without EC + // peek buffer API is present + virtual bool hasPeekBufferAPI () const override { return true; } + + // return number of byte accessible by peekBuffer() + virtual size_t peekAvailable () override { return WiFiClientSecureCtx::available(); } + + // return a pointer to available data buffer (size = peekAvailable()) + // semantic forbids any kind of read() before calling peekConsume() + virtual const char* peekBuffer () override; + + // consume bytes after use (see peekBuffer) + virtual void peekConsume (size_t consume) override; + protected: bool _connectSSL(const char *hostName); // Do initial SSL handshake @@ -287,6 +301,19 @@ class WiFiClientSecure : public WiFiClient { static bool probeMaxFragmentLength(const char *hostname, uint16_t port, uint16_t len); static bool probeMaxFragmentLength(const String& host, uint16_t port, uint16_t len); + // peek buffer API is present + virtual bool hasPeekBufferAPI () const override { return true; } + + // return number of byte accessible by peekBuffer() + virtual size_t peekAvailable () override { return _ctx->available(); } + + // return a pointer to available data buffer (size = peekAvailable()) + // semantic forbids any kind of read() before calling peekConsume() + virtual const char* peekBuffer () override { return _ctx->peekBuffer(); } + + // consume bytes after use (see peekBuffer) + virtual void peekConsume (size_t consume) override { return _ctx->peekConsume(consume); } + private: std::shared_ptr _ctx; diff --git a/libraries/ESP8266WiFi/src/include/ClientContext.h b/libraries/ESP8266WiFi/src/include/ClientContext.h index 8095e402..e3c9e6c4 100644 --- a/libraries/ESP8266WiFi/src/include/ClientContext.h +++ b/libraries/ESP8266WiFi/src/include/ClientContext.h @@ -29,7 +29,8 @@ typedef void (*discard_cb_t)(void*, ClientContext*); extern "C" void esp_yield(); extern "C" void esp_schedule(); -#include "DataSource.h" +#include +#include bool getDefaultPrivateGlobalSyncValue (); @@ -374,7 +375,8 @@ public: if (!_pcb) { return 0; } - return _write_from_source(new BufferDataSource(data, size)); + StreamConstPtr ptr(data, size); + return _write_from_source(&ptr); } size_t write(Stream& stream) @@ -382,7 +384,7 @@ public: if (!_pcb) { return 0; } - return _write_from_source(new BufferedStreamDataSource(stream, stream.available())); + return _write_from_source(&stream); } size_t write_P(PGM_P buf, size_t size) @@ -390,8 +392,8 @@ public: if (!_pcb) { return 0; } - ProgmemStream stream(buf, size); - return _write_from_source(new BufferedStreamDataSource(stream, size)); + StreamConstPtr ptr(buf, size); + return _write_from_source(&ptr); } void keepAlive (uint16_t idle_sec = TCP_DEFAULT_KEEPALIVE_IDLE_SEC, uint16_t intv_sec = TCP_DEFAULT_KEEPALIVE_INTERVAL_SEC, uint8_t count = TCP_DEFAULT_KEEPALIVE_COUNT) @@ -436,6 +438,29 @@ public: _sync = sync; } + // return a pointer to available data buffer (size = peekAvailable()) + // semantic forbids any kind of read() before calling peekConsume() + const char* peekBuffer () + { + if (!_rx_buf) + return nullptr; + return (const char*)_rx_buf->payload + _rx_buf_offset; + } + + // return number of byte accessible by peekBuffer() + size_t peekAvailable () + { + if (!_rx_buf) + return 0; + return _rx_buf->len - _rx_buf_offset; + } + + // consume bytes after use (see peekBuffer) + void peekConsume (size_t consume) + { + _consume(consume); + } + protected: bool _is_timeout() @@ -452,7 +477,7 @@ protected: } } - size_t _write_from_source(DataSource* ds) + size_t _write_from_source(Stream* ds) { assert(_datasource == nullptr); assert(!_send_waiting); @@ -468,7 +493,6 @@ protected: if (_is_timeout()) { DEBUGV(":wtmo\r\n"); } - delete _datasource; _datasource = nullptr; break; } @@ -495,20 +519,20 @@ protected: return false; } - DEBUGV(":wr %d %d\r\n", _datasource->available(), _written); + DEBUGV(":wr %d %d\r\n", _datasource->peekAvailable(), _written); bool has_written = false; while (_datasource) { if (state() == CLOSED) return false; - size_t next_chunk_size = std::min((size_t)tcp_sndbuf(_pcb), _datasource->available()); + size_t next_chunk_size = std::min((size_t)tcp_sndbuf(_pcb), _datasource->peekAvailable()); if (!next_chunk_size) break; - const uint8_t* buf = _datasource->get_buffer(next_chunk_size); + const char* buf = _datasource->peekBuffer(); uint8_t flags = 0; - if (next_chunk_size < _datasource->available()) + if (next_chunk_size < _datasource->peekAvailable()) // PUSH is meant for peer, telling to give data to user app as soon as received // PUSH "may be set" when sender has finished sending a "meaningful" data block // PUSH does not break Nagle @@ -522,15 +546,15 @@ protected: err_t err = tcp_write(_pcb, buf, next_chunk_size, flags); - DEBUGV(":wrc %d %d %d\r\n", next_chunk_size, _datasource->available(), (int)err); + DEBUGV(":wrc %d %d %d\r\n", next_chunk_size, _datasource->peekAvailable(), (int)err); if (err == ERR_OK) { - _datasource->release_buffer(buf, next_chunk_size); + _datasource->peekConsume(next_chunk_size); _written += next_chunk_size; has_written = true; } else { - // ERR_MEM(-1) is a valid error meaning - // "come back later". It leaves state() opened + // ERR_MEM(-1) is a valid error meaning + // "come back later". It leaves state() opened break; } } @@ -565,8 +589,6 @@ protected: void _consume(size_t size) { - if(_pcb) - tcp_recved(_pcb, size); ptrdiff_t left = _rx_buf->len - _rx_buf_offset - size; if(left > 0) { _rx_buf_offset += size; @@ -583,6 +605,8 @@ protected: pbuf_ref(_rx_buf); pbuf_free(head); } + if(_pcb) + tcp_recved(_pcb, size); } err_t _recv(tcp_pcb* pcb, pbuf* pb, err_t err) @@ -683,7 +707,7 @@ private: discard_cb_t _discard_cb; void* _discard_cb_arg; - DataSource* _datasource = nullptr; + Stream* _datasource = nullptr; size_t _written = 0; uint32_t _timeout_ms = 5000; uint32_t _op_start_time = 0; diff --git a/libraries/ESP8266WiFi/src/include/DataSource.h b/libraries/ESP8266WiFi/src/include/DataSource.h deleted file mode 100644 index 2a0bfed2..00000000 --- a/libraries/ESP8266WiFi/src/include/DataSource.h +++ /dev/null @@ -1,154 +0,0 @@ -/* DataSource.h - a read-only object similar to Stream, but with less methods - * Copyright (c) 2016 Ivan Grokhotkov. All rights reserved. - * This file is distributed under MIT license. - */ -#ifndef DATASOURCE_H -#define DATASOURCE_H - -#include - -class DataSource { -public: - virtual ~DataSource() {} - virtual size_t available() = 0; - virtual const uint8_t* get_buffer(size_t size) = 0; - virtual void release_buffer(const uint8_t* buffer, size_t size) = 0; - -}; - -class BufferDataSource : public DataSource { -public: - BufferDataSource(const uint8_t* data, size_t size) : - _data(data), - _size(size) - { - } - - size_t available() override - { - return _size - _pos; - } - - const uint8_t* get_buffer(size_t size) override - { - (void)size; - assert(_pos + size <= _size); - return _data + _pos; - } - - void release_buffer(const uint8_t* buffer, size_t size) override - { - (void)buffer; - assert(buffer == _data + _pos); - _pos += size; - } - -protected: - const uint8_t* _data; - const size_t _size; - size_t _pos = 0; -}; - -template -class BufferedStreamDataSource : public DataSource { -public: - BufferedStreamDataSource(TStream& stream, size_t size) : - _stream(stream), - _size(size) - { - } - - size_t available() override - { - return _size - _pos; - } - - const uint8_t* get_buffer(size_t size) override - { - assert(_pos + size <= _size); - - //Data that was already read from the stream but not released (e.g. if tcp_write error occured). Otherwise this should be 0. - const size_t stream_read = _streamPos - _pos; - - //Min required buffer size: max(requested size, previous stream data already in buffer) - const size_t min_buffer_size = size > stream_read ? size : stream_read; - - //Buffer too small? - if (_bufferSize < min_buffer_size) { - uint8_t *new_buffer = new uint8_t[min_buffer_size]; - //If stream reading is ahead, than some data is already in the old buffer and needs to be copied to new resized buffer - if (_buffer && stream_read > 0) { - memcpy(new_buffer, _buffer.get(), stream_read); - } - _buffer.reset(new_buffer); - _bufferSize = min_buffer_size; - } - - //Fetch remaining data from stream - //If error in tcp_write in ClientContext::_write_some() occured earlier and therefore release_buffer was not called last time, than the requested stream data is already in the buffer. - if (size > stream_read) { - //Remaining bytes to read from stream - const size_t stream_rem = size - stream_read; - const size_t cb = _stream.readBytes(reinterpret_cast(_buffer.get() + stream_read), stream_rem); - assert(cb == stream_rem); - (void)cb; - _streamPos += stream_rem; - } - return _buffer.get(); - - } - - void release_buffer(const uint8_t* buffer, size_t size) override - { - if (size == 0) { - return; - } - - (void)buffer; - _pos += size; - - //Cannot release more than acquired through get_buffer - assert(_pos <= _streamPos); - - //Release less than requested with get_buffer? - if (_pos < _streamPos) { - // Move unreleased stream data in buffer to front - assert(_buffer); - memmove(_buffer.get(), _buffer.get() + size, _streamPos - _pos); - } - } - -protected: - TStream & _stream; - std::unique_ptr _buffer; - size_t _size; - size_t _pos = 0; - size_t _bufferSize = 0; - size_t _streamPos = 0; -}; - -class ProgmemStream -{ -public: - ProgmemStream(PGM_P buf, size_t size) : - _buf(buf), - _left(size) - { - } - - size_t readBytes(char* dst, size_t size) - { - size_t will_read = (_left < size) ? _left : size; - memcpy_P((void*)dst, (PGM_VOID_P)_buf, will_read); - _left -= will_read; - _buf += will_read; - return will_read; - } - -protected: - PGM_P _buf; - size_t _left; -}; - - -#endif //DATASOURCE_H diff --git a/libraries/LittleFS/src/LittleFS.h b/libraries/LittleFS/src/LittleFS.h index bf77026b..26870243 100644 --- a/libraries/LittleFS/src/LittleFS.h +++ b/libraries/LittleFS/src/LittleFS.h @@ -379,7 +379,7 @@ public: return result; } - size_t read(uint8_t* buf, size_t size) override { + int read(uint8_t* buf, size_t size) override { if (!_opened || !_fd | !buf) { return 0; } diff --git a/libraries/Netdump/src/Netdump.h b/libraries/Netdump/src/Netdump.h index 8ef45328..0e8b6cbb 100644 --- a/libraries/Netdump/src/Netdump.h +++ b/libraries/Netdump/src/Netdump.h @@ -75,7 +75,7 @@ private: WiFiClient tcpDumpClient; char* packetBuffer = nullptr; - size_t bufferIndex = 0; + int bufferIndex = 0; static constexpr int tcpBufferSize = 2048; static constexpr int maxPcapLength = 1024; diff --git a/libraries/SDFS/src/SDFS.h b/libraries/SDFS/src/SDFS.h index 0051852a..8a5c0ff5 100644 --- a/libraries/SDFS/src/SDFS.h +++ b/libraries/SDFS/src/SDFS.h @@ -287,7 +287,7 @@ public: return _opened ? _fd->write(buf, size) : -1; } - size_t read(uint8_t* buf, size_t size) override + int read(uint8_t* buf, size_t size) override { return _opened ? _fd->read(buf, size) : -1; } diff --git a/libraries/esp8266/examples/StreamString/StreamString.ino b/libraries/esp8266/examples/StreamString/StreamString.ino new file mode 100644 index 00000000..c04f55f3 --- /dev/null +++ b/libraries/esp8266/examples/StreamString/StreamString.ino @@ -0,0 +1,188 @@ + +// this example sketch in the public domain is also a host and device test + +#include +#include + +void loop() { + delay(1000); +} + +void checksketch(const char* what, const char* res1, const char* res2) { + if (strcmp(res1, res2) == 0) { + Serial << "PASSED: Test " << what << " (result: '" << res1 << "')\n"; + } else { + Serial << "FAILED: Test " << what << ": '" << res1 << "' <> '" << res2 << "' !\n"; + } +} + +#ifndef check +#define check(what, res1, res2) checksketch(what, res1, res2) +#endif + +void testStringPtrProgmem() { + static const char inProgmem [] PROGMEM = "I am in progmem"; + auto inProgmem2 = F("I am too in progmem"); + + int heap = (int)ESP.getFreeHeap(); + auto stream1 = StreamConstPtr(inProgmem, sizeof(inProgmem) - 1); + auto stream2 = StreamConstPtr(inProgmem2); + Serial << stream1 << " - " << stream2 << "\n"; + heap -= (int)ESP.getFreeHeap(); + check("NO heap occupation while streaming progmem strings", String(heap).c_str(), "0"); +} + +void testStreamString() { + String inputString = "hello"; + StreamString result; + + // By default, reading a S2Stream(String) or a StreamString will consume the String. + // It can be disabled by calling ::resetPointer(), (not default) + // and reenabled by calling ::setConsume(). (default) + // + // In default consume mode, reading a byte or a block will remove it from + // the String. Operations are O(n²). + // + // In non-default non-consume mode, it will just move a pointer. That one + // can be ::resetPointer(pos) anytime. See the example below. + + + // The String included in 'result' will not be modified by read: + // (this is not the default) + result.resetPointer(); + + { + // We use a a lighter StreamConstPtr(input) to make a read-only Stream out of + // a String that obviously should not be modified during the time the + // StreamConstPtr instance is used. It is used as a source to be sent to + // 'result'. + + result.clear(); + StreamConstPtr(inputString).sendAll(result); + StreamConstPtr(inputString).sendAll(result); + StreamConstPtr(inputString).sendAll(result); + check("StreamConstPtr.sendAll(StreamString)", result.c_str(), "hellohellohello"); + } + + { + // equivalent of the above + + result.clear(); + result << inputString; + result << inputString << inputString; + check("StreamString< +#include + +#define check(what, res1, res2) CHECK(strcmp(res1, res2) == 0) + +#include "../../../libraries/esp8266/examples/StreamString/StreamString.ino" + +BS_ENV_DECLARE(); + +bool pretest () +{ + return true; +} + +void setup () +{ + Serial.begin(115200); + BS_RUN(Serial); +} + +TEST_CASE("StreamString tests", "[StreamString]") +{ + testStream(); +} diff --git a/tests/device/test_sw_WiFiServer/test_sw_WiFiServer.py b/tests/device/test_sw_WiFiServer/test_sw_WiFiServer.py index b64de33e..86552547 100644 --- a/tests/device/test_sw_WiFiServer/test_sw_WiFiServer.py +++ b/tests/device/test_sw_WiFiServer/test_sw_WiFiServer.py @@ -11,6 +11,7 @@ def setup_echo_server(e): global stop_client_thread global client_thread def echo_client_thread(): + time.sleep(1) # let some time for mDNS to start server_address = socket.gethostbyname('esp8266-wfs-test.local') count = 0 while count < 5 and not stop_client_thread: diff --git a/tests/host/Makefile b/tests/host/Makefile index cc100618..8cb9d892 100644 --- a/tests/host/Makefile +++ b/tests/host/Makefile @@ -76,7 +76,7 @@ $(shell mkdir -p $(BINDIR)) CORE_CPP_FILES := \ $(addprefix $(abspath $(CORE_PATH))/,\ debug.cpp \ - StreamString.cpp \ + StreamSend.cpp \ Stream.cpp \ WString.cpp \ Print.cpp \ diff --git a/tests/host/common/Arduino.cpp b/tests/host/common/Arduino.cpp index c02457f1..780c4adc 100644 --- a/tests/host/common/Arduino.cpp +++ b/tests/host/common/Arduino.cpp @@ -80,4 +80,3 @@ cont_t* g_pcont = NULL; extern "C" void cont_yield(cont_t*) { } - diff --git a/tests/host/common/ClientContextSocket.cpp b/tests/host/common/ClientContextSocket.cpp index b2366fa7..2d1d7c6e 100644 --- a/tests/host/common/ClientContextSocket.cpp +++ b/tests/host/common/ClientContextSocket.cpp @@ -89,7 +89,8 @@ ssize_t mockFillInBuf (int sock, char* ccinbuf, size_t& ccinbufsize) if (ret == 0) { // connection closed - return -1; + // nothing is read + return 0; } if (ret == -1) @@ -97,16 +98,20 @@ ssize_t mockFillInBuf (int sock, char* ccinbuf, size_t& ccinbufsize) if (errno != EAGAIN) { fprintf(stderr, MOCK "ClientContext::(read/peek fd=%i): filling buffer for %zd bytes: %s\n", sock, maxread, strerror(errno)); + // error return -1; } ret = 0; } + ccinbufsize += ret; return ret; } ssize_t mockPeekBytes (int sock, char* dst, size_t usersize, int timeout_ms, char* ccinbuf, size_t& ccinbufsize) { + // usersize==0: peekAvailable() + if (usersize > CCBUFSIZE) mockverbose("CCBUFSIZE(%d) should be increased by %zd bytes (-> %zd)\n", CCBUFSIZE, usersize - CCBUFSIZE, usersize); @@ -114,7 +119,7 @@ ssize_t mockPeekBytes (int sock, char* dst, size_t usersize, int timeout_ms, cha size_t retsize = 0; do { - if (usersize <= ccinbufsize) + if (usersize && usersize <= ccinbufsize) { // data already buffered retsize = usersize; @@ -123,7 +128,14 @@ ssize_t mockPeekBytes (int sock, char* dst, size_t usersize, int timeout_ms, cha // check incoming data data if (mockFillInBuf(sock, ccinbuf, ccinbufsize) < 0) + { return -1; + } + + if (usersize == 0 && ccinbufsize) + // peekAvailable + return ccinbufsize; + if (usersize <= ccinbufsize) { // data just received @@ -179,7 +191,7 @@ ssize_t mockWrite (int sock, const uint8_t* data, size_t size, int timeout_ms) #endif if (ret == -1) { - fprintf(stderr, MOCK "ClientContext::read: write(%d): %s\n", sock, strerror(errno)); + fprintf(stderr, MOCK "ClientContext::write/send(%d): %s\n", sock, strerror(errno)); return -1; } sent += ret; @@ -187,6 +199,8 @@ ssize_t mockWrite (int sock, const uint8_t* data, size_t size, int timeout_ms) fprintf(stderr, MOCK "ClientContext::write: sent %d bytes (%zd / %zd)\n", ret, sent, size); } } - fprintf(stderr, MOCK "ClientContext::write: total sent %zd bytes\n", sent); +#ifdef DEBUG_ESP_WIFI + mockverbose(MOCK "ClientContext::write: total sent %zd bytes\n", sent); +#endif return sent; } diff --git a/tests/host/common/MockUART.cpp b/tests/host/common/MockUART.cpp index 825a4dc0..c5d75ad9 100644 --- a/tests/host/common/MockUART.cpp +++ b/tests/host/common/MockUART.cpp @@ -491,3 +491,9 @@ uart_detect_baudrate(int uart_nr) } }; + + +size_t uart_peek_available (uart_t* uart) { return 0; } +const char* uart_peek_buffer (uart_t* uart) { return nullptr; } +void uart_peek_consume (uart_t* uart, size_t consume) { (void)uart; (void)consume; } + diff --git a/tests/host/common/include/ClientContext.h b/tests/host/common/include/ClientContext.h index 31366ac0..4eaee8ac 100644 --- a/tests/host/common/include/ClientContext.h +++ b/tests/host/common/include/ClientContext.h @@ -27,7 +27,7 @@ class WiFiClient; extern "C" void esp_yield(); extern "C" void esp_schedule(); -#include +#include bool getDefaultPrivateGlobalSyncValue (); @@ -300,6 +300,33 @@ public: _sync = sync; } + // return a pointer to available data buffer (size = peekAvailable()) + // semantic forbids any kind of read() before calling peekConsume() + const char* peekBuffer () + { + return _inbuf; + } + + // return number of byte accessible by peekBuffer() + size_t peekAvailable () + { + ssize_t ret = mockPeekBytes(_sock, nullptr, 0, 0, _inbuf, _inbufsize); + if (ret < 0) + { + abort(); + return 0; + } + return _inbufsize; + } + + // consume bytes after use (see peekBuffer) + void peekConsume (size_t consume) + { + assert(consume <= _inbufsize); + memmove(_inbuf, _inbuf + consume, _inbufsize - consume); + _inbufsize -= consume; + } + private: discard_cb_t _discard_cb = nullptr; diff --git a/tests/restyle.sh b/tests/restyle.sh index 9234c539..ea454069 100755 --- a/tests/restyle.sh +++ b/tests/restyle.sh @@ -15,8 +15,10 @@ libraries/ESP8266mDNS libraries/Wire libraries/lwIP* cores/esp8266/Lwip* -cores/esp8266/core_esp8266_si2c.cpp cores/esp8266/debug* +cores/esp8266/core_esp8266_si2c.cpp +cores/esp8266/StreamString.* +cores/esp8266/StreamSend.* libraries/Netdump " -- GitLab