Direct-BT v3.3.0-1-gc2d430c
Direct-BT - Direct Bluetooth Programming.
io_util.hpp
Go to the documentation of this file.
1/*
2 * Author: Sven Gothel <sgothel@jausoft.com>
3 * Copyright (c) 2021-2023 Gothel Software e.K.
4 * Copyright (c) 2021 ZAFENA AB
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining
7 * a copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sublicense, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice shall be
15 * included in all copies or substantial portions of the Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
20 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
24 */
25
26#ifndef JAU_IO_UTIL_HPP_
27#define JAU_IO_UTIL_HPP_
28
29#include <string>
30#include <cstdint>
31#include <thread>
32#include <vector>
33
34#include <jau/basic_types.hpp>
36#include <jau/ringbuffer.hpp>
37#include <jau/functional.hpp>
38
39namespace jau::io {
40 /** @defgroup IOUtils IO Utilities
41 * Input and Output (IO) types and functionality.
42 *
43 * @{
44 */
45
46 template<typename T> using secure_vector = std::vector<T, jau::callocator_sec<T>>;
47
48 typedef std::basic_string<char, std::char_traits<char>, jau::callocator_sec<char>> secure_string;
49
51
52 extern const size_t BEST_URLSTREAM_RINGBUFFER_SIZE;
53
54 /**
55 * I/O direction, read or write
56 */
57 enum class io_dir_t : int8_t {
58 /** Read Operation */
59 READ = 0,
60
61 /** Write Operation */
62 WRITE = 1
63 };
64
65 /**
66 * Asynchronous I/O operation result value
67 */
68 enum class async_io_result_t : int8_t {
69 /** Operation failed. */
70 FAILED = -1,
71
72 /** Operation still in progress. */
73 NONE = 0,
74
75 /** Operation succeeded. */
76 SUCCESS = 1
77 };
79
80 /**
81 * Stream consumer function
82 * - `bool consumer(secure_vector<uint8_t>& data, bool is_final)`
83 *
84 * Returns true to signal continuation, false to end streaming.
85 */
86 typedef jau::function<bool(secure_vector<uint8_t>& /* data */, bool /* is_final */)> StreamConsumerFunc;
87
88 /**
89 * Synchronous byte input stream reader from given file path using the given StreamConsumerFunc consumer_fn.
90 *
91 * To abort streaming, user may return `false` from the given `consumer_func`.
92 *
93 * It is guaranteed that consumer_fn() is called with `is_final=true` once at the end,
94 * even if `input_file` stream has zero size.
95 *
96 * @param input_file input file name path, `-` denotes std::stdin.
97 * @param buffer secure std::vector buffer, passed down to consumer_fn
98 * @param consumer_fn StreamConsumerFunc consumer for each received heap of bytes, returning true to continue stream of false to abort.
99 * @return total bytes read or 0 if error
100 */
101 uint64_t read_file(const std::string& input_file,
103 const StreamConsumerFunc& consumer_fn) noexcept;
104
105 class ByteInStream; // fwd
106
107 /**
108 * Synchronous byte input stream reader using the given StreamConsumerFunc consumer_fn.
109 *
110 * To abort streaming, user may return `false` from the given `consumer_func`.
111 *
112 * It is guaranteed that consumer_fn() is called with `is_final=true` once at the end,
113 * even input stream has zero size.
114 *
115 * @param in the input byte stream to read from
116 * @param buffer secure std::vector buffer, passed down to consumer_fn
117 * @param consumer_fn StreamConsumerFunc consumer for each received heap of bytes, returning true to continue stream of false to abort.
118 * @return total bytes read or 0 if error
119 */
120 uint64_t read_stream(ByteInStream& in,
122 const StreamConsumerFunc& consumer_fn) noexcept;
123
124 /**
125 * Synchronous double-buffered byte input stream reader using the given StreamConsumerFunc consumer_fn.
126 *
127 * To abort streaming, user may return `false` from the given `consumer_func`.
128 *
129 * It is guaranteed that consumer_fn() is called with `is_final=true` once at the end,
130 * even if input stream has zero size.
131 *
132 * Implementation reads one buffer ahead in respect to consumer_fn(). <br/>
133 * If reading zero bytes on the next buffer,
134 * it propagates the end-of-file (EOF) to the previous buffer which will be send via consumer_fn() next.<br/>
135 *
136 * This way, the consumer_fn() will always receive its `is_final` flag on the last sent bytes (size > 0)
137 * even if the content-size is unknown (pipe). <br/>
138 * Hence it allows e.g. decryption to work where the final data chunck must be processed as such.
139 *
140 * @param in the input byte stream to read from
141 * @param buffer1 secure std::vector buffer, passed down to consumer_fn
142 * @param buffer2 secure std::vector buffer, passed down to consumer_fn
143 * @param consumer_fn StreamConsumerFunc consumer for each received heap of bytes, returning true to continue stream of false to abort.
144 * @return total bytes read or 0 if error
145 */
146 uint64_t read_stream(ByteInStream& in,
148 const StreamConsumerFunc& consumer_fn) noexcept;
149
150 /**
151 * Synchronous URL stream reader using the given StreamConsumerFunc consumer_fn.
152 *
153 * To abort streaming, user may return `false` from the given `consumer_func`.
154 *
155 * Standard implementation uses [curl](https://curl.se/),
156 * hence all [*libcurl* network protocols](https://curl.se/docs/url-syntax.html) are supported,
157 * see jau::io::uri::supported_protocols().
158 *
159 * If the uri-sheme doesn't match a supported protocol, see jau::io::uri::protocol_supported(),
160 * function returns immediately with zero bytes.
161 *
162 * @param url the URL to open a connection to and stream bytes from
163 * @param buffer secure std::vector buffer, passed down to consumer_fn
164 * @param consumer_fn StreamConsumerFunc consumer for each received heap of bytes, returning true to continue stream of false to abort.
165 * @return total bytes read or 0 if transmission error or protocol of given url is not supported
166 */
167 uint64_t read_url_stream(const std::string& url,
169 const StreamConsumerFunc& consumer_fn) noexcept;
170
171 /**
172 * Synchronization for URL header completion
173 * as used by asynchronous read_url_stream().
174 *
175 * @see url_header_sync::completed()
176 */
178 private:
179 std::mutex m_sync;
180 std::condition_variable m_cv;
181 jau::relaxed_atomic_bool m_completed;
182
183 public:
185 : m_completed(false)
186 { }
187
188 /**
189 * Returns whether URL header is completed.
190 *
191 * Completion is reached in any of the following cases
192 * - Final (http) CRLF message received
193 * - Any http header error response received
194 * - First data package received
195 * - End of operation
196 */
197 bool completed() const noexcept { return m_completed; }
198
199 /**
200 * Notify completion, see completed()
201 */
202 void notify_complete() noexcept;
203
204 /**
205 * Wait until completed() has been reached.
206 * @param timeout maximum duration in fractions of seconds to wait, where fractions_i64::zero waits infinitely
207 * @return true if completed within timeout, otherwise false
208 */
209 bool wait_until_completion(const jau::fraction_i64& timeout) noexcept;
210 };
211
212 /**
213 * Asynchronous URL read content using the given byte jau::ringbuffer, allowing parallel reading.
214 *
215 * To abort streaming, user may set given reference `results` to a value other than async_io_result_t::NONE.
216 *
217 * Standard implementation uses [curl](https://curl.se/),
218 * hence all [*libcurl* network protocols](https://curl.se/docs/url-syntax.html) are supported,
219 * see jau::io::uri::supported_protocols().
220 *
221 * If the uri-sheme doesn't match a supported protocol, see jau::io::uri::protocol_supported(),
222 * function returns with nullptr.
223 *
224 * @param url the URL to open a connection to and stream bytes from
225 * @param buffer the ringbuffer destination to write into
226 * @param header_sync synchronization object for URL header completion
227 * @param has_content_length indicating whether content_length is known from server
228 * @param content_length tracking the content_length
229 * @param total_read tracking the total_read
230 * @param result reference to tracking async_io_result_t. If set to other than async_io_result_t::NONE while streaming, streaming is aborted.
231 * @return the url background reading thread unique-pointer or nullptr if protocol of given url is not supported
232 */
233 std::unique_ptr<std::thread> read_url_stream(const std::string& url,
234 ByteRingbuffer& buffer,
235 jau::io::url_header_sync& header_sync,
236 jau::relaxed_atomic_bool& has_content_length,
237 jau::relaxed_atomic_uint64& content_length,
238 jau::relaxed_atomic_uint64& total_read,
239 relaxed_atomic_async_io_result_t& result) noexcept;
240
241 void print_stats(const std::string& prefix, const uint64_t& out_bytes_total, const jau::fraction_i64& td) noexcept;
242
243 /**@}*/
244
245 /**
246 * Limited URI toolkit to query handled protocols by the IO implementation.
247 *
248 * The URI scheme functionality exposed here is limited and only provided to decide whether the used implementation
249 * is able to handle the protocol. This is not a replacement for a proper URI class.
250 */
251 namespace uri_tk {
252 /** \addtogroup IOUtils
253 *
254 * @{
255 */
256
257 /**
258 * Returns a list of supported protocol supported by [*libcurl* network protocols](https://curl.se/docs/url-syntax.html),
259 * queried at runtime.
260 * @see protocol_supported()
261 */
262 std::vector<std::string_view> supported_protocols() noexcept;
263
264 /**
265 * Returns the valid uri-scheme from given uri,
266 * which is empty if no valid scheme is included.
267 *
268 * The given uri must include at least a colon after the uri-scheme part.
269 *
270 * @param uri an uri
271 * @return valid uri-scheme, empty if non found
272 */
273 std::string_view get_scheme(const std::string_view& uri) noexcept;
274
275 /**
276 * Returns true if the uri-scheme of given uri matches a supported by [*libcurl* network protocols](https://curl.se/docs/url-syntax.html) otherwise false.
277 *
278 * The uri-scheme is retrieved via get_scheme() passing given uri, hence must include at least a colon after the uri-scheme part.
279 *
280 * The *libcurl* supported protocols is queried at runtime, see supported_protocols().
281 *
282 * @param uri an uri to test
283 * @return true if the uri-scheme of given uri is supported, otherwise false.
284 * @see supported_protocols()
285 * @see get_scheme()
286 */
287 bool protocol_supported(const std::string_view& uri) noexcept;
288
289 /**
290 * Returns true if the uri-scheme of given uri matches the local `file` protocol, i.e. starts with `file://`.
291 * @param uri an uri to test
292 */
293 bool is_local_file_protocol(const std::string_view& uri) noexcept;
294
295 /**
296 * Returns true if the uri-scheme of given uri matches the `http` or `https` protocol, i.e. starts with `http:` or `https:`.
297 * @param uri an uri to test
298 */
299 bool is_httpx_protocol(const std::string_view& uri) noexcept;
300
301 /**@}*/
302 }
303
304} // namespace elevator::io
305
306#endif /* JAU_IO_UTIL_HPP_ */
Class template jau::function is a general-purpose static-polymorphic function wrapper.
Abstract byte input stream object.
Synchronization for URL header completion as used by asynchronous read_url_stream().
Definition: io_util.hpp:177
bool wait_until_completion(const jau::fraction_i64 &timeout) noexcept
Wait until completed() has been reached.
Definition: io_util.cpp:442
void notify_complete() noexcept
Notify completion, see completed()
Definition: io_util.cpp:434
url_header_sync() noexcept
Definition: io_util.hpp:184
bool completed() const noexcept
Returns whether URL header is completed.
Definition: io_util.hpp:197
std::vector< T, jau::callocator_sec< T > > secure_vector
Definition: io_util.hpp:46
bool is_local_file_protocol(const std::string_view &uri) noexcept
Returns true if the uri-scheme of given uri matches the local file protocol, i.e.
Definition: io_util.cpp:204
io_dir_t
I/O direction, read or write.
Definition: io_util.hpp:57
uint64_t read_stream(ByteInStream &in, secure_vector< uint8_t > &buffer, const StreamConsumerFunc &consumer_fn) noexcept
Synchronous byte input stream reader using the given StreamConsumerFunc consumer_fn.
Definition: io_util.cpp:59
uint64_t read_url_stream(const std::string &url, secure_vector< uint8_t > &buffer, const StreamConsumerFunc &consumer_fn) noexcept
Synchronous URL stream reader using the given StreamConsumerFunc consumer_fn.
Definition: io_util.cpp:309
jau::function< bool(secure_vector< uint8_t > &, bool)> StreamConsumerFunc
Stream consumer function.
Definition: io_util.hpp:86
bool is_httpx_protocol(const std::string_view &uri) noexcept
Returns true if the uri-scheme of given uri matches the http or https protocol, i....
Definition: io_util.cpp:208
std::string_view get_scheme(const std::string_view &uri) noexcept
Returns the valid uri-scheme from given uri, which is empty if no valid scheme is included.
Definition: io_util.cpp:182
std::vector< std::string_view > supported_protocols() noexcept
Returns a list of supported protocol supported by libcurl network protocols, queried at runtime.
Definition: io_util.cpp:159
async_io_result_t
Asynchronous I/O operation result value.
Definition: io_util.hpp:68
std::basic_string< char, std::char_traits< char >, jau::callocator_sec< char > > secure_string
Definition: io_util.hpp:48
jau::ringbuffer< uint8_t, size_t > ByteRingbuffer
Definition: io_util.hpp:50
const size_t BEST_URLSTREAM_RINGBUFFER_SIZE
Definition: byte_stream.cpp:73
void print_stats(const std::string &prefix, const uint64_t &out_bytes_total, const jau::fraction_i64 &td) noexcept
Definition: io_util.cpp:761
jau::ordered_atomic< async_io_result_t, std::memory_order_relaxed > relaxed_atomic_async_io_result_t
Definition: io_util.hpp:78
uint64_t read_file(const std::string &input_file, secure_vector< uint8_t > &buffer, const StreamConsumerFunc &consumer_fn) noexcept
Synchronous byte input stream reader from given file path using the given StreamConsumerFunc consumer...
Definition: io_util.cpp:46
bool protocol_supported(const std::string_view &uri) noexcept
Returns true if the uri-scheme of given uri matches a supported by libcurl network protocols otherwis...
Definition: io_util.cpp:194
@ READ
Read Operation.
@ WRITE
Write Operation.
@ NONE
Operation still in progress.
@ FAILED
Operation failed.
@ SUCCESS
Operation succeeded.
__pack(...): Produces MSVC, clang and gcc compatible lead-in and -out macros.
Definition: backtrace.hpp:32
STL namespace.
A simple secure allocator for integral types using POSIX C functions: ::malloc() and ::free().