jaulib v1.3.8
Jau Support Library (C++, Java, ..)
Loading...
Searching...
No Matches
test_iostream01.cpp
Go to the documentation of this file.
1/*
2 * Author: Sven Gothel <sgothel@jausoft.com>
3 * Copyright (c) 2021 Gothel Software e.K.
4 * Copyright (c) 2021 ZAFENA AB
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining
7 * a copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sublicense, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice shall be
15 * included in all copies or substantial portions of the Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
20 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
24 */
25
26#include <cassert>
27#include <cinttypes>
28#include <cstring>
29
30#include <thread>
31#include <pthread.h>
32
33#include <jau/test/catch2_ext.hpp>
34
35#include <jau/file_util.hpp>
36#include <jau/io_util.hpp>
37#include <jau/byte_stream.hpp>
38
39#include <jau/debug.hpp>
40
41#include "test_httpd.hpp"
42
43extern "C" {
44 #include <unistd.h>
45}
46
47using namespace jau::fractions_i64_literals;
48using namespace jau::int_literals;
49
51 public:
52 const std::string url_input_root = "http://localhost:8080/";
53 const std::string basename_10kiB = "testfile_data_10kiB.bin";
54
56 // produce fresh demo data
57
59 {
60 std::string one_line = "Hello World, this is a test and I like it. Exactly 100 characters long. 0123456780 abcdefghjklmnop..";
62
63 REQUIRE( ofs.good() == true );
64 REQUIRE( ofs.is_open() == true );
65
66 for(size_t i=0; i < 1024_uz * 10_uz; i+=one_line.size()) { // 10kiB
67 REQUIRE( one_line.size() == ofs.write(one_line.data(), one_line.size()) );
68 }
69 }
71 int res = std::system("killall mini_httpd");
72 (void)res;
73 const std::string cwd = jau::fs::get_cwd();
74 const std::string cmd = std::string(mini_httpd_exe)+" -p 8080 -l "+cwd+"/mini_httpd.log";
75 jau::PLAIN_PRINT(true, "%s", cmd.c_str());
76 res = std::system(cmd.c_str());
77 (void)res;
78 }
79 }
80
83 int res = std::system("killall mini_httpd");
84 (void)res;
85 }
86 }
87
89 {
90 std::vector<std::string_view> protos = jau::io::uri_tk::supported_protocols();
91 jau::PLAIN_PRINT(true, "test00_protocols: Supported protocols: %zu: %s", protos.size(), jau::to_string(protos, ",").c_str());
92#ifdef USE_LIBCURL
93 REQUIRE( 0 < protos.size() );
94#else
95 REQUIRE( 0 == protos.size() );
96#endif
97 }
98 const bool http_support_expected = jau::io::uri_tk::protocol_supported("http:");
99 const bool file_support_expected = jau::io::uri_tk::protocol_supported("file:");
100 {
101 const std::string url = url_input_root + basename_10kiB;
102 REQUIRE( false == jau::io::uri_tk::is_local_file_protocol(url) );
103 REQUIRE( http_support_expected == jau::io::uri_tk::protocol_supported(url) );
104 }
105 {
106 const std::string url = "https://localhost:8080/" + basename_10kiB;
107 REQUIRE( false == jau::io::uri_tk::is_local_file_protocol(url) );
108 REQUIRE( http_support_expected == jau::io::uri_tk::protocol_supported(url) );
109 }
110 {
111 const std::string url = "file://" + basename_10kiB;
112 REQUIRE( true == jau::io::uri_tk::is_local_file_protocol(url) );
113 REQUIRE( file_support_expected == jau::io::uri_tk::protocol_supported(url) );
114 }
115 {
116 const std::string url = "lala://localhost:8080/" + basename_10kiB;
117 REQUIRE( false == jau::io::uri_tk::is_local_file_protocol(url) );
118 REQUIRE( false == jau::io::uri_tk::protocol_supported(url) );
119 }
120 {
121 // sync read_url_stream w/ unknown protocol
122 const std::string url = "lala://localhost:8080/" + basename_10kiB;
124 size_t consumed_calls = 0;
125 uint64_t consumed_total_bytes = 0;
126 jau::io::StreamConsumerFunc consume = [&](jau::io::secure_vector<uint8_t>& data, bool is_final) noexcept -> bool {
127 (void)is_final;
128 consumed_calls++;
129 consumed_total_bytes += data.size();
130 return true;
131 };
132 uint64_t http_total_bytes = jau::io::read_url_stream(url, buffer, consume);
133 REQUIRE( 0 == http_total_bytes );
134 REQUIRE( consumed_total_bytes == http_total_bytes );
135 REQUIRE( 0 == consumed_calls );
136 }
137 {
138 // sync read_url_stream w/ unknown protocol
139 const std::string url = "lala://localhost:8080/" + basename_10kiB;
140
142 jau::io::SyncStreamResponseRef res = jau::io::read_url_stream_sync(nullptr, url, nullptr, &rb, nullptr);
143 REQUIRE( res->header_resp.completed() == true );
144 REQUIRE( res->has_content_length == false );
145 REQUIRE( res->content_length == 0 );
146 REQUIRE( res->failed() == true );
147 }
148 {
149 // async read_url_stream w/ unknown protocol
150 const std::string url = "lala://localhost:8080/" + basename_10kiB;
151
153 jau::io::AsyncStreamResponseRef res = jau::io::read_url_stream_async(nullptr, url, nullptr, &rb, nullptr);
154 REQUIRE( !res->thread.joinable() );
155 REQUIRE( res->header_resp.completed() == true );
156 REQUIRE( res->has_content_length == false );
157 REQUIRE( res->content_length == 0 );
158 REQUIRE( res->failed() == true );
159 }
160 }
161
164 jau::PLAIN_PRINT(true, "http not supported, abort\n");
165 return;
166 }
167 const jau::fs::file_stats in_stats(basename_10kiB);
168 const size_t file_size = in_stats.size();
169 const std::string url_input = url_input_root + basename_10kiB;
170
171 jau::io::ByteOutStream_File outfile("testfile01_01_out.bin");
172 REQUIRE( outfile.good() );
173 REQUIRE( outfile.is_open() );
174
176 size_t consumed_calls = 0;
177 uint64_t consumed_total_bytes = 0;
178 jau::io::StreamConsumerFunc consume = [&](jau::io::secure_vector<uint8_t>& data, bool is_final) noexcept -> bool {
179 consumed_calls++;
180 if( data.size() != outfile.write(data.data(), data.size()) ) {
181 return false;
182 }
183 consumed_total_bytes += data.size();
184 jau::PLAIN_PRINT(true, "test01_sync_ok #%zu: consumed size %zu, total %" PRIu64 ", capacity %zu, final %d",
185 consumed_calls, data.size(), consumed_total_bytes, data.capacity(), is_final );
186 return true;
187 };
188 uint64_t http_total_bytes = jau::io::read_url_stream(url_input, buffer, consume);
189 const uint64_t out_bytes_total = outfile.tellp();
190 jau::PLAIN_PRINT(true, "test01_sync_ok Done: total %" PRIu64 ", capacity %zu", consumed_total_bytes, buffer.capacity());
191
192 REQUIRE( file_size == http_total_bytes );
193 REQUIRE( consumed_total_bytes == http_total_bytes );
194 REQUIRE( consumed_total_bytes == out_bytes_total );
195 }
196
199 jau::PLAIN_PRINT(true, "http not supported, abort\n");
200 return;
201 }
202 const std::string url_input = url_input_root + "doesnt_exists.txt";
203
204 jau::io::ByteOutStream_File outfile("testfile02_01_out.bin");
205 REQUIRE( outfile.good() );
206 REQUIRE( outfile.is_open() );
207
209 size_t consumed_calls = 0;
210 uint64_t consumed_total_bytes = 0;
211 jau::io::StreamConsumerFunc consume = [&](jau::io::secure_vector<uint8_t>& data, bool is_final) noexcept -> bool {
212 consumed_calls++;
213 if( data.size() != outfile.write(data.data(), data.size()) ) {
214 return false;
215 }
216 consumed_total_bytes += data.size();
217 jau::PLAIN_PRINT(true, "test02_sync_404 #%zu: consumed size %zu, total %" PRIu64 ", capacity %zu, final %d",
218 consumed_calls, data.size(), consumed_total_bytes, data.capacity(), is_final );
219 return true;
220 };
221 uint64_t http_total_bytes = jau::io::read_url_stream(url_input, buffer, consume);
222 const uint64_t out_bytes_total = outfile.tellp();
223 jau::PLAIN_PRINT(true, "test02_sync_404 Done: total %" PRIu64 ", capacity %zu", consumed_total_bytes, buffer.capacity());
224
225 REQUIRE( 0 == http_total_bytes );
226 REQUIRE( consumed_total_bytes == http_total_bytes );
227 REQUIRE( consumed_total_bytes == out_bytes_total );
228 }
229
232 jau::PLAIN_PRINT(true, "http not supported, abort\n");
233 return;
234 }
235 const jau::fs::file_stats in_stats(basename_10kiB);
236 const size_t file_size = in_stats.size();
237 const std::string url_input = url_input_root + basename_10kiB;
238
239 jau::io::ByteOutStream_File outfile("testfile11_01_out.bin");
240 REQUIRE( outfile.good() );
241 REQUIRE( outfile.is_open() );
242
243 constexpr const size_t buffer_size = 4096;
245
246 jau::io::AsyncStreamResponseRef res = jau::io::read_url_stream_async(nullptr, url_input, nullptr, &rb, nullptr);
247 REQUIRE( res->failed() == false );
248
249 jau::io::secure_vector<uint8_t> buffer(buffer_size);
250 size_t consumed_loops = 0;
251 uint64_t consumed_total_bytes = 0;
252
253 while( res->processing() || !rb.isEmpty() ) {
254 consumed_loops++;
255 // const size_t consumed_bytes = content_length >= 0 ? std::min(buffer_size, content_length - consumed_total_bytes) : rb.getSize();
256 const size_t consumed_bytes = rb.getBlocking(buffer.data(), buffer_size, 1, 500_ms);
257 consumed_total_bytes += consumed_bytes;
258 jau::PLAIN_PRINT(true, "test11_async_ok.0 #%zu: consumed[this %zu, total %" PRIu64 ", result %d, rb %s",
259 consumed_loops, consumed_bytes, consumed_total_bytes, res->result.load(), rb.toString().c_str() );
260 if( consumed_bytes != outfile.write(buffer.data(), consumed_bytes) ) {
261 break;
262 }
263 }
264 const uint64_t out_bytes_total = outfile.tellp();
265 jau::PLAIN_PRINT(true, "test11_async_ok.X Done: total %" PRIu64 ", result %d, rb %s",
266 consumed_total_bytes, (int)res->result.load(), rb.toString().c_str() );
267
268 res->thread.join();
269
270 REQUIRE( res->header_resp.completed() == true );
271 REQUIRE( res->has_content_length == true );
272 REQUIRE( res->content_length == file_size );
273 REQUIRE( res->content_length == consumed_total_bytes );
274 REQUIRE( res->content_length == out_bytes_total );
275 REQUIRE( res->success() == true );
276 }
277
280 jau::PLAIN_PRINT(true, "http not supported, abort\n");
281 return;
282 }
283 const std::string url_input = url_input_root + "doesnt_exists.txt";
284
285 jau::io::ByteOutStream_File outfile("testfile12_01_out.bin");
286 REQUIRE( outfile.good() );
287 REQUIRE( outfile.is_open() );
288
289 constexpr const size_t buffer_size = 4096;
291
292 jau::io::AsyncStreamResponseRef res = jau::io::read_url_stream_async(nullptr, url_input, nullptr, &rb, nullptr);
293 REQUIRE( res->failed() == false );
294
295 jau::io::secure_vector<uint8_t> buffer(buffer_size);
296 size_t consumed_loops = 0;
297 uint64_t consumed_total_bytes = 0;
298
299 while( res->processing() || !rb.isEmpty() ) {
300 consumed_loops++;
301 // const size_t consumed_bytes = content_length >= 0 ? std::min(buffer_size, content_length - consumed_total_bytes) : rb.getSize();
302 const size_t consumed_bytes = rb.getBlocking(buffer.data(), buffer_size, 1, 500_ms);
303 consumed_total_bytes += consumed_bytes;
304 jau::PLAIN_PRINT(true, "test12_async_404.0 #%zu: consumed[this %zu, total %" PRIu64 ", result %d, rb %s",
305 consumed_loops, consumed_bytes, consumed_total_bytes, res->result.load(), rb.toString().c_str() );
306 if( consumed_bytes != outfile.write(reinterpret_cast<char*>(buffer.data()), consumed_bytes) ) {
307 break;
308 }
309 }
310 const uint64_t out_bytes_total = outfile.tellp();
311 jau::PLAIN_PRINT(true, "test12_async_404.X Done: total %" PRIu64 ", result %d, rb %s",
312 consumed_total_bytes, (int)res->result.load(), rb.toString().c_str() );
313
314 res->thread.join();
315
316 REQUIRE( res->header_resp.completed() == true );
317 REQUIRE( res->has_content_length == false );
318 REQUIRE( res->content_length == 0 );
319 REQUIRE( res->content_length == consumed_total_bytes );
320 REQUIRE( res->content_length == out_bytes_total );
321 REQUIRE( res->failed() == true );
322 REQUIRE( res->header_resp.response_code() == 404 );
323 }
324
325};
326
327METHOD_AS_TEST_CASE( TestIOStream01::test00_protocols, "TestIOStream01 - test00_protocols");
328METHOD_AS_TEST_CASE( TestIOStream01::test01_sync_ok, "TestIOStream01 - test01_sync_ok");
329METHOD_AS_TEST_CASE( TestIOStream01::test02_sync_404, "TestIOStream01 - test02_sync_404");
330METHOD_AS_TEST_CASE( TestIOStream01::test11_async_ok, "TestIOStream01 - test11_async_ok");
331METHOD_AS_TEST_CASE( TestIOStream01::test12_async_404, "TestIOStream01 - test12_async_404");
332
const std::string basename_10kiB
const std::string url_input_root
Platform agnostic representation of POSIX ::lstat() and ::stat() for a given pathname.
uint64_t size() const noexcept
Returns the size in bytes of this element if is_file(), otherwise zero.
File based byte output stream, including named file descriptor.
size_t write(const void *, size_t) noexcept override
Write to the data sink.
uint64_t tellp() const noexcept override
Returns the output position indicator.
bool is_open() const noexcept override
Checks if the stream has an associated file.
bool good() const noexcept
Checks if no error nor eof() has occurred i.e.
bool getBlocking(Value_type &result, const fraction_i64 &timeout, bool &timeout_occurred) noexcept
Dequeues the oldest enqueued element.
bool isEmpty() const noexcept
Returns true if this ring buffer is empty, otherwise false.
std::string toString() const noexcept
Returns a short string representation incl.
std::string to_string(const endian_t v) noexcept
Return std::string representation of the given endian.
std::string get_cwd() noexcept
Return the current working directory or empty on failure.
Definition file_util.cpp:84
bool remove(const std::string &path, const traverse_options topts=traverse_options::none) noexcept
Remove the given path.
AsyncStreamResponseRef read_url_stream_async(net_tk_handle handle, const std::string &url, http::PostRequestPtr httpPostReq, ByteRingbuffer *buffer, const AsyncStreamConsumerFunc &consumer_fn) noexcept
Asynchronous URL stream reader using the given AsyncStreamConsumerFunc consumer_fn.
Definition io_util.cpp:1119
SyncStreamResponseRef read_url_stream_sync(net_tk_handle handle, const std::string &url, http::PostRequestPtr httpPostReq, ByteRingbuffer *buffer, const SyncStreamConsumerFunc &consumer_fn) noexcept
Synchronous URL stream reader using the given SyncStreamConsumerFunc consumer_fn.
Definition io_util.cpp:1093
bool is_local_file_protocol(const std::string_view &uri) noexcept
Returns true if the uri-scheme of given uri matches the local file protocol, i.e.
Definition io_util.cpp:206
uint64_t read_url_stream(const std::string &url, secure_vector< uint8_t > &buffer, const StreamConsumerFunc &consumer_fn) noexcept
Synchronous URL stream reader using the given StreamConsumerFunc consumer_fn.
Definition io_util.cpp:313
jau::function< bool(secure_vector< uint8_t > &, bool)> StreamConsumerFunc
Stream consumer function.
Definition io_util.hpp:98
std::vector< std::string_view > supported_protocols() noexcept
Returns a list of supported protocol supported by libcurl network protocols, queried at runtime.
Definition io_util.cpp:161
std::shared_ptr< SyncStreamResponse > SyncStreamResponseRef
Definition io_util.hpp:283
std::shared_ptr< AsyncStreamResponse > AsyncStreamResponseRef
Definition io_util.hpp:364
jau::ringbuffer< uint8_t, size_t > ByteRingbuffer
Definition io_util.hpp:50
const size_t BEST_URLSTREAM_RINGBUFFER_SIZE
std::vector< T, jau::callocator_sec< T > > secure_vector
Definition io_util.hpp:46
bool protocol_supported(const std::string_view &uri) noexcept
Returns true if the uri-scheme of given uri matches a supported by libcurl network protocols otherwis...
Definition io_util.cpp:196
void PLAIN_PRINT(const bool printPrefix, const char *format,...) noexcept
Use for unconditional plain messages, prefix '[elapsed_time] ' if printPrefix == true.
Definition debug.cpp:258
constexpr std::string_view mini_httpd_exe
METHOD_AS_TEST_CASE(TestIOStream01::test00_protocols, "TestIOStream01 - test00_protocols")