Input and Output (IO) types and functionality. More...

Namespaces
namespace	jau::io::http

Classes
struct	jau::io::AsyncStreamResponse
	Asynchronous stream response. More...

class	jau::io::ByteInStream
	Abstract byte input stream object. More...

class	jau::io::ByteInStream_Feed
	Ringbuffer-Based byte input stream with an externally provisioned data feed. More...

class	jau::io::ByteInStream_File
	File based byte input stream, including named file descriptor. More...

class	jau::io::ByteInStream_Recorder
	Wrapped byte input stream with the capability to record the read byte stream at will. More...

class	jau::io::ByteInStream_SecMemory
	Secure Memory-Based byte input stream. More...

class	jau::io::ByteInStream_URL
	Ringbuffer-Based byte input stream with a URL connection provisioned data feed. More...

class	jau::io::ByteOutStream
	Abstract byte output stream object, to write data to a sink. More...

class	jau::io::ByteOutStream_File
	File based byte output stream, including named file descriptor. More...

class	jau::io::iostate_func
	Supporting std::basic_ios's iostate functionality for all ByteInStream implementations. More...

struct	jau::io::SyncStreamResponse
	Synchronous stream response. More...

class	jau::io::url_header_resp
	Synchronized URL header response completion as used by asynchronous read_url_stream(). More...

Typedefs
typedef jau::function< bool(AsyncStreamResponse &, const uint8_t *, size_t, bool)>	jau::io::AsyncStreamConsumerFunc
	Asynchronous stream consumer function.

using	jau::io::AsyncStreamResponseRef = std::shared_ptr<AsyncStreamResponse>

typedef jau::ringbuffer< uint8_t, size_t >	jau::io::ByteRingbuffer

using	jau::io::net_tk_handle = void*

typedef jau::ordered_atomic< io_result_t, std::memory_order_relaxed >	jau::io::relaxed_atomic_io_result_t

typedef std::basic_string< char, std::char_traits< char >, jau::callocator_sec< char > >	jau::io::secure_string

template<typename T>
using	jau::io::secure_vector = std::vector<T, jau::callocator_sec<T>>

typedef jau::function< bool(secure_vector< uint8_t > &, bool)>	jau::io::StreamConsumerFunc
	Stream consumer function.

typedef jau::function< bool(SyncStreamResponse &, const uint8_t *, size_t, bool)>	jau::io::SyncStreamConsumerFunc
	Synchronous stream consumer function.

using	jau::io::SyncStreamResponseRef = std::shared_ptr<SyncStreamResponse>

Enumerations
enum class	jau::io::io_dir_t : int8_t { jau::io::io_dir_t::READ = 0 , jau::io::io_dir_t::WRITE = 1 }
	I/O direction, read or write. More...

enum class	jau::io::io_result_t : int8_t { jau::io::io_result_t::FAILED = -1 , jau::io::io_result_t::NONE = 0 , jau::io::io_result_t::SUCCESS = 1 }
	I/O operation result value. More...

enum class	jau::io::iostate : uint32_t { jau::io::iostate::none = 0 , jau::io::iostate::goodbit = 0 , jau::io::iostate::badbit = 1U << 0 , jau::io::iostate::eofbit = 1U << 1 , jau::io::iostate::failbit = 1U << 2 , jau::io::iostate::timeout = 1U << 3 }

Functions
net_tk_handle	jau::io::create_net_tk_handle () noexcept
	creates a reusable handle, free with free_net_tk_handle() after use.

void	jau::io::free_net_tk_handle (net_tk_handle handle) noexcept
	frees a handle after use created by create_net_tk_handle()

std::string_view	jau::io::uri_tk::get_scheme (const std::string_view &uri) noexcept
	Returns the valid uri-scheme from given uri, which is empty if no valid scheme is included.

bool	jau::io::uri_tk::is_httpx_protocol (const std::string_view &uri) noexcept
	Returns true if the uri-scheme of given uri matches the `http` or `https` protocol, i.e.

bool	jau::io::uri_tk::is_local_file_protocol (const std::string_view &uri) noexcept
	Returns true if the uri-scheme of given uri matches the local `file` protocol, i.e.

	jau::io::JAU_MAKE_BITFIELD_ENUM_STRING (iostate, goodbit, badbit, eofbit, failbit, timeout)

std::ostream &	jau::io::operator<< (std::ostream &os, io_result_t v)

void	jau::io::print_stats (const std::string &prefix, const uint64_t &out_bytes_total, const jau::fraction_i64 &td) noexcept

bool	jau::io::uri_tk::protocol_supported (const std::string_view &uri) noexcept
	Returns true if the uri-scheme of given uri matches a supported by libcurl network protocols otherwise false.

uint64_t	jau::io::read_file (const std::string &input_file, secure_vector< uint8_t > &buffer, const StreamConsumerFunc &consumer_fn) noexcept
	Synchronous byte input stream reader from given file path using the given StreamConsumerFunc consumer_fn.

uint64_t	jau::io::read_stream (ByteInStream &in, secure_vector< uint8_t > &buffer, const StreamConsumerFunc &consumer_fn) noexcept
	Synchronous byte input stream reader using the given StreamConsumerFunc consumer_fn.

uint64_t	jau::io::read_stream (ByteInStream &in, secure_vector< uint8_t > &buffer1, secure_vector< uint8_t > &buffer2, const StreamConsumerFunc &consumer_fn) noexcept
	Synchronous double-buffered byte input stream reader using the given StreamConsumerFunc consumer_fn.

uint64_t	jau::io::read_url_stream (const std::string &url, secure_vector< uint8_t > &buffer, const StreamConsumerFunc &consumer_fn) noexcept
	Synchronous URL stream reader using the given StreamConsumerFunc consumer_fn.

AsyncStreamResponseRef	jau::io::read_url_stream_async (net_tk_handle handle, const std::string &url, http::PostRequestPtr httpPostReq, ByteRingbuffer *buffer, const AsyncStreamConsumerFunc &consumer_fn) noexcept
	Asynchronous URL stream reader using the given AsyncStreamConsumerFunc consumer_fn.

SyncStreamResponseRef	jau::io::read_url_stream_sync (net_tk_handle handle, const std::string &url, http::PostRequestPtr httpPostReq, ByteRingbuffer *buffer, const SyncStreamConsumerFunc &consumer_fn) noexcept
	Synchronous URL stream reader using the given SyncStreamConsumerFunc consumer_fn.

std::vector< std::string_view >	jau::io::uri_tk::supported_protocols () noexcept
	Returns a list of supported protocol supported by libcurl network protocols, queried at runtime.

std::unique_ptr< ByteInStream >	jau::io::to_ByteInStream (const std::string &path_or_uri, jau::fraction_i64 timeout=20_s) noexcept
	Parses the given path_or_uri, if it matches a supported protocol, see jau::io::uri::protocol_supported(), but is not a local file, see jau::io::uri::is_local_file_protocol(), ByteInStream_URL is being attempted.

std::string	jau::io::toString (io_result_t v) noexcept

Variables
const size_t	jau::io::BEST_URLSTREAM_RINGBUFFER_SIZE = 2_uz * 16384_uz

Detailed Description

Input and Output (IO) types and functionality.

Typedef Documentation

◆ secure_vector

template<typename T>

using jau::io::secure_vector = std::vector<T, jau::callocator_sec<T>>

Definition at line 46 of file io_util.hpp.

◆ secure_string

typedef std::basic_string<char, std::char_traits<char>, jau::callocator_sec<char> > jau::io::secure_string

Definition at line 48 of file io_util.hpp.

◆ ByteRingbuffer

typedef jau::ringbuffer<uint8_t, size_t> jau::io::ByteRingbuffer

Definition at line 50 of file io_util.hpp.

◆ relaxed_atomic_io_result_t

typedef jau::ordered_atomic<io_result_t, std::memory_order_relaxed> jau::io::relaxed_atomic_io_result_t

Definition at line 78 of file io_util.hpp.

◆ StreamConsumerFunc

typedef jau::function<bool(secure_vector<uint8_t>& , bool )> jau::io::StreamConsumerFunc

Stream consumer function.

bool consumer(secure_vector<uint8_t>& data, bool is_final)

Returns true to signal continuation, false to end streaming.

Definition at line 98 of file io_util.hpp.

◆ net_tk_handle

using jau::io::net_tk_handle = void*

Definition at line 239 of file io_util.hpp.

◆ SyncStreamResponseRef

using jau::io::SyncStreamResponseRef = std::shared_ptr<SyncStreamResponse>

Definition at line 283 of file io_util.hpp.

◆ SyncStreamConsumerFunc

typedef jau::function<bool(SyncStreamResponse& , const uint8_t* , size_t , bool )> jau::io::SyncStreamConsumerFunc

Synchronous stream consumer function.

bool consumer(AsyncStreamResponse& resp, const uint8_t* data , size_t len, bool is_final)

Returns true to signal continuation, false to end streaming.

Definition at line 291 of file io_util.hpp.

◆ AsyncStreamResponseRef

using jau::io::AsyncStreamResponseRef = std::shared_ptr<AsyncStreamResponse>

Definition at line 364 of file io_util.hpp.

◆ AsyncStreamConsumerFunc

typedef jau::function<bool(AsyncStreamResponse& , const uint8_t* , size_t , bool )> jau::io::AsyncStreamConsumerFunc

Asynchronous stream consumer function.

bool consumer(AsyncStreamResponse& resp, const uint8_t* data , size_t len, bool is_final)

Returns true to signal continuation, false to end streaming.

Definition at line 372 of file io_util.hpp.

Enumeration Type Documentation

◆ iostate

enum class jau::io::iostate : uint32_t

strong

Enumerator
none	No error occurred nor has EOS being reached. Value is no bit set!
goodbit	No error occurred nor has EOS being reached. Value is no bit set!
badbit	Irrecoverable stream error, including loss of integrity of the underlying stream or media.
eofbit	An input operation reached the end of its stream.
failbit	Input or output operation failed (formatting or extraction error).
timeout	Input or output operation failed due to timeout.

Definition at line 55 of file byte_stream.hpp.

◆ io_dir_t

enum class jau::io::io_dir_t : int8_t

strong

I/O direction, read or write.

Enumerator
READ	Read Operation.
WRITE	Write Operation.

Definition at line 57 of file io_util.hpp.

◆ io_result_t

enum class jau::io::io_result_t : int8_t

strong

I/O operation result value.

Enumerator
FAILED	Operation failed.
NONE	Operation still in progress.
SUCCESS	Operation succeeded.

Definition at line 68 of file io_util.hpp.

Function Documentation

◆ JAU_MAKE_BITFIELD_ENUM_STRING()

jau::io::JAU_MAKE_BITFIELD_ENUM_STRING	(	iostate	,
		goodbit	,
		badbit	,
		eofbit	,
		failbit	,
		timeout	)

◆ to_ByteInStream()

std::unique_ptr< ByteInStream > jau::io::to_ByteInStream	(	const std::string &	path_or_uri,
		jau::fraction_i64	timeout = 20_s )

noexcept

Parses the given path_or_uri, if it matches a supported protocol, see jau::io::uri::protocol_supported(), but is not a local file, see jau::io::uri::is_local_file_protocol(), ByteInStream_URL is being attempted.

If the above fails, ByteInStream_File is attempted.

If non of the above leads to a ByteInStream without ByteInStream::fail(), nullptr is returned.

Parameters

path_or_uri	given path or uri for with a ByteInStream instance shall be established.
timeout	in case `path_or_uri` resolves to ByteInStream_URL, timeout is being used as maximum duration to wait for next bytes at ByteInStream_URL::available(), defaults to 20_s

Returns: a working ByteInStream w/o ByteInStream::fail() or nullptr

Definition at line 415 of file byte_stream.cpp.

Here is the caller graph for this function:

◆ toString()

std::string jau::io::toString ( io_result_t v )

inlinenoexcept

Definition at line 80 of file io_util.hpp.

Here is the caller graph for this function:

◆ operator<<()

std::ostream & jau::io::operator<<	(	std::ostream &	os,
		io_result_t	v )

inline

Definition at line 87 of file io_util.hpp.

◆ read_file()

uint64_t jau::io::read_file	(	const std::string &	input_file,
		secure_vector< uint8_t > &	buffer,
		const StreamConsumerFunc &	consumer_fn )

noexcept

Synchronous byte input stream reader from given file path using the given StreamConsumerFunc consumer_fn.

To abort streaming, user may return false from the given consumer_func.

It is guaranteed that consumer_fn() is called with is_final=true once at the end, even if input_file stream has zero size.

Parameters

input_file	input file name path, `-` denotes std::stdin.
buffer	secure std::vector buffer, passed down to consumer_fn
consumer_fn	StreamConsumerFunc consumer for each received heap of bytes, returning true to continue stream of false to abort.

Returns: total bytes read or 0 if error

Definition at line 48 of file io_util.cpp.

◆ read_stream() [1/2]

uint64_t jau::io::read_stream	(	ByteInStream &	in,
		secure_vector< uint8_t > &	buffer,
		const StreamConsumerFunc &	consumer_fn )

noexcept

Synchronous byte input stream reader using the given StreamConsumerFunc consumer_fn.

To abort streaming, user may return false from the given consumer_func.

It is guaranteed that consumer_fn() is called with is_final=true once at the end, even input stream has zero size.

Parameters

in	the input byte stream to read from
buffer	secure std::vector buffer, passed down to consumer_fn
consumer_fn	StreamConsumerFunc consumer for each received heap of bytes, returning true to continue stream of false to abort.

Returns: total bytes read or 0 if error

Definition at line 61 of file io_util.cpp.

Here is the caller graph for this function:

◆ read_stream() [2/2]

uint64_t jau::io::read_stream	(	ByteInStream &	in,
		secure_vector< uint8_t > &	buffer1,
		secure_vector< uint8_t > &	buffer2,
		const StreamConsumerFunc &	consumer_fn )

noexcept

Synchronous double-buffered byte input stream reader using the given StreamConsumerFunc consumer_fn.

To abort streaming, user may return false from the given consumer_func.

It is guaranteed that consumer_fn() is called with is_final=true once at the end, even if input stream has zero size.

Implementation reads one buffer ahead in respect to consumer_fn().
If reading zero bytes on the next buffer, it propagates the end-of-file (EOF) to the previous buffer which will be send via consumer_fn() next.

This way, the consumer_fn() will always receive its is_final flag on the last sent bytes (size > 0) even if the content-size is unknown (pipe).
Hence it allows e.g. decryption to work where the final data chunck must be processed as such.

Parameters

in	the input byte stream to read from
buffer1	secure std::vector buffer, passed down to consumer_fn
buffer2	secure std::vector buffer, passed down to consumer_fn
consumer_fn	StreamConsumerFunc consumer for each received heap of bytes, returning true to continue stream of false to abort.

Returns: total bytes read or 0 if error

Definition at line 102 of file io_util.cpp.

◆ read_url_stream()

uint64_t jau::io::read_url_stream	(	const std::string &	url,
		secure_vector< uint8_t > &	buffer,
		const StreamConsumerFunc &	consumer_fn )

noexcept

Synchronous URL stream reader using the given StreamConsumerFunc consumer_fn.

To abort streaming, user may return false from the given consumer_func.

Standard implementation uses curl, hence all libcurl network protocols are supported, see jau::io::uri::supported_protocols().

If the uri-sheme doesn't match a supported protocol, see jau::io::uri::protocol_supported(), function returns immediately with zero bytes.

Environment variables:

jau_io_net_ssl_verifypeer=[true|false] to enable or disable SSL peer verification. Defaults to true.
jau_io_net_verbose=[true|false] to enable or disable verbose on stderr stream communication. Defaults to false.

Parameters

url	the URL to open a connection to and stream bytes from
buffer	secure std::vector buffer, passed down to consumer_fn
consumer_fn	StreamConsumerFunc consumer for each received heap of bytes, returning true to continue stream of false to abort.

Returns: total bytes read or 0 if transmission error or protocol of given url is not supported

Definition at line 313 of file io_util.cpp.

Here is the caller graph for this function:

◆ create_net_tk_handle()

net_tk_handle jau::io::create_net_tk_handle ( )

noexcept

creates a reusable handle, free with free_net_tk_handle() after use.

Definition at line 1072 of file io_util.cpp.

Here is the caller graph for this function:

◆ free_net_tk_handle()

void jau::io::free_net_tk_handle ( net_tk_handle handle )

noexcept

frees a handle after use created by create_net_tk_handle()

Definition at line 1081 of file io_util.cpp.

Here is the caller graph for this function:

◆ read_url_stream_sync()

SyncStreamResponseRef jau::io::read_url_stream_sync	(	net_tk_handle	handle,
		const std::string &	url,
		http::PostRequestPtr	httpPostReq,
		ByteRingbuffer *	buffer,
		const SyncStreamConsumerFunc &	consumer_fn )

noexcept

Synchronous URL stream reader using the given SyncStreamConsumerFunc consumer_fn.

Function returns after completion.

To abort streaming, (1) user may return false from the given consumer_func. Asynchronous URL read content using the given byte jau::ringbuffer, allowing parallel reading.

To abort streaming, (2) user may set given reference results to a value other than async_io_result_t::NONE.

Standard implementation uses curl, hence all libcurl network protocols are supported, see jau::io::uri::supported_protocols().

If the uri-sheme doesn't match a supported protocol, see jau::io::uri::protocol_supported(), SyncStreamResponse::failed() returns true.

Environment variables:

jau_io_net_ssl_verifypeer=[true|false] to enable or disable SSL peer verification. Defaults to true.
jau_io_net_verbose=[true|false] to enable or disable verbose on stderr stream communication. Defaults to false.

Parameters

handle	optional reused user-pooled net toolkit handle, see create_net_tk_handle(). Pass nullptr to use an own local handle.
url	the URL to open a connection to and stream bytes from
httpPostReq	optional HTTP POST request data, maybe nullptr
buffer	optional jau::ringbuffer<uint8_t>, if not nullptr will be synchronously filled with received data
consumer_fn	SyncStreamConsumerFunc consumer for each received heap of bytes, returning true to continue stream of false to abort.

Returns: new SyncStreamResponse reference. If protocol of given url is not supported, SyncStreamResponse::failed() returns true.

Definition at line 1093 of file io_util.cpp.

Here is the caller graph for this function:

◆ read_url_stream_async()

AsyncStreamResponseRef jau::io::read_url_stream_async	(	net_tk_handle	handle,
		const std::string &	url,
		http::PostRequestPtr	httpPostReq,
		ByteRingbuffer *	buffer,
		const AsyncStreamConsumerFunc &	consumer_fn )

noexcept

Asynchronous URL stream reader using the given AsyncStreamConsumerFunc consumer_fn.

Function returns immediately.

To abort streaming, (1) user may return false from the given consumer_func. Asynchronous URL read content using the given byte jau::ringbuffer, allowing parallel reading.

To abort streaming, (2) user may set given reference results to a value other than async_io_result_t::NONE.

Standard implementation uses curl, hence all libcurl network protocols are supported, see jau::io::uri::supported_protocols().

If the uri-sheme doesn't match a supported protocol, see jau::io::uri::protocol_supported(), AsyncStreamResponse::failed() returns true.

Environment variables:

jau_io_net_ssl_verifypeer=[true|false] to enable or disable SSL peer verification. Defaults to true.
jau_io_net_verbose=[true|false] to enable or disable verbose on stderr stream communication. Defaults to false.

Parameters

handle	optional reused user-pooled net toolkit handle, see create_net_tk_handle(). Pass nullptr to use an own local handle.
url	the URL to open a connection to and stream bytes from
httpPostReq	optional HTTP POST request data, maybe nullptr
buffer	optional jau::ringbuffer<uint8_t>, if not nullptr will be asynchronously filled with received data
consumer_fn	AsyncStreamConsumerFunc consumer for each received heap of bytes, returning true to continue stream of false to abort.

Returns: new AsyncStreamResponse reference. If protocol of given url is not supported, AsyncStreamResponse::failed() returns true.

Definition at line 1119 of file io_util.cpp.

Here is the caller graph for this function:

◆ print_stats()

void jau::io::print_stats	(	const std::string &	prefix,
		const uint64_t &	out_bytes_total,
		const jau::fraction_i64 &	td )

noexcept

Definition at line 1149 of file io_util.cpp.

Here is the caller graph for this function:

◆ supported_protocols()

std::vector< std::string_view > jau::io::uri_tk::supported_protocols ( )

noexcept

Returns a list of supported protocol supported by libcurl network protocols, queried at runtime.

See also: protocol_supported()

Definition at line 161 of file io_util.cpp.

Here is the caller graph for this function:

◆ get_scheme()

std::string_view jau::io::uri_tk::get_scheme ( const std::string_view & uri )

noexcept

Returns the valid uri-scheme from given uri, which is empty if no valid scheme is included.

The given uri must include at least a colon after the uri-scheme part.

Parameters

uri an uri

Returns: valid uri-scheme, empty if non found

Definition at line 184 of file io_util.cpp.

Here is the caller graph for this function:

◆ protocol_supported()

bool jau::io::uri_tk::protocol_supported ( const std::string_view & uri )

noexcept

Returns true if the uri-scheme of given uri matches a supported by libcurl network protocols otherwise false.

The uri-scheme is retrieved via get_scheme() passing given uri, hence must include at least a colon after the uri-scheme part.

The libcurl supported protocols is queried at runtime, see supported_protocols().

Parameters

uri	an uri to test

Returns: true if the uri-scheme of given uri is supported, otherwise false.

See also: supported_protocols(); get_scheme()

Definition at line 196 of file io_util.cpp.

Here is the caller graph for this function:

◆ is_local_file_protocol()

bool jau::io::uri_tk::is_local_file_protocol ( const std::string_view & uri )

noexcept

Returns true if the uri-scheme of given uri matches the local file protocol, i.e.

starts with file://.

Parameters

uri	an uri to test

Definition at line 206 of file io_util.cpp.

Here is the caller graph for this function:

◆ is_httpx_protocol()

bool jau::io::uri_tk::is_httpx_protocol ( const std::string_view & uri )

noexcept

Returns true if the uri-scheme of given uri matches the http or https protocol, i.e.

starts with http: or https:.

Parameters

uri	an uri to test

Definition at line 210 of file io_util.cpp.

Variable Documentation

◆ BEST_URLSTREAM_RINGBUFFER_SIZE

const size_t jau::io::BEST_URLSTREAM_RINGBUFFER_SIZE = 2_uz * 16384_uz

extern

Definition at line 72 of file byte_stream.cpp.

Namespaces

Classes

Typedefs

Enumerations

Functions

Variables

Detailed Description

Typedef Documentation

◆ secure_vector

◆ secure_string

◆ ByteRingbuffer

◆ relaxed_atomic_io_result_t

◆ StreamConsumerFunc

◆ net_tk_handle

◆ SyncStreamResponseRef

◆ SyncStreamConsumerFunc

◆ AsyncStreamResponseRef

◆ AsyncStreamConsumerFunc

Enumeration Type Documentation

◆ iostate

◆ io_dir_t

◆ io_result_t

Function Documentation

◆ JAU_MAKE_BITFIELD_ENUM_STRING()

◆ to_ByteInStream()

◆ toString()

◆ operator<<()

◆ read_file()

◆ read_stream() [1/2]

◆ read_stream() [2/2]

◆ read_url_stream()

◆ create_net_tk_handle()

◆ free_net_tk_handle()

◆ read_url_stream_sync()

◆ read_url_stream_async()

◆ print_stats()

◆ supported_protocols()

◆ get_scheme()

◆ protocol_supported()

◆ is_local_file_protocol()

◆ is_httpx_protocol()

Variable Documentation

◆ BEST_URLSTREAM_RINGBUFFER_SIZE