jaulib v1.3.0
Jau Support Library (C++, Java, ..)
TestByteStream01.java
Go to the documentation of this file.
1/*
2 * Author: Sven Gothel <sgothel@jausoft.com>
3 * Copyright (c) 2022 Gothel Software e.K.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining
6 * a copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sublicense, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be
14 * included in all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
23 */
24
25package jau.test.io;
26
27import java.nio.ByteBuffer;
28import java.nio.charset.Charset;
29import java.nio.file.Path;
30import java.nio.file.Paths;
31import java.util.ArrayList;
32import java.util.Arrays;
33import java.util.List;
34
35import org.jau.fs.FMode;
36import org.jau.fs.FileStats;
37import org.jau.fs.FileUtil;
38import org.jau.fs.TraverseOptions;
39import org.jau.io.Buffers;
40import org.jau.io.ByteInStream;
41import org.jau.io.ByteInStreamUtil;
42import org.jau.io.ByteInStream_Feed;
43import org.jau.io.ByteInStream_File;
44import org.jau.io.ByteInStream_URL;
45import org.jau.io.ByteOutStream_File;
46import org.jau.io.PrintUtil;
47import org.jau.io.UriTk;
48import org.junit.AfterClass;
49import org.junit.Assert;
50import org.junit.FixMethodOrder;
51import org.junit.Test;
52import org.junit.runners.MethodSorters;
53
54import jau.pkg.PlatformRuntime;
55import jau.test.junit.util.JunitTracer;
56
57@FixMethodOrder(MethodSorters.NAME_ASCENDING)
58public class TestByteStream01 extends JunitTracer {
59 static final boolean DEBUG = false;
60
61 static final String payload_version = "0";
62 static final String payload_version_parent = "0";
63
64 static final int IDX_11kiB = 0;
65 static final int IDX_65MiB = 1;
66 static List<String> fname_payload_lst = new ArrayList<String>();
67 static List<String> fname_payload_copy_lst = new ArrayList<String>();
68 static List<Long> fname_payload_size_lst = new ArrayList<Long>();
69
70 static boolean file_exists(final String name) {
71 final FileStats stats = new FileStats( name );
72 return stats.is_file();
73 }
74
75 static long file_size(final String name) {
76 final FileStats stats = new FileStats( name );
77 return stats.size();
78 }
79
80 static boolean remove_file(final String name) {
81 final FileStats stats = new FileStats( name );
82 try {
83 if( stats.is_file() ) {
84 if( !FileUtil.remove(name, TraverseOptions.none) ) {
85 PrintUtil.println(System.err, "Remove.1: Failed deletion of existing file "+name);
86 return false;
87 }
88 return true;
89 }
90 return !stats.exists();
91 } catch (final Exception ex) {
92 PrintUtil.println(System.err, "Remove.2: Failed deletion of existing file "+name+": "+ex.getMessage());
93 ex.printStackTrace();
94 }
95 return false;
96 }
97
98 static boolean add_test_file(final String name, final long size_limit) {
99 Assert.assertTrue( remove_file(name) );
100 Assert.assertTrue( remove_file(name+".enc") );
101 Assert.assertTrue( remove_file(name+".enc.dec") );
102 long size = 0;
103 {
104 final String one_line = "Hello World, this is a test and I like it. Exactly 100 characters long. 0123456780 abcdefghjklmnop..";
105 final Charset charset = Charset.forName("ASCII");
106 final byte[] one_line_bytes = one_line.getBytes(charset);
107
108 Assert.assertFalse( file_exists(name) );
109 try (ByteOutStream_File out = new ByteOutStream_File(name, FMode.def_file)) {
110 Assert.assertTrue( out.good() );
111 Assert.assertTrue( out.is_open() );
112 {
113 final int line_len = one_line_bytes.length;
114 for(size=0; size < size_limit; size+=line_len) {
115 if( line_len != out.write( one_line_bytes, 0, line_len ) ) {
116 PrintUtil.fprintf_td(System.err, "Write %d bytes to test file failed: %s", line_len, out.toString());
117 return false;
118 }
119 }
120 if( 1 != out.write( one_line_bytes, 0, 1 ) ) { // make it odd
121 PrintUtil.fprintf_td(System.err, "Write %d bytes to test file failed: %s", 1, out.toString());
122 return false;
123 }
124 size += 1;
125 }
126 }
127 }
128 fname_payload_lst.add(name);
129 fname_payload_copy_lst.add(name+".copy");
130 fname_payload_size_lst.add( Long.valueOf(size) );
131 return true;
132 }
133
134 static {
136 Assert.assertTrue( add_test_file("test_cipher_01_11kiB.bin", 1024*11) );
137 Assert.assertTrue( add_test_file("test_cipher_02_65MiB.bin", 1024*1024*65) );
138 }
139
140 static boolean system(final String[] command) {
141 Process proc = null;
142 try{
143 proc = Runtime.getRuntime().exec(command);
144 proc.waitFor();
145 return true;
146 }
147 catch(final Exception ex)
148 {
149 if(proc!=null) {
150 proc.destroy();
151 }
152 ex.printStackTrace();
153 }
154 return false;
155 }
156
157 static final String mini_httpd_exe() {
158 final String os_name = System.getProperty("os.name");
159 if( "FreeBSD".equals(os_name) ) {
160 return "/usr/local/sbin/mini_httpd";
161 } else {
162 return "/usr/sbin/mini_httpd";
163 }
164 }
165
166 @AfterClass
167 public static void httpd_stop() {
168 if( UriTk.protocol_supported("http:") ) {
169 Assert.assertTrue( system(new String[]{"killall", "mini_httpd"}) );
170 }
171 }
172
173 static void httpd_start() {
174 if( UriTk.protocol_supported("http:") ) {
175 Assert.assertTrue( system(new String[]{"killall", "mini_httpd"}) );
176 final Path path = Paths.get("");
177 final String directoryName = path.toAbsolutePath().toString();
178 final String[] cmd = new String[]{mini_httpd_exe(), "-p", "8080", "-l", directoryName+"/mini_httpd.log"};
179 PrintUtil.fprintf_td(System.err, "%s%n", Arrays.toString(cmd));
180 Assert.assertTrue( system(cmd) );
181 }
182 }
183
184 final static String url_input_root = "http://localhost:8080/";
185
186
187 static boolean transfer_std(final ByteInStream input, final String output_fname, final int buffer_size) {
188 final long _t0 = org.jau.sys.Clock.currentTimeMillis();
189 PrintUtil.fprintf_td(System.err, "Transfer Start: %s%n", input);
190 remove_file(output_fname);
191
192 if( file_exists( output_fname ) ) {
193 return false;
194 }
195 final long[] out_bytes_payload = { 0 };
196 try ( ByteOutStream_File out = new ByteOutStream_File(output_fname, FMode.def_file) ) {
197 // final ByteOutStream_File[] out = { _out };
198 Assert.assertTrue( out.good() );
199
200 final ByteInStreamUtil.StreamConsumer1 consumer = (final byte[] data, final int data_len, final boolean is_final) -> {
201 if( !is_final && ( !input.has_content_size() || out_bytes_payload[0] + data_len < input.content_size() ) ) {
202 final int written = out.write( data, 0, data_len );
203 out_bytes_payload[0] += written;
204 return data_len == written; // continue ..
205 } else {
206 final int written = out.write( data, 0, data_len );
207 out_bytes_payload[0] += written;
208 return false; // EOS
209 }
210 };
211 final byte[] io_buffer = new byte[buffer_size];
212 final long in_bytes_total = ByteInStreamUtil.read_stream(input, io_buffer, consumer);
213 input.closeStream();
214 out.closeStream();
215
216 if ( 0==in_bytes_total || input.fail() ) {
217 PrintUtil.fprintf_td(System.err, "ByteStream copy failed: Input file read failed in %s, out %s%n", input, out);
218 return false;
219 }
220 if ( out.fail() ) {
221 PrintUtil.fprintf_td(System.err, "ByteStream copy failed: Output file write failed in %s, out %s%n", input, out);
222 return false;
223 }
224 }
225
226 final long _td = org.jau.sys.Clock.currentTimeMillis() - _t0;
227 ByteInStreamUtil.print_stats("Transfer "+output_fname, out_bytes_payload[0], _td);
228 PrintUtil.fprintf_td(System.err, "Transfer End: %s%n", input.toString());
229
230 return true;
231 }
232
233 static boolean transfer_nio(final ByteInStream input, final String output_fname, final int buffer_size) {
234 final long _t0 = org.jau.sys.Clock.currentTimeMillis();
235 PrintUtil.fprintf_td(System.err, "Transfer Start: %s%n", input);
236 remove_file(output_fname);
237
238 if( file_exists( output_fname ) ) {
239 return false;
240 }
241 final long[] out_bytes_payload = { 0 };
242 try ( ByteOutStream_File out = new ByteOutStream_File(output_fname, FMode.def_file) ) {
243 Assert.assertTrue( out.good() );
244
245 final ByteInStreamUtil.StreamConsumer2 consumer = (final ByteBuffer data, final boolean is_final) -> {
246 final int data_len = data.remaining();
247 if( !is_final && ( !input.has_content_size() || out_bytes_payload[0] + data_len < input.content_size() ) ) {
248 final int written = out.write(data);
249 data.rewind();
250 out_bytes_payload[0] += written;
251 return written == data_len; // continue ..
252 } else {
253 final int written = out.write(data);
254 data.rewind();
255 out_bytes_payload[0] += written;
256 return false; // EOS
257 }
258 };
259 final ByteBuffer io_buffer = Buffers.newDirectByteBuffer(buffer_size);
260 final long in_bytes_total = ByteInStreamUtil.read_stream(input, io_buffer, consumer);
261 input.closeStream();
262 out.closeStream();
263
264 if ( 0==in_bytes_total || input.fail() ) {
265 PrintUtil.fprintf_td(System.err, "ByteStream copy failed: Input file read failed in %s, out %s%n", input, out);
266 return false;
267 }
268 if ( out.fail() ) {
269 PrintUtil.fprintf_td(System.err, "ByteStream copy failed: Output file write failed in %s, out %s%n", input, out);
270 return false;
271 }
272 }
273
274 final long _td = org.jau.sys.Clock.currentTimeMillis() - _t0;
275 ByteInStreamUtil.print_stats("Transfer "+output_fname, out_bytes_payload[0], _td);
276 PrintUtil.fprintf_td(System.err, "Transfer End: %s%n", input.toString());
277
278 return true;
279 }
280
281 @Test(timeout = 10000)
282 public final void test00a_protocols_error() {
284 final boolean http_support_expected = UriTk.protocol_supported("http:");
285 final boolean file_support_expected = UriTk.protocol_supported("file:");
286 httpd_start();
287 {
288 final List<String> protos = UriTk.supported_protocols();
289 PrintUtil.fprintf_td(System.err, "test00_protocols: Supported protocols: %d: %s%n", protos.size(), protos);
290 if( http_support_expected ) { // assume no http -> no curl
291 Assert.assertTrue( 0 < protos.size() );
292 } else {
293 Assert.assertTrue( 0 == protos.size() );
294 }
295 }
296 final int file_idx = IDX_11kiB;
297 {
298 final String url = "not_exiting_file.txt";
299 Assert.assertFalse( UriTk.is_local_file_protocol(url) );
300 Assert.assertFalse( UriTk.protocol_supported(url) );
301
302 try( final ByteInStream in = ByteInStreamUtil.to_ByteInStream(url) ) {
303 if( null != in ) {
304 PrintUtil.fprintf_td(System.err, "test00_protocols: not_exiting_file: %s%n", in);
305 }
306 Assert.assertNull(in);
307 }
308 }
309 {
310 final String url = "file://not_exiting_file_uri.txt";
311 Assert.assertTrue( UriTk.is_local_file_protocol(url) );
312 Assert.assertEquals( file_support_expected, UriTk.protocol_supported(url) );
313
314 try( final ByteInStream in = ByteInStreamUtil.to_ByteInStream(url) ) {
315 if( null != in ) {
316 PrintUtil.fprintf_td(System.err, "test00_protocols: not_exiting_file_uri: %s%n", in);
317 }
318 Assert.assertNull(in);
319 }
320 }
321 {
322 final String url = "lala://localhost:8080/" + fname_payload_lst.get(file_idx);
323 Assert.assertFalse( UriTk.is_local_file_protocol(url) );
324 Assert.assertFalse( UriTk.protocol_supported(url) );
325
326 try( final ByteInStream in = ByteInStreamUtil.to_ByteInStream(url) ) {
327 if( null != in ) {
328 PrintUtil.fprintf_td(System.err, "test00_protocols: not_exiting_protocol_uri: %s%n", in);
329 }
330 Assert.assertNull(in);
331 }
332 }
333 {
334 final String url = url_input_root + "not_exiting_http_uri.txt";
335 Assert.assertFalse( UriTk.is_local_file_protocol(url) );
336 Assert.assertEquals( http_support_expected, UriTk.protocol_supported(url) );
337
338 try( final ByteInStream in = ByteInStreamUtil.to_ByteInStream(url) ) {
339 if( http_support_expected ) {
340 Assert.assertNotNull(in);
341 try { Thread.sleep(100); } catch (final Throwable t) {} // time to read 404 response
342 PrintUtil.fprintf_td(System.err, "test00_protocols: not_exiting_http_uri: %s%n", in);
343 Assert.assertFalse( in.good() );
344 Assert.assertTrue( in.fail() );
345 Assert.assertEquals( 0, in.content_size() );
346 } else {
347 Assert.assertNull(in);
348 }
349 }
350 }
351 }
352
353 @Test(timeout = 10000)
354 public final void test00b_protocols_ok() {
356 final boolean http_support_expected = UriTk.protocol_supported("http:");
357 final boolean file_support_expected = UriTk.protocol_supported("file:");
358 httpd_start();
359 final int file_idx = IDX_11kiB;
360 {
361 final String url = fname_payload_lst.get(file_idx);
362 Assert.assertFalse( UriTk.is_local_file_protocol(url) );
363 Assert.assertFalse( UriTk.protocol_supported(url) );
364
365 try( final ByteInStream in = ByteInStreamUtil.to_ByteInStream(url) ) {
366 if( null != in ) {
367 PrintUtil.fprintf_td(System.err, "test00_protocols: local-file-0: %s%n", in);
368 }
369 Assert.assertNotEquals( null, in );
370 Assert.assertFalse( in.fail() );
371
372 final boolean res = transfer_nio(in, fname_payload_copy_lst.get(file_idx), 4096);
373 Assert.assertTrue( res );
374
375 Assert.assertTrue( file_exists( fname_payload_copy_lst.get(file_idx) ) );
376 final long copy_size = file_size(fname_payload_copy_lst.get(file_idx));
377 Assert.assertEquals( in.content_size(), copy_size );
378 Assert.assertEquals( fname_payload_size_lst.get(file_idx).longValue(), copy_size );
379 Assert.assertTrue( FileUtil.compare(in.id(), fname_payload_copy_lst.get(file_idx), true /* verbose */) );
380 }
381 }
382 {
383 final String url = "file://" + fname_payload_lst.get(file_idx);
384 Assert.assertTrue( UriTk.is_local_file_protocol(url) );
385 Assert.assertEquals( file_support_expected, UriTk.protocol_supported(url) );
386
387 try( final ByteInStream in = ByteInStreamUtil.to_ByteInStream(url) ) {
388 if( null != in ) {
389 PrintUtil.fprintf_td(System.err, "test00_protocols: local-file-1: %s%n", in);
390 }
391 Assert.assertNotNull( in );
392 Assert.assertFalse( in.fail() );
393
394 final boolean res = transfer_nio(in, fname_payload_copy_lst.get(file_idx), 4096);
395 Assert.assertTrue( res );
396
397 Assert.assertTrue( file_exists( fname_payload_copy_lst.get(file_idx) ) );
398 final long copy_size = file_size(fname_payload_copy_lst.get(file_idx));
399 Assert.assertEquals( in.content_size(), copy_size );
400 Assert.assertEquals( fname_payload_size_lst.get(file_idx).longValue(), copy_size );
401 Assert.assertTrue( FileUtil.compare(fname_payload_lst.get(file_idx), fname_payload_copy_lst.get(file_idx), true /* verbose */) );
402 }
403 }
404 {
405 final String url = url_input_root + fname_payload_lst.get(file_idx);
406 Assert.assertFalse( UriTk.is_local_file_protocol(url) );
407 Assert.assertEquals( http_support_expected, UriTk.protocol_supported(url) );
408
409 try( final ByteInStream in = ByteInStreamUtil.to_ByteInStream(url) ) {
410 if( null != in ) {
411 PrintUtil.fprintf_td(System.err, "test00_protocols: http: %s%n", in);
412 }
413 if( http_support_expected ) {
414 Assert.assertNotNull( in );
415 Assert.assertFalse( in.fail() );
416
417 final boolean res = transfer_nio(in, fname_payload_copy_lst.get(file_idx), 4096);
418 Assert.assertTrue( res );
419
420 Assert.assertTrue( file_exists( fname_payload_copy_lst.get(file_idx) ) );
421 final long copy_size = file_size(fname_payload_copy_lst.get(file_idx));
422 Assert.assertEquals( in.content_size(), copy_size );
423 Assert.assertEquals( fname_payload_size_lst.get(file_idx).longValue(), copy_size );
424 Assert.assertTrue( FileUtil.compare(fname_payload_lst.get(file_idx), fname_payload_copy_lst.get(file_idx), true /* verbose */) );
425 } else {
426 Assert.assertNull( in );
427 }
428 }
429 }
430 }
431
432 @Test(timeout = 10000)
433 public void test01_copy_file_ok_11kiB() {
435 final int file_idx = IDX_11kiB;
436 try( final ByteInStream_File data_stream = new ByteInStream_File(fname_payload_lst.get(file_idx)) ) {
437 final boolean res = transfer_nio(data_stream, fname_payload_copy_lst.get(file_idx), 4096);
438 Assert.assertTrue( res );
439
440 Assert.assertTrue( file_exists( fname_payload_copy_lst.get(file_idx) ) );
441 final long copy_size = file_size(fname_payload_copy_lst.get(file_idx));
442 Assert.assertEquals( data_stream.content_size(), copy_size );
443 Assert.assertEquals( fname_payload_size_lst.get(file_idx).longValue(), copy_size );
444 Assert.assertTrue( FileUtil.compare(fname_payload_lst.get(file_idx), fname_payload_copy_lst.get(file_idx), true /* verbose */) );
445 }
446 }
447
448 @Test(timeout = 10000)
449 public void test02_copy_file_ok_65MiB_nio_buff4k() {
451 final int file_idx = IDX_65MiB;
452 try( final ByteInStream_File data_stream = new ByteInStream_File(fname_payload_lst.get(file_idx)) ) {
453 final boolean res = transfer_nio(data_stream, fname_payload_copy_lst.get(file_idx), 4096);
454 Assert.assertTrue( res );
455
456 Assert.assertTrue( file_exists( fname_payload_copy_lst.get(file_idx) ) );
457 final long copy_size = file_size(fname_payload_copy_lst.get(file_idx));
458 Assert.assertEquals( data_stream.content_size(), copy_size );
459 Assert.assertEquals( fname_payload_size_lst.get(file_idx).longValue(), copy_size );
460 Assert.assertTrue( FileUtil.compare(fname_payload_lst.get(file_idx), fname_payload_copy_lst.get(file_idx), true /* verbose */) );
461 }
462 }
463
464 @Test(timeout = 10000)
465 public void test03_copy_file_ok_65MiB_std_buff4k() {
467 final int file_idx = IDX_65MiB;
468 try( final ByteInStream_File data_stream = new ByteInStream_File(fname_payload_lst.get(file_idx)) ) {
469 final boolean res = transfer_std(data_stream, fname_payload_copy_lst.get(file_idx), 4096);
470 Assert.assertTrue( res );
471
472 Assert.assertTrue( file_exists( fname_payload_copy_lst.get(file_idx) ) );
473 final long copy_size = file_size(fname_payload_copy_lst.get(file_idx));
474 Assert.assertEquals( data_stream.content_size(), copy_size );
475 Assert.assertEquals( fname_payload_size_lst.get(file_idx).longValue(), copy_size );
476 Assert.assertTrue( FileUtil.compare(fname_payload_lst.get(file_idx), fname_payload_copy_lst.get(file_idx), true /* verbose */) );
477 }
478 }
479
480 @Test(timeout = 10000)
481 public void test04_copy_file_ok_65MiB_nio_buff32k() {
483 final int file_idx = IDX_65MiB;
484 try( final ByteInStream_File data_stream = new ByteInStream_File(fname_payload_lst.get(file_idx)) ) {
485 final boolean res = transfer_nio(data_stream, fname_payload_copy_lst.get(file_idx), 32768);
486 Assert.assertTrue( res );
487
488 Assert.assertTrue( file_exists( fname_payload_copy_lst.get(file_idx) ) );
489 final long copy_size = file_size(fname_payload_copy_lst.get(file_idx));
490 Assert.assertEquals( data_stream.content_size(), copy_size );
491 Assert.assertEquals( fname_payload_size_lst.get(file_idx).longValue(), copy_size );
492 Assert.assertTrue( FileUtil.compare(fname_payload_lst.get(file_idx), fname_payload_copy_lst.get(file_idx), true /* verbose */) );
493 }
494 }
495
496 @Test(timeout = 10000)
497 public void test05_copy_file_ok_65MiB_std_buff32k() {
499 final int file_idx = IDX_65MiB;
500 try( final ByteInStream_File data_stream = new ByteInStream_File(fname_payload_lst.get(file_idx)) ) {
501 final boolean res = transfer_std(data_stream, fname_payload_copy_lst.get(file_idx), 32768);
502 Assert.assertTrue( res );
503
504 Assert.assertTrue( file_exists( fname_payload_copy_lst.get(file_idx) ) );
505 final long copy_size = file_size(fname_payload_copy_lst.get(file_idx));
506 Assert.assertEquals( data_stream.content_size(), copy_size );
507 Assert.assertEquals( fname_payload_size_lst.get(file_idx).longValue(), copy_size );
508 Assert.assertTrue( FileUtil.compare(fname_payload_lst.get(file_idx), fname_payload_copy_lst.get(file_idx), true /* verbose */) );
509 }
510 }
511
512 @Test(timeout = 10000)
513 public void test11_copy_http_ok_nio_buff32k() {
515 if( !UriTk.protocol_supported("http:") ) {
516 PrintUtil.fprintf_td(System.err, "http not supported, abort%n");
517 return;
518 }
519 httpd_start();
520 {
521 final int file_idx = IDX_11kiB;
522
523 final String uri_original = url_input_root + fname_payload_lst.get(file_idx);
524
525 try( final ByteInStream_URL data_stream = new ByteInStream_URL(uri_original, 500) ) {
526 final boolean res = transfer_nio(data_stream, fname_payload_copy_lst.get(file_idx), 32768);
527 Assert.assertTrue( res );
528
529 Assert.assertTrue( file_exists( fname_payload_copy_lst.get(file_idx) ) );
530 final long copy_size = file_size(fname_payload_copy_lst.get(file_idx));
531 Assert.assertEquals( data_stream.content_size(), copy_size );
532 Assert.assertEquals( fname_payload_size_lst.get(file_idx).longValue(), copy_size );
533 Assert.assertTrue( FileUtil.compare(fname_payload_lst.get(file_idx), fname_payload_copy_lst.get(file_idx), true /* verbose */) );
534 }
535 }
536 {
537 final int file_idx = IDX_65MiB;
538
539 final String uri_original = url_input_root + fname_payload_lst.get(file_idx);
540
541 try( final ByteInStream_URL data_stream = new ByteInStream_URL(uri_original, 500) ) {
542 final boolean res = transfer_nio(data_stream, fname_payload_copy_lst.get(file_idx), 32768);
543 Assert.assertTrue( res );
544
545 Assert.assertTrue( file_exists( fname_payload_copy_lst.get(file_idx) ) );
546 final long copy_size = file_size(fname_payload_copy_lst.get(file_idx));
547 Assert.assertEquals( data_stream.content_size(), copy_size );
548 Assert.assertEquals( fname_payload_size_lst.get(file_idx).longValue(), copy_size );
549 Assert.assertTrue( FileUtil.compare(fname_payload_lst.get(file_idx), fname_payload_copy_lst.get(file_idx), true /* verbose */) );
550 }
551 }
552 }
553
554 @Test(timeout = 10000)
555 public void test12_copy_http_404() {
557 if( !UriTk.protocol_supported("http:") ) {
558 PrintUtil.fprintf_td(System.err, "http not supported, abort%n");
559 return;
560 }
561 httpd_start();
562 {
563 final int file_idx = IDX_11kiB;
564
565 final String uri_original = url_input_root + "doesnt_exists.txt";
566
567 try( final ByteInStream_URL data_stream = new ByteInStream_URL(uri_original, 500) ) {
568 final boolean res = transfer_nio(data_stream, fname_payload_copy_lst.get(file_idx), 4096);
569 Assert.assertFalse( res );
570
571 Assert.assertTrue( file_exists( fname_payload_copy_lst.get(file_idx) ) );
572 final long copy_size = file_size(fname_payload_copy_lst.get(file_idx));
573 Assert.assertTrue( data_stream.fail() );
574 Assert.assertFalse( data_stream.has_content_size() );
575 Assert.assertEquals( data_stream.content_size(), 0 );
576 Assert.assertEquals( 0, copy_size );
577 }
578 }
579 }
580
581 static Thread executeOffThread(final Runnable runobj, final String threadName, final boolean detach) {
582 final Thread t = new Thread( runobj, threadName );
583 t.setDaemon( detach );
584 t.start();
585 return t;
586 }
587
588 // throttled, no content size, interruptReader() via set_eof() will avoid timeout
589 static void feed_source_00(final ByteInStream_Feed data_feed, final int feed_size) {
590 // long xfer_total = 0;
591 try( final ByteInStream_File data_stream = new ByteInStream_File(data_feed.id() ) ) {
592 final byte buffer[] = new byte[feed_size];
593 while( data_stream.good() ) {
594 final int count = data_stream.read(buffer, 0, buffer.length);
595 if( 0 < count ) {
596 // xfer_total += count;
597 if( data_feed.write(buffer, 0, count) ) {
598 try { Thread.sleep(16); } catch (final Throwable t) {}
599 } else {
600 break;
601 }
602 }
603 }
604 // probably set after transfering due to above sleep, which also ends when total size has been reached.
605 data_feed.set_eof( data_feed.fail() ? -1 /* FAIL */ : 1 /* SUCCESS */ );
606 }
607 }
608
609 // throttled, with content size
610 static void feed_source_01(final ByteInStream_Feed data_feed, final int feed_size) {
611 long xfer_total = 0;
612 try( final ByteInStream_File data_stream = new ByteInStream_File(data_feed.id() ) ) {
613 final long file_size = data_stream.content_size();
614 data_feed.set_content_size( file_size );
615 final byte buffer[] = new byte[feed_size];
616 while( data_stream.good() && xfer_total < file_size ) {
617 final int count = data_stream.read(buffer, 0, buffer.length);
618 if( 0 < count ) {
619 xfer_total += count;
620 if( data_feed.write(buffer, 0, count) ) {
621 try { Thread.sleep(16); } catch (final Throwable t) {}
622 } else {
623 break;
624 }
625 }
626 }
627 // probably set after transfering due to above sleep, which also ends when total size has been reached.
628 data_feed.set_eof( !data_feed.fail() && xfer_total == file_size ? 1 /* SUCCESS */ : -1 /* FAILED */);
629 }
630 }
631
632 // full speed, with content size
633 static void feed_source_10_nio(final ByteInStream_Feed data_feed, final int feed_size) {
634 long xfer_total = 0;
635 try( final ByteInStream_File data_stream = new ByteInStream_File(data_feed.id() ) ) {
636 final long file_size = data_stream.content_size();
637 data_feed.set_content_size( file_size );
638 final ByteBuffer buffer = Buffers.newDirectByteBuffer(feed_size);
639 while( data_stream.good() && xfer_total < file_size ) {
640 final int count = data_stream.read(buffer);
641 if( 0 < count ) {
642 xfer_total += count;
643 if( !data_feed.write(buffer) ) {
644 break;
645 }
646 }
647 }
648 data_feed.set_eof( !data_feed.fail() && xfer_total == file_size ? 1 /* SUCCESS */ : -1 /* FAILED */);
649 }
650 }
651
652 // full speed, with content size
653 static void feed_source_10_std(final ByteInStream_Feed data_feed, final int feed_size) {
654 long xfer_total = 0;
655 try( final ByteInStream_File data_stream = new ByteInStream_File(data_feed.id() ) ) {
656 final long file_size = data_stream.content_size();
657 data_feed.set_content_size( file_size );
658 final byte buffer[] = new byte[feed_size];
659 while( data_stream.good() && xfer_total < file_size ) {
660 final int count = data_stream.read(buffer, 0, buffer.length);
661 if( 0 < count ) {
662 xfer_total += count;
663 if( !data_feed.write(buffer, 0, count) ) {
664 break;
665 }
666 }
667 }
668 data_feed.set_eof( !data_feed.fail() && xfer_total == file_size ? 1 /* SUCCESS */ : -1 /* FAILED */);
669 }
670 }
671
672 // full speed, no content size, interrupting @ 1024 bytes within our header
673 static void feed_source_20(final ByteInStream_Feed data_feed, final int feed_size) {
674 long xfer_total = 0;
675 try( final ByteInStream_File data_stream = new ByteInStream_File(data_feed.id() ) ) {
676 final byte buffer[] = new byte[feed_size];
677 while( data_stream.good() ) {
678 final int count = data_stream.read(buffer, 0, buffer.length);
679 if( 0 < count ) {
680 xfer_total += count;
681 if( data_feed.write(buffer, 0, count) ) {
682 if( xfer_total >= 1024 ) {
683 data_feed.set_eof( -1 /* FAILED */ ); // calls data_feed->interruptReader();
684 return;
685 }
686 } else {
687 break;
688 }
689 }
690 }
691 // probably set after transfering due to above sleep, which also ends when total size has been reached.
692 // data_feed.set_eof( 1 /* SUCCESS */ );
693 }
694 }
695
696 // full speed, with content size, interrupting 1/4 way
697 static void feed_source_21(final ByteInStream_Feed data_feed, final int feed_size) {
698 long xfer_total = 0;
699 try( final ByteInStream_File data_stream = new ByteInStream_File(data_feed.id() ) ) {
700 final long file_size = data_stream.content_size();
701 data_feed.set_content_size( file_size );
702 final byte buffer[] = new byte[feed_size];
703 while( data_stream.good() ) {
704 final int count = data_stream.read(buffer, 0, buffer.length);
705 if( 0 < count ) {
706 xfer_total += count;
707 if( data_feed.write(buffer, 0, count) ) {
708 if( xfer_total >= file_size/4 ) {
709 data_feed.set_eof( -1 /* FAILED */ ); // calls data_feed->interruptReader();
710 return;
711 }
712 } else {
713 break;
714 }
715 }
716 }
717 // probably set after transfering due to above sleep, which also ends when total size has been reached.
718 // data_feed.set_eof( 1 /* SUCCESS */ );
719 }
720 }
721
722 @Test(timeout = 10000)
723 public void test20_copy_fed_ok_buff4k_feed1k() {
725 final int buffer_size = 4096;
726 final int feed_size = 1024;
727 {
728 final int file_idx = IDX_11kiB;
729
730 // full speed, with content size
731 try( final ByteInStream_Feed data_feed = new ByteInStream_Feed(fname_payload_lst.get(file_idx), 500) ) {
732 final Thread feeder_thread = executeOffThread( () -> { feed_source_10_nio(data_feed, feed_size); }, "test21_copy_fed_ok::feed_source_10", false /* detach */);
733
734 final boolean res = transfer_nio(data_feed, fname_payload_copy_lst.get(file_idx), buffer_size);
735 try {
736 feeder_thread.join(1000);
737 } catch (final InterruptedException e) { }
738 Assert.assertTrue( res );
739
740 Assert.assertTrue( file_exists( fname_payload_copy_lst.get(file_idx) ) );
741 final long copy_size = file_size(fname_payload_copy_lst.get(file_idx));
742 Assert.assertEquals( data_feed.content_size(), copy_size );
743 Assert.assertEquals( fname_payload_size_lst.get(file_idx).longValue(), copy_size );
744 Assert.assertTrue( FileUtil.compare(fname_payload_lst.get(file_idx), fname_payload_copy_lst.get(file_idx), true /* verbose */) );
745 }
746
747 // throttled, with content size
748 try( final ByteInStream_Feed data_feed = new ByteInStream_Feed(fname_payload_lst.get(file_idx), 500) ) {
749 final Thread feeder_thread = executeOffThread( () -> { feed_source_01(data_feed, feed_size); }, "test21_copy_fed_ok::feed_source_01", false /* detach */);
750
751 final boolean res = transfer_nio(data_feed, fname_payload_copy_lst.get(file_idx), buffer_size);
752 try {
753 feeder_thread.join(1000);
754 } catch (final InterruptedException e) { }
755 Assert.assertTrue( res );
756
757 Assert.assertTrue( file_exists( fname_payload_copy_lst.get(file_idx) ) );
758 final long copy_size = file_size(fname_payload_copy_lst.get(file_idx));
759 Assert.assertEquals( data_feed.content_size(), copy_size );
760 Assert.assertEquals( fname_payload_size_lst.get(file_idx).longValue(), copy_size );
761 Assert.assertTrue( FileUtil.compare(fname_payload_lst.get(file_idx), fname_payload_copy_lst.get(file_idx), true /* verbose */) );
762 }
763
764 // throttled, no content size, interruptReader() via set_eof() will avoid timeout
765 try( final ByteInStream_Feed data_feed = new ByteInStream_Feed(fname_payload_lst.get(file_idx), 500) ) {
766 final Thread feeder_thread = executeOffThread( () -> { feed_source_00(data_feed, feed_size); }, "test21_copy_fed_ok::feed_source_00", false /* detach */);
767
768 final boolean res = transfer_nio(data_feed, fname_payload_copy_lst.get(file_idx), buffer_size);
769 try {
770 feeder_thread.join(1000);
771 } catch (final InterruptedException e) { }
772 Assert.assertTrue( res );
773
774 Assert.assertTrue( file_exists( fname_payload_copy_lst.get(file_idx) ) );
775 final long copy_size = file_size(fname_payload_copy_lst.get(file_idx));
776 Assert.assertEquals( data_feed.content_size(), 0 );
777 Assert.assertEquals( fname_payload_size_lst.get(file_idx).longValue(), copy_size );
778 Assert.assertTrue( FileUtil.compare(fname_payload_lst.get(file_idx), fname_payload_copy_lst.get(file_idx), true /* verbose */) );
779 }
780 }
781 {
782 final int file_idx = IDX_65MiB;
783
784 // full speed, with content size
785 try( final ByteInStream_Feed data_feed = new ByteInStream_Feed(fname_payload_lst.get(file_idx), 500) ) {
786 final Thread feeder_thread = executeOffThread( () -> { feed_source_10_nio(data_feed, feed_size); }, "test21_copy_fed_ok2::feed_source_10", false /* detach */);
787
788 final boolean res = transfer_nio(data_feed, fname_payload_copy_lst.get(file_idx), buffer_size);
789 try {
790 feeder_thread.join(1000);
791 } catch (final InterruptedException e) { }
792 Assert.assertTrue( res );
793
794 Assert.assertTrue( file_exists( fname_payload_copy_lst.get(file_idx) ) );
795 final long copy_size = file_size(fname_payload_copy_lst.get(file_idx));
796 Assert.assertEquals( data_feed.content_size(), copy_size );
797 Assert.assertEquals( fname_payload_size_lst.get(file_idx).longValue(), copy_size );
798 Assert.assertTrue( FileUtil.compare(fname_payload_lst.get(file_idx), fname_payload_copy_lst.get(file_idx), true /* verbose */) );
799 }
800 }
801 }
802
803 @Test(timeout = 10000)
804 public void test21_copy_fed_ok_buff32k() {
806 final int buffer_size = 32768;
807 final int feed_size = 32768;
808 {
809 final int file_idx = IDX_11kiB;
810
811 // full speed, with content size
812 try( final ByteInStream_Feed data_feed = new ByteInStream_Feed(fname_payload_lst.get(file_idx), 500) ) {
813 final Thread feeder_thread = executeOffThread( () -> { feed_source_10_nio(data_feed, feed_size); }, "test21_copy_fed_ok::feed_source_10", false /* detach */);
814
815 final boolean res = transfer_nio(data_feed, fname_payload_copy_lst.get(file_idx), buffer_size);
816 try {
817 feeder_thread.join(1000);
818 } catch (final InterruptedException e) { }
819 Assert.assertTrue( res );
820
821 Assert.assertTrue( file_exists( fname_payload_copy_lst.get(file_idx) ) );
822 final long copy_size = file_size(fname_payload_copy_lst.get(file_idx));
823 Assert.assertEquals( data_feed.content_size(), copy_size );
824 Assert.assertEquals( fname_payload_size_lst.get(file_idx).longValue(), copy_size );
825 Assert.assertTrue( FileUtil.compare(fname_payload_lst.get(file_idx), fname_payload_copy_lst.get(file_idx), true /* verbose */) );
826 }
827
828 // throttled, with content size
829 try( final ByteInStream_Feed data_feed = new ByteInStream_Feed(fname_payload_lst.get(file_idx), 500) ) {
830 final Thread feeder_thread = executeOffThread( () -> { feed_source_01(data_feed, feed_size); }, "test21_copy_fed_ok::feed_source_01", false /* detach */);
831
832 final boolean res = transfer_nio(data_feed, fname_payload_copy_lst.get(file_idx), buffer_size);
833 try {
834 feeder_thread.join(1000);
835 } catch (final InterruptedException e) { }
836 Assert.assertTrue( res );
837
838 Assert.assertTrue( file_exists( fname_payload_copy_lst.get(file_idx) ) );
839 final long copy_size = file_size(fname_payload_copy_lst.get(file_idx));
840 Assert.assertEquals( data_feed.content_size(), copy_size );
841 Assert.assertEquals( fname_payload_size_lst.get(file_idx).longValue(), copy_size );
842 Assert.assertTrue( FileUtil.compare(fname_payload_lst.get(file_idx), fname_payload_copy_lst.get(file_idx), true /* verbose */) );
843 }
844
845 // throttled, no content size, interruptReader() via set_eof() will avoid timeout
846 try( final ByteInStream_Feed data_feed = new ByteInStream_Feed(fname_payload_lst.get(file_idx), 500) ) {
847 final Thread feeder_thread = executeOffThread( () -> { feed_source_00(data_feed, feed_size); }, "test21_copy_fed_ok::feed_source_00", false /* detach */);
848
849 final boolean res = transfer_nio(data_feed, fname_payload_copy_lst.get(file_idx), buffer_size);
850 try {
851 feeder_thread.join(1000);
852 } catch (final InterruptedException e) { }
853 Assert.assertTrue( res );
854
855 Assert.assertTrue( file_exists( fname_payload_copy_lst.get(file_idx) ) );
856 final long copy_size = file_size(fname_payload_copy_lst.get(file_idx));
857 Assert.assertEquals( data_feed.content_size(), 0 );
858 Assert.assertEquals( fname_payload_size_lst.get(file_idx).longValue(), copy_size );
859 Assert.assertTrue( FileUtil.compare(fname_payload_lst.get(file_idx), fname_payload_copy_lst.get(file_idx), true /* verbose */) );
860 }
861 }
862 }
863
864 @Test(timeout = 10000)
865 public void test22_copy_fed_ok_buff32k_nio() {
867 final int buffer_size = 32768;
868 final int feed_size = 32768;
869 {
870 final int file_idx = IDX_65MiB;
871
872 // full speed, with content size
873 try( final ByteInStream_Feed data_feed = new ByteInStream_Feed(fname_payload_lst.get(file_idx), 500) ) {
874 final Thread feeder_thread = executeOffThread( () -> { feed_source_10_nio(data_feed, feed_size); }, "test21_copy_fed_ok2::feed_source_10", false /* detach */);
875
876 final boolean res = transfer_nio(data_feed, fname_payload_copy_lst.get(file_idx), buffer_size);
877 try {
878 feeder_thread.join(1000);
879 } catch (final InterruptedException e) { }
880 Assert.assertTrue( res );
881
882 Assert.assertTrue( file_exists( fname_payload_copy_lst.get(file_idx) ) );
883 final long copy_size = file_size(fname_payload_copy_lst.get(file_idx));
884 Assert.assertEquals( data_feed.content_size(), copy_size );
885 Assert.assertEquals( fname_payload_size_lst.get(file_idx).longValue(), copy_size );
886 Assert.assertTrue( FileUtil.compare(fname_payload_lst.get(file_idx), fname_payload_copy_lst.get(file_idx), true /* verbose */) );
887 }
888 }
889 }
890
891 @Test(timeout = 10000)
892 public void test23_copy_fed_ok_buff32k_std() {
894 final int buffer_size = 32768;
895 final int feed_size = 32768;
896 {
897 final int file_idx = IDX_65MiB;
898
899 // full speed, with content size
900 try( final ByteInStream_Feed data_feed = new ByteInStream_Feed(fname_payload_lst.get(file_idx), 500) ) {
901 final Thread feeder_thread = executeOffThread( () -> { feed_source_10_std(data_feed, feed_size); }, "test23_copy_fed_ok2::feed_source_10", false /* detach */);
902
903 final boolean res = transfer_std(data_feed, fname_payload_copy_lst.get(file_idx), buffer_size);
904 try {
905 feeder_thread.join(1000);
906 } catch (final InterruptedException e) { }
907 Assert.assertTrue( res );
908
909 Assert.assertTrue( file_exists( fname_payload_copy_lst.get(file_idx) ) );
910 final long copy_size = file_size(fname_payload_copy_lst.get(file_idx));
911 Assert.assertEquals( data_feed.content_size(), copy_size );
912 Assert.assertEquals( fname_payload_size_lst.get(file_idx).longValue(), copy_size );
913 Assert.assertTrue( FileUtil.compare(fname_payload_lst.get(file_idx), fname_payload_copy_lst.get(file_idx), true /* verbose */) );
914 }
915 }
916 }
917
918 @Test(timeout = 10000)
919 public void test24_copy_fed_irq() {
921 final int buffer_size = 4096;
922 final int feed_size = 1024;
923 {
924 final int file_idx = IDX_65MiB;
925
926 // full speed, no content size, interrupting @ 1024 bytes within our header
927 try( final ByteInStream_Feed data_feed = new ByteInStream_Feed(fname_payload_lst.get(file_idx), 500) ) {
928 final Thread feeder_thread = executeOffThread( () -> { feed_source_20(data_feed, feed_size); }, "test22_copy_fed_irq::feed_source_20", false /* detach */);
929
930 final boolean res = transfer_nio(data_feed, fname_payload_copy_lst.get(file_idx), buffer_size);
931 try {
932 feeder_thread.join(1000);
933 } catch (final InterruptedException e) { }
934 Assert.assertFalse( res );
935
936 Assert.assertTrue( file_exists( fname_payload_copy_lst.get(file_idx) ) );
937 final long copy_size = file_size(fname_payload_copy_lst.get(file_idx));
938 Assert.assertFalse( data_feed.has_content_size() );
939 Assert.assertEquals( data_feed.content_size(), 0 );
940 Assert.assertTrue( fname_payload_size_lst.get(file_idx).longValue() > copy_size );
941 Assert.assertFalse( FileUtil.compare(fname_payload_lst.get(file_idx), fname_payload_copy_lst.get(file_idx), true /* verbose */) );
942 }
943
944 // full speed, with content size, interrupting 1/4 way
945 try( final ByteInStream_Feed data_feed = new ByteInStream_Feed(fname_payload_lst.get(file_idx), 500) ) {
946 final Thread feeder_thread = executeOffThread( () -> { feed_source_21(data_feed, feed_size); }, "test22_copy_fed_irq::feed_source_21", false /* detach */);
947
948 final boolean res = transfer_nio(data_feed, fname_payload_copy_lst.get(file_idx), buffer_size);
949 try {
950 feeder_thread.join(1000);
951 } catch (final InterruptedException e) { }
952 Assert.assertFalse( res );
953
954 Assert.assertTrue( file_exists( fname_payload_copy_lst.get(file_idx) ) );
955 final long copy_size = file_size(fname_payload_copy_lst.get(file_idx));
956 Assert.assertTrue( data_feed.has_content_size() );
957 Assert.assertEquals( fname_payload_size_lst.get(file_idx).longValue(), data_feed.content_size() );
958 Assert.assertTrue( data_feed.content_size() > copy_size );
959 Assert.assertFalse( FileUtil.compare(fname_payload_lst.get(file_idx), fname_payload_copy_lst.get(file_idx), true /* verbose */) );
960 }
961 }
962 }
963
964 public static void main(final String args[]) {
965 org.junit.runner.JUnitCore.main(TestByteStream01.class.getName());
966 }
967};
968
Initialized by org.jau.sys.RuntimeProps.
static void main(final String args[])
Generic file type and POSIX protection mode bits as used in file_stats, touch(), mkdir() etc.
Definition: FMode.java:37
static final FMode def_file
Default file protection bit: Safe default: POSIX S_IRUSR | S_IWUSR | S_IRGRP or read_usr | write_usr ...
Definition: FMode.java:161
Platform agnostic representation of POSIX ::lstat() and ::stat() for a given pathname.
Definition: FileStats.java:41
boolean is_file()
Returns true if entity is a file, might be in combination with is_link().
Definition: FileStats.java:386
boolean exists()
Returns true if entity does not exist, exclusive bit.
Definition: FileStats.java:395
long size()
Returns the size in bytes of this element if is_file(), otherwise zero.
Definition: FileStats.java:341
Native file types and functionality.
Definition: FileUtil.java:33
static boolean remove(final String path, final TraverseOptions topts)
Remove the given path.
Definition: FileUtil.java:304
static native boolean compare(final String source1, final String source2, final boolean verbose)
Compare the bytes of both files, denoted by source1 and source2.
Filesystem traverse options used to visit() path elements.
static final TraverseOptions none
Ringbuffer-Based byte input stream with an externally provisioned data feed.
File based byte input stream, including named file descriptor.
Ringbuffer-Based byte input stream with a URL connection provisioned data feed.
File based byte output stream, including named file descriptor.
static void fprintf_td(final PrintStream out, final String format, final Object ... args)
Convenient PrintStream#printf(String, Object...) invocation, prepending the elapsedTimeMillis() times...
Definition: PrintUtil.java:37
static void println(final PrintStream out, final String msg)
Convenient PrintStream#println(String) invocation, prepending the elapsedTimeMillis() timestamp.
Definition: PrintUtil.java:46
Limited URI toolkit to query handled protocols by the IO implementation.
Definition: UriTk.java:34
static native boolean is_local_file_protocol(final String uri)
Returns true if the uri-scheme of given uri matches the local file protocol, i.e.
static native List< String > supported_protocols()
Returns a list of supported protocol supported by libcurl network protocols, queried at runtime.
static native boolean protocol_supported(final String uri)
Returns true if the uri-scheme of given uri matches a supported by libcurl network protocols otherwis...
This class represents an abstract byte input stream object.
static ByteInStream to_ByteInStream(final String path_or_uri, final long timeoutMS)
Parses the given path_or_uri, if it matches a supported protocol, see org.jau.io.UriTk#protocol_suppo...
Abstract byte input stream object.