1 ///////////////////////////////////////////////////////////////////////////////
2 //
3 /// \file coder.c
4 /// \brief Compresses or uncompresses a file
5 //
6 // Author: Lasse Collin
7 //
8 // This file has been put into the public domain.
9 // You can do whatever you want with this file.
10 //
11 ///////////////////////////////////////////////////////////////////////////////
12
13 #include "private.h"
14
15
16 /// Return value type for coder_init().
17 enum coder_init_ret {
18 CODER_INIT_NORMAL,
19 CODER_INIT_PASSTHRU,
20 CODER_INIT_ERROR,
21 };
22
23
24 enum operation_mode opt_mode = MODE_COMPRESS;
25 enum format_type opt_format = FORMAT_AUTO;
26 bool opt_auto_adjust = true;
27 bool opt_single_stream = false;
28 uint64_t opt_block_size = 0;
29 uint64_t *opt_block_list = NULL;
30
31
32 /// Stream used to communicate with liblzma
33 static lzma_stream strm = LZMA_STREAM_INIT;
34
35 /// Filters needed for all encoding all formats, and also decoding in raw data
36 static lzma_filter filters[LZMA_FILTERS_MAX + 1];
37
38 /// Input and output buffers
39 static io_buf in_buf;
40 static io_buf out_buf;
41
42 /// Number of filters. Zero indicates that we are using a preset.
43 static uint32_t filters_count = 0;
44
45 /// Number of the preset (0-9)
46 static uint32_t preset_number = LZMA_PRESET_DEFAULT;
47
48 /// Integrity check type
49 static lzma_check check;
50
51 /// This becomes false if the --check=CHECK option is used.
52 static bool check_default = true;
53
54 #if defined(HAVE_ENCODERS) && defined(MYTHREAD_ENABLED)
55 static lzma_mt mt_options = {
56 .flags = 0,
57 .timeout = 300,
58 .filters = filters,
59 };
60 #endif
61
62
63 extern void
coder_set_check(lzma_check new_check)64 coder_set_check(lzma_check new_check)
65 {
66 check = new_check;
67 check_default = false;
68 return;
69 }
70
71
72 static void
forget_filter_chain(void)73 forget_filter_chain(void)
74 {
75 // Setting a preset makes us forget a possibly defined custom
76 // filter chain.
77 while (filters_count > 0) {
78 --filters_count;
79 free(filters[filters_count].options);
80 filters[filters_count].options = NULL;
81 }
82
83 return;
84 }
85
86
87 extern void
coder_set_preset(uint32_t new_preset)88 coder_set_preset(uint32_t new_preset)
89 {
90 preset_number &= ~LZMA_PRESET_LEVEL_MASK;
91 preset_number |= new_preset;
92 forget_filter_chain();
93 return;
94 }
95
96
97 extern void
coder_set_extreme(void)98 coder_set_extreme(void)
99 {
100 preset_number |= LZMA_PRESET_EXTREME;
101 forget_filter_chain();
102 return;
103 }
104
105
106 extern void
coder_add_filter(lzma_vli id,void * options)107 coder_add_filter(lzma_vli id, void *options)
108 {
109 if (filters_count == LZMA_FILTERS_MAX)
110 message_fatal(_("Maximum number of filters is four"));
111
112 filters[filters_count].id = id;
113 filters[filters_count].options = options;
114 ++filters_count;
115
116 // Setting a custom filter chain makes us forget the preset options.
117 // This makes a difference if one specifies e.g. "xz -9 --lzma2 -e"
118 // where the custom filter chain resets the preset level back to
119 // the default 6, making the example equivalent to "xz -6e".
120 preset_number = LZMA_PRESET_DEFAULT;
121
122 return;
123 }
124
125
126 static void lzma_attribute((__noreturn__))
memlimit_too_small(uint64_t memory_usage)127 memlimit_too_small(uint64_t memory_usage)
128 {
129 message(V_ERROR, _("Memory usage limit is too low for the given "
130 "filter setup."));
131 message_mem_needed(V_ERROR, memory_usage);
132 tuklib_exit(E_ERROR, E_ERROR, false);
133 }
134
135
136 extern void
coder_set_compression_settings(void)137 coder_set_compression_settings(void)
138 {
139 // The default check type is CRC64, but fallback to CRC32
140 // if CRC64 isn't supported by the copy of liblzma we are
141 // using. CRC32 is always supported.
142 if (check_default) {
143 check = LZMA_CHECK_CRC64;
144 if (!lzma_check_is_supported(check))
145 check = LZMA_CHECK_CRC32;
146 }
147
148 // Options for LZMA1 or LZMA2 in case we are using a preset.
149 static lzma_options_lzma opt_lzma;
150
151 if (filters_count == 0) {
152 // We are using a preset. This is not a good idea in raw mode
153 // except when playing around with things. Different versions
154 // of this software may use different options in presets, and
155 // thus make uncompressing the raw data difficult.
156 if (opt_format == FORMAT_RAW) {
157 // The message is shown only if warnings are allowed
158 // but the exit status isn't changed.
159 message(V_WARNING, _("Using a preset in raw mode "
160 "is discouraged."));
161 message(V_WARNING, _("The exact options of the "
162 "presets may vary between software "
163 "versions."));
164 }
165
166 // Get the preset for LZMA1 or LZMA2.
167 if (lzma_lzma_preset(&opt_lzma, preset_number))
168 message_bug();
169
170 // Use LZMA2 except with --format=lzma we use LZMA1.
171 filters[0].id = opt_format == FORMAT_LZMA
172 ? LZMA_FILTER_LZMA1 : LZMA_FILTER_LZMA2;
173 filters[0].options = &opt_lzma;
174 filters_count = 1;
175 }
176
177 // Terminate the filter options array.
178 filters[filters_count].id = LZMA_VLI_UNKNOWN;
179
180 // If we are using the .lzma format, allow exactly one filter
181 // which has to be LZMA1.
182 if (opt_format == FORMAT_LZMA && (filters_count != 1
183 || filters[0].id != LZMA_FILTER_LZMA1))
184 message_fatal(_("The .lzma format supports only "
185 "the LZMA1 filter"));
186
187 // If we are using the .xz format, make sure that there is no LZMA1
188 // filter to prevent LZMA_PROG_ERROR.
189 if (opt_format == FORMAT_XZ)
190 for (size_t i = 0; i < filters_count; ++i)
191 if (filters[i].id == LZMA_FILTER_LZMA1)
192 message_fatal(_("LZMA1 cannot be used "
193 "with the .xz format"));
194
195 // Print the selected filter chain.
196 message_filters_show(V_DEBUG, filters);
197
198 // The --flush-timeout option requires LZMA_SYNC_FLUSH support
199 // from the filter chain. Currently threaded encoder doesn't support
200 // LZMA_SYNC_FLUSH so single-threaded mode must be used.
201 if (opt_mode == MODE_COMPRESS && opt_flush_timeout != 0) {
202 for (size_t i = 0; i < filters_count; ++i) {
203 switch (filters[i].id) {
204 case LZMA_FILTER_LZMA2:
205 case LZMA_FILTER_DELTA:
206 break;
207
208 default:
209 message_fatal(_("The filter chain is "
210 "incompatible with --flush-timeout"));
211 }
212 }
213
214 if (hardware_threads_get() > 1) {
215 message(V_WARNING, _("Switching to single-threaded "
216 "mode due to --flush-timeout"));
217 hardware_threads_set(1);
218 }
219 }
220
221 // Get the memory usage. Note that if --format=raw was used,
222 // we can be decompressing.
223 const uint64_t memory_limit = hardware_memlimit_get(opt_mode);
224 uint64_t memory_usage = UINT64_MAX;
225 if (opt_mode == MODE_COMPRESS) {
226 #ifdef HAVE_ENCODERS
227 # ifdef MYTHREAD_ENABLED
228 if (opt_format == FORMAT_XZ && hardware_threads_get() > 1) {
229 mt_options.threads = hardware_threads_get();
230 mt_options.block_size = opt_block_size;
231 mt_options.check = check;
232 memory_usage = lzma_stream_encoder_mt_memusage(
233 &mt_options);
234 if (memory_usage != UINT64_MAX)
235 message(V_DEBUG, _("Using up to %" PRIu32
236 " threads."),
237 mt_options.threads);
238 } else
239 # endif
240 {
241 memory_usage = lzma_raw_encoder_memusage(filters);
242 }
243 #endif
244 } else {
245 #ifdef HAVE_DECODERS
246 memory_usage = lzma_raw_decoder_memusage(filters);
247 #endif
248 }
249
250 if (memory_usage == UINT64_MAX)
251 message_fatal(_("Unsupported filter chain or filter options"));
252
253 // Print memory usage info before possible dictionary
254 // size auto-adjusting.
255 //
256 // NOTE: If only encoder support was built, we cannot show the
257 // what the decoder memory usage will be.
258 message_mem_needed(V_DEBUG, memory_usage);
259 #ifdef HAVE_DECODERS
260 if (opt_mode == MODE_COMPRESS) {
261 const uint64_t decmem = lzma_raw_decoder_memusage(filters);
262 if (decmem != UINT64_MAX)
263 message(V_DEBUG, _("Decompression will need "
264 "%s MiB of memory."), uint64_to_str(
265 round_up_to_mib(decmem), 0));
266 }
267 #endif
268
269 if (memory_usage <= memory_limit)
270 return;
271
272 // If --no-adjust was used or we didn't find LZMA1 or
273 // LZMA2 as the last filter, give an error immediately.
274 // --format=raw implies --no-adjust.
275 if (!opt_auto_adjust || opt_format == FORMAT_RAW)
276 memlimit_too_small(memory_usage);
277
278 assert(opt_mode == MODE_COMPRESS);
279
280 #ifdef HAVE_ENCODERS
281 # ifdef MYTHREAD_ENABLED
282 if (opt_format == FORMAT_XZ && mt_options.threads > 1) {
283 // Try to reduce the number of threads before
284 // adjusting the compression settings down.
285 do {
286 // FIXME? The real single-threaded mode has
287 // lower memory usage, but it's not comparable
288 // because it doesn't write the size info
289 // into Block Headers.
290 if (--mt_options.threads == 0)
291 memlimit_too_small(memory_usage);
292
293 memory_usage = lzma_stream_encoder_mt_memusage(
294 &mt_options);
295 if (memory_usage == UINT64_MAX)
296 message_bug();
297
298 } while (memory_usage > memory_limit);
299
300 message(V_WARNING, _("Adjusted the number of threads "
301 "from %s to %s to not exceed "
302 "the memory usage limit of %s MiB"),
303 uint64_to_str(hardware_threads_get(), 0),
304 uint64_to_str(mt_options.threads, 1),
305 uint64_to_str(round_up_to_mib(
306 memory_limit), 2));
307 }
308 # endif
309
310 if (memory_usage <= memory_limit)
311 return;
312
313 // Look for the last filter if it is LZMA2 or LZMA1, so we can make
314 // it use less RAM. With other filters we don't know what to do.
315 size_t i = 0;
316 while (filters[i].id != LZMA_FILTER_LZMA2
317 && filters[i].id != LZMA_FILTER_LZMA1) {
318 if (filters[i].id == LZMA_VLI_UNKNOWN)
319 memlimit_too_small(memory_usage);
320
321 ++i;
322 }
323
324 // Decrease the dictionary size until we meet the memory
325 // usage limit. First round down to full mebibytes.
326 lzma_options_lzma *opt = filters[i].options;
327 const uint32_t orig_dict_size = opt->dict_size;
328 opt->dict_size &= ~((UINT32_C(1) << 20) - 1);
329 while (true) {
330 // If it is below 1 MiB, auto-adjusting failed. We could be
331 // more sophisticated and scale it down even more, but let's
332 // see if many complain about this version.
333 //
334 // FIXME: Displays the scaled memory usage instead
335 // of the original.
336 if (opt->dict_size < (UINT32_C(1) << 20))
337 memlimit_too_small(memory_usage);
338
339 memory_usage = lzma_raw_encoder_memusage(filters);
340 if (memory_usage == UINT64_MAX)
341 message_bug();
342
343 // Accept it if it is low enough.
344 if (memory_usage <= memory_limit)
345 break;
346
347 // Otherwise 1 MiB down and try again. I hope this
348 // isn't too slow method for cases where the original
349 // dict_size is very big.
350 opt->dict_size -= UINT32_C(1) << 20;
351 }
352
353 // Tell the user that we decreased the dictionary size.
354 message(V_WARNING, _("Adjusted LZMA%c dictionary size "
355 "from %s MiB to %s MiB to not exceed "
356 "the memory usage limit of %s MiB"),
357 filters[i].id == LZMA_FILTER_LZMA2
358 ? '2' : '1',
359 uint64_to_str(orig_dict_size >> 20, 0),
360 uint64_to_str(opt->dict_size >> 20, 1),
361 uint64_to_str(round_up_to_mib(memory_limit), 2));
362 #endif
363
364 return;
365 }
366
367
368 #ifdef HAVE_DECODERS
369 /// Return true if the data in in_buf seems to be in the .xz format.
370 static bool
is_format_xz(void)371 is_format_xz(void)
372 {
373 // Specify the magic as hex to be compatible with EBCDIC systems.
374 static const uint8_t magic[6] = { 0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00 };
375 return strm.avail_in >= sizeof(magic)
376 && memcmp(in_buf.u8, magic, sizeof(magic)) == 0;
377 }
378
379
380 /// Return true if the data in in_buf seems to be in the .lzma format.
381 static bool
is_format_lzma(void)382 is_format_lzma(void)
383 {
384 // The .lzma header is 13 bytes.
385 if (strm.avail_in < 13)
386 return false;
387
388 // Decode the LZMA1 properties.
389 lzma_filter filter = { .id = LZMA_FILTER_LZMA1 };
390 if (lzma_properties_decode(&filter, NULL, in_buf.u8, 5) != LZMA_OK)
391 return false;
392
393 // A hack to ditch tons of false positives: We allow only dictionary
394 // sizes that are 2^n or 2^n + 2^(n-1) or UINT32_MAX. LZMA_Alone
395 // created only files with 2^n, but accepts any dictionary size.
396 // If someone complains, this will be reconsidered.
397 lzma_options_lzma *opt = filter.options;
398 const uint32_t dict_size = opt->dict_size;
399 free(opt);
400
401 if (dict_size != UINT32_MAX) {
402 uint32_t d = dict_size - 1;
403 d |= d >> 2;
404 d |= d >> 3;
405 d |= d >> 4;
406 d |= d >> 8;
407 d |= d >> 16;
408 ++d;
409 if (d != dict_size || dict_size == 0)
410 return false;
411 }
412
413 // Another hack to ditch false positives: Assume that if the
414 // uncompressed size is known, it must be less than 256 GiB.
415 // Again, if someone complains, this will be reconsidered.
416 uint64_t uncompressed_size = 0;
417 for (size_t i = 0; i < 8; ++i)
418 uncompressed_size |= (uint64_t)(in_buf.u8[5 + i]) << (i * 8);
419
420 if (uncompressed_size != UINT64_MAX
421 && uncompressed_size > (UINT64_C(1) << 38))
422 return false;
423
424 return true;
425 }
426 #endif
427
428
429 /// Detect the input file type (for now, this done only when decompressing),
430 /// and initialize an appropriate coder. Return value indicates if a normal
431 /// liblzma-based coder was initialized (CODER_INIT_NORMAL), if passthru
432 /// mode should be used (CODER_INIT_PASSTHRU), or if an error occurred
433 /// (CODER_INIT_ERROR).
434 static enum coder_init_ret
coder_init(file_pair * pair)435 coder_init(file_pair *pair)
436 {
437 lzma_ret ret = LZMA_PROG_ERROR;
438
439 if (opt_mode == MODE_COMPRESS) {
440 #ifdef HAVE_ENCODERS
441 switch (opt_format) {
442 case FORMAT_AUTO:
443 // args.c ensures this.
444 assert(0);
445 break;
446
447 case FORMAT_XZ:
448 # ifdef MYTHREAD_ENABLED
449 if (hardware_threads_get() > 1)
450 ret = lzma_stream_encoder_mt(
451 &strm, &mt_options);
452 else
453 # endif
454 ret = lzma_stream_encoder(
455 &strm, filters, check);
456 break;
457
458 case FORMAT_LZMA:
459 ret = lzma_alone_encoder(&strm, filters[0].options);
460 break;
461
462 case FORMAT_RAW:
463 ret = lzma_raw_encoder(&strm, filters);
464 break;
465 }
466 #endif
467 } else {
468 #ifdef HAVE_DECODERS
469 uint32_t flags = 0;
470
471 // It seems silly to warn about unsupported check if the
472 // check won't be verified anyway due to --ignore-check.
473 if (opt_ignore_check)
474 flags |= LZMA_IGNORE_CHECK;
475 else
476 flags |= LZMA_TELL_UNSUPPORTED_CHECK;
477
478 if (!opt_single_stream)
479 flags |= LZMA_CONCATENATED;
480
481 // We abuse FORMAT_AUTO to indicate unknown file format,
482 // for which we may consider passthru mode.
483 enum format_type init_format = FORMAT_AUTO;
484
485 switch (opt_format) {
486 case FORMAT_AUTO:
487 if (is_format_xz())
488 init_format = FORMAT_XZ;
489 else if (is_format_lzma())
490 init_format = FORMAT_LZMA;
491 break;
492
493 case FORMAT_XZ:
494 if (is_format_xz())
495 init_format = FORMAT_XZ;
496 break;
497
498 case FORMAT_LZMA:
499 if (is_format_lzma())
500 init_format = FORMAT_LZMA;
501 break;
502
503 case FORMAT_RAW:
504 init_format = FORMAT_RAW;
505 break;
506 }
507
508 switch (init_format) {
509 case FORMAT_AUTO:
510 // Unknown file format. If --decompress --stdout
511 // --force have been given, then we copy the input
512 // as is to stdout. Checking for MODE_DECOMPRESS
513 // is needed, because we don't want to do use
514 // passthru mode with --test.
515 if (opt_mode == MODE_DECOMPRESS
516 && opt_stdout && opt_force)
517 return CODER_INIT_PASSTHRU;
518
519 ret = LZMA_FORMAT_ERROR;
520 break;
521
522 case FORMAT_XZ:
523 ret = lzma_stream_decoder(&strm,
524 hardware_memlimit_get(
525 MODE_DECOMPRESS), flags);
526 break;
527
528 case FORMAT_LZMA:
529 ret = lzma_alone_decoder(&strm,
530 hardware_memlimit_get(
531 MODE_DECOMPRESS));
532 break;
533
534 case FORMAT_RAW:
535 // Memory usage has already been checked in
536 // coder_set_compression_settings().
537 ret = lzma_raw_decoder(&strm, filters);
538 break;
539 }
540
541 // Try to decode the headers. This will catch too low
542 // memory usage limit in case it happens in the first
543 // Block of the first Stream, which is where it very
544 // probably will happen if it is going to happen.
545 if (ret == LZMA_OK && init_format != FORMAT_RAW) {
546 strm.next_out = NULL;
547 strm.avail_out = 0;
548 ret = lzma_code(&strm, LZMA_RUN);
549 }
550 #endif
551 }
552
553 if (ret != LZMA_OK) {
554 message_error("%s: %s", pair->src_name, message_strm(ret));
555 if (ret == LZMA_MEMLIMIT_ERROR)
556 message_mem_needed(V_ERROR, lzma_memusage(&strm));
557
558 return CODER_INIT_ERROR;
559 }
560
561 return CODER_INIT_NORMAL;
562 }
563
564
565 /// Resolve conflicts between opt_block_size and opt_block_list in single
566 /// threaded mode. We want to default to opt_block_list, except when it is
567 /// larger than opt_block_size. If this is the case for the current Block
568 /// at *list_pos, then we break into smaller Blocks. Otherwise advance
569 /// to the next Block in opt_block_list, and break apart if needed.
570 static void
split_block(uint64_t * block_remaining,uint64_t * next_block_remaining,size_t * list_pos)571 split_block(uint64_t *block_remaining,
572 uint64_t *next_block_remaining,
573 size_t *list_pos)
574 {
575 if (*next_block_remaining > 0) {
576 // The Block at *list_pos has previously been split up.
577 assert(hardware_threads_get() == 1);
578 assert(opt_block_size > 0);
579 assert(opt_block_list != NULL);
580
581 if (*next_block_remaining > opt_block_size) {
582 // We have to split the current Block at *list_pos
583 // into another opt_block_size length Block.
584 *block_remaining = opt_block_size;
585 } else {
586 // This is the last remaining split Block for the
587 // Block at *list_pos.
588 *block_remaining = *next_block_remaining;
589 }
590
591 *next_block_remaining -= *block_remaining;
592
593 } else {
594 // The Block at *list_pos has been finished. Go to the next
595 // entry in the list. If the end of the list has been reached,
596 // reuse the size of the last Block.
597 if (opt_block_list[*list_pos + 1] != 0)
598 ++*list_pos;
599
600 *block_remaining = opt_block_list[*list_pos];
601
602 // If in single-threaded mode, split up the Block if needed.
603 // This is not needed in multi-threaded mode because liblzma
604 // will do this due to how threaded encoding works.
605 if (hardware_threads_get() == 1 && opt_block_size > 0
606 && *block_remaining > opt_block_size) {
607 *next_block_remaining
608 = *block_remaining - opt_block_size;
609 *block_remaining = opt_block_size;
610 }
611 }
612 }
613
614
615 /// Compress or decompress using liblzma.
616 static bool
coder_normal(file_pair * pair)617 coder_normal(file_pair *pair)
618 {
619 // Encoder needs to know when we have given all the input to it.
620 // The decoders need to know it too when we are using
621 // LZMA_CONCATENATED. We need to check for src_eof here, because
622 // the first input chunk has been already read if decompressing,
623 // and that may have been the only chunk we will read.
624 lzma_action action = pair->src_eof ? LZMA_FINISH : LZMA_RUN;
625
626 lzma_ret ret;
627
628 // Assume that something goes wrong.
629 bool success = false;
630
631 // block_remaining indicates how many input bytes to encode before
632 // finishing the current .xz Block. The Block size is set with
633 // --block-size=SIZE and --block-list. They have an effect only when
634 // compressing to the .xz format. If block_remaining == UINT64_MAX,
635 // only a single block is created.
636 uint64_t block_remaining = UINT64_MAX;
637
638 // next_block_remining for when we are in single-threaded mode and
639 // the Block in --block-list is larger than the --block-size=SIZE.
640 uint64_t next_block_remaining = 0;
641
642 // Position in opt_block_list. Unused if --block-list wasn't used.
643 size_t list_pos = 0;
644
645 // Handle --block-size for single-threaded mode and the first step
646 // of --block-list.
647 if (opt_mode == MODE_COMPRESS && opt_format == FORMAT_XZ) {
648 // --block-size doesn't do anything here in threaded mode,
649 // because the threaded encoder will take care of splitting
650 // to fixed-sized Blocks.
651 if (hardware_threads_get() == 1 && opt_block_size > 0)
652 block_remaining = opt_block_size;
653
654 // If --block-list was used, start with the first size.
655 //
656 // For threaded case, --block-size specifies how big Blocks
657 // the encoder needs to be prepared to create at maximum
658 // and --block-list will simultaneously cause new Blocks
659 // to be started at specified intervals. To keep things
660 // logical, the same is done in single-threaded mode. The
661 // output is still not identical because in single-threaded
662 // mode the size info isn't written into Block Headers.
663 if (opt_block_list != NULL) {
664 if (block_remaining < opt_block_list[list_pos]) {
665 assert(hardware_threads_get() == 1);
666 next_block_remaining = opt_block_list[list_pos]
667 - block_remaining;
668 } else {
669 block_remaining = opt_block_list[list_pos];
670 }
671 }
672 }
673
674 strm.next_out = out_buf.u8;
675 strm.avail_out = IO_BUFFER_SIZE;
676
677 while (!user_abort) {
678 // Fill the input buffer if it is empty and we aren't
679 // flushing or finishing.
680 if (strm.avail_in == 0 && action == LZMA_RUN) {
681 strm.next_in = in_buf.u8;
682 strm.avail_in = io_read(pair, &in_buf,
683 my_min(block_remaining,
684 IO_BUFFER_SIZE));
685
686 if (strm.avail_in == SIZE_MAX)
687 break;
688
689 if (pair->src_eof) {
690 action = LZMA_FINISH;
691
692 } else if (block_remaining != UINT64_MAX) {
693 // Start a new Block after every
694 // opt_block_size bytes of input.
695 block_remaining -= strm.avail_in;
696 if (block_remaining == 0)
697 action = LZMA_FULL_BARRIER;
698 }
699
700 if (action == LZMA_RUN && flush_needed)
701 action = LZMA_SYNC_FLUSH;
702 }
703
704 // Let liblzma do the actual work.
705 ret = lzma_code(&strm, action);
706
707 // Write out if the output buffer became full.
708 if (strm.avail_out == 0) {
709 if (opt_mode != MODE_TEST && io_write(pair, &out_buf,
710 IO_BUFFER_SIZE - strm.avail_out))
711 break;
712
713 strm.next_out = out_buf.u8;
714 strm.avail_out = IO_BUFFER_SIZE;
715 }
716
717 if (ret == LZMA_STREAM_END && (action == LZMA_SYNC_FLUSH
718 || action == LZMA_FULL_BARRIER)) {
719 if (action == LZMA_SYNC_FLUSH) {
720 // Flushing completed. Write the pending data
721 // out immediatelly so that the reading side
722 // can decompress everything compressed so far.
723 if (io_write(pair, &out_buf, IO_BUFFER_SIZE
724 - strm.avail_out))
725 break;
726
727 strm.next_out = out_buf.u8;
728 strm.avail_out = IO_BUFFER_SIZE;
729
730 // Set the time of the most recent flushing.
731 mytime_set_flush_time();
732 } else {
733 // Start a new Block after LZMA_FULL_BARRIER.
734 if (opt_block_list == NULL) {
735 assert(hardware_threads_get() == 1);
736 assert(opt_block_size > 0);
737 block_remaining = opt_block_size;
738 } else {
739 split_block(&block_remaining,
740 &next_block_remaining,
741 &list_pos);
742 }
743 }
744
745 // Start a new Block after LZMA_FULL_FLUSH or continue
746 // the same block after LZMA_SYNC_FLUSH.
747 action = LZMA_RUN;
748
749 } else if (ret != LZMA_OK) {
750 // Determine if the return value indicates that we
751 // won't continue coding.
752 const bool stop = ret != LZMA_NO_CHECK
753 && ret != LZMA_UNSUPPORTED_CHECK;
754
755 if (stop) {
756 // Write the remaining bytes even if something
757 // went wrong, because that way the user gets
758 // as much data as possible, which can be good
759 // when trying to get at least some useful
760 // data out of damaged files.
761 if (opt_mode != MODE_TEST && io_write(pair,
762 &out_buf, IO_BUFFER_SIZE
763 - strm.avail_out))
764 break;
765 }
766
767 if (ret == LZMA_STREAM_END) {
768 if (opt_single_stream) {
769 io_fix_src_pos(pair, strm.avail_in);
770 success = true;
771 break;
772 }
773
774 // Check that there is no trailing garbage.
775 // This is needed for LZMA_Alone and raw
776 // streams.
777 if (strm.avail_in == 0 && !pair->src_eof) {
778 // Try reading one more byte.
779 // Hopefully we don't get any more
780 // input, and thus pair->src_eof
781 // becomes true.
782 strm.avail_in = io_read(
783 pair, &in_buf, 1);
784 if (strm.avail_in == SIZE_MAX)
785 break;
786
787 assert(strm.avail_in == 0
788 || strm.avail_in == 1);
789 }
790
791 if (strm.avail_in == 0) {
792 assert(pair->src_eof);
793 success = true;
794 break;
795 }
796
797 // We hadn't reached the end of the file.
798 ret = LZMA_DATA_ERROR;
799 assert(stop);
800 }
801
802 // If we get here and stop is true, something went
803 // wrong and we print an error. Otherwise it's just
804 // a warning and coding can continue.
805 if (stop) {
806 message_error("%s: %s", pair->src_name,
807 message_strm(ret));
808 } else {
809 message_warning("%s: %s", pair->src_name,
810 message_strm(ret));
811
812 // When compressing, all possible errors set
813 // stop to true.
814 assert(opt_mode != MODE_COMPRESS);
815 }
816
817 if (ret == LZMA_MEMLIMIT_ERROR) {
818 // Display how much memory it would have
819 // actually needed.
820 message_mem_needed(V_ERROR,
821 lzma_memusage(&strm));
822 }
823
824 if (stop)
825 break;
826 }
827
828 // Show progress information under certain conditions.
829 message_progress_update();
830 }
831
832 return success;
833 }
834
835
836 /// Copy from input file to output file without processing the data in any
837 /// way. This is used only when trying to decompress unrecognized files
838 /// with --decompress --stdout --force, so the output is always stdout.
839 static bool
coder_passthru(file_pair * pair)840 coder_passthru(file_pair *pair)
841 {
842 while (strm.avail_in != 0) {
843 if (user_abort)
844 return false;
845
846 if (io_write(pair, &in_buf, strm.avail_in))
847 return false;
848
849 strm.total_in += strm.avail_in;
850 strm.total_out = strm.total_in;
851 message_progress_update();
852
853 strm.avail_in = io_read(pair, &in_buf, IO_BUFFER_SIZE);
854 if (strm.avail_in == SIZE_MAX)
855 return false;
856 }
857
858 return true;
859 }
860
861
862 extern void
coder_run(const char * filename)863 coder_run(const char *filename)
864 {
865 // Set and possibly print the filename for the progress message.
866 message_filename(filename);
867
868 // Try to open the input file.
869 file_pair *pair = io_open_src(filename);
870 if (pair == NULL)
871 return;
872
873 // Assume that something goes wrong.
874 bool success = false;
875
876 if (opt_mode == MODE_COMPRESS) {
877 strm.next_in = NULL;
878 strm.avail_in = 0;
879 } else {
880 // Read the first chunk of input data. This is needed
881 // to detect the input file type.
882 strm.next_in = in_buf.u8;
883 strm.avail_in = io_read(pair, &in_buf, IO_BUFFER_SIZE);
884 }
885
886 if (strm.avail_in != SIZE_MAX) {
887 // Initialize the coder. This will detect the file format
888 // and, in decompression or testing mode, check the memory
889 // usage of the first Block too. This way we don't try to
890 // open the destination file if we see that coding wouldn't
891 // work at all anyway. This also avoids deleting the old
892 // "target" file if --force was used.
893 const enum coder_init_ret init_ret = coder_init(pair);
894
895 if (init_ret != CODER_INIT_ERROR && !user_abort) {
896 // Don't open the destination file when --test
897 // is used.
898 if (opt_mode == MODE_TEST || !io_open_dest(pair)) {
899 // Remember the current time. It is needed
900 // for progress indicator and for timed
901 // flushing.
902 mytime_set_start_time();
903
904 // Initialize the progress indicator.
905 const uint64_t in_size
906 = pair->src_st.st_size <= 0
907 ? 0 : pair->src_st.st_size;
908 message_progress_start(&strm, in_size);
909
910 // Do the actual coding or passthru.
911 if (init_ret == CODER_INIT_NORMAL)
912 success = coder_normal(pair);
913 else
914 success = coder_passthru(pair);
915
916 message_progress_end(success);
917 }
918 }
919 }
920
921 // Close the file pair. It needs to know if coding was successful to
922 // know if the source or target file should be unlinked.
923 io_close(pair, success);
924
925 return;
926 }
927
928
929 #ifndef NDEBUG
930 extern void
coder_free(void)931 coder_free(void)
932 {
933 lzma_end(&strm);
934 return;
935 }
936 #endif
937