ip7z/7zip
1
0
Fork 0
mirror of https://github.com/ip7z/7zip.git synced 2026-04-21 06:03:40 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge branch 'ip7z:main' into main

Browse source
This commit is contained in:
unxed 2026-04-17 13:43:00 +02:00 • committed by GitHub
commit 92bf145ec3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
168 changed files with 7753 additions and 3463 deletions
Download patch file Download diff file Expand all files Collapse all files

860
Asm/x86/Sort.asm Normal file
View file

@ -0,0 +1,860 @@
; SortTest.asm -- ASM version of HeapSort() function
; Igor Pavlov : Public domain
include ../../../../Asm/x86/7zAsm.asm
MY_ASM_START
ifndef Z7_SORT_ASM_USE_SEGMENT
if (IS_LINUX gt 0)
; Z7_SORT_ASM_USE_SEGMENT equ 1
else
; Z7_SORT_ASM_USE_SEGMENT equ 1
endif
endif
ifdef Z7_SORT_ASM_USE_SEGMENT
_TEXT$Z7_SORT SEGMENT ALIGN(64) 'CODE'
MY_ALIGN macro num:req
align num
endm
else
MY_ALIGN macro num:req
; We expect that ".text" is aligned for 16-bytes.
; So we don't need large alignment inside our function.
align 16
endm
endif
MY_ALIGN_16 macro
MY_ALIGN 16
endm
MY_ALIGN_32 macro
MY_ALIGN 32
endm
MY_ALIGN_64 macro
MY_ALIGN 64
endm
ifdef x64
NUM_PREFETCH_LEVELS equ 3 ; to prefetch 1x 64-bytes line (is good for most cases)
; NUM_PREFETCH_LEVELS equ 4 ; to prefetch 2x 64-bytes lines (better for big arrays)
acc equ x0
k equ r0
k_x equ x0
p equ r1
s equ r2
s_x equ x2
a0 equ x3
t0 equ a0
a3 equ x5
qq equ a3
a1 equ x6
t1 equ a1
t1_r equ r6
a2 equ x7
t2 equ a2
i equ r8
e0 equ x8
e1 equ x9
num_last equ r10
num_last_x equ x10
next4_lim equ r11
pref_lim equ r12
SORT_2_WITH_TEMP_REG macro b0, b1, temp_reg
mov temp_reg, b0
cmp b0, b1
cmovae b0, b1 ; min
cmovae b1, temp_reg ; max
endm
SORT macro b0, b1
SORT_2_WITH_TEMP_REG b0, b1, acc
endm
LOAD macro dest:req, index:req
mov dest, [p + 4 * index]
endm
STORE macro reg:req, index:req
mov [p + 4 * index], reg
endm
if (NUM_PREFETCH_LEVELS gt 3)
num_prefetches equ (1 SHL (NUM_PREFETCH_LEVELS - 3))
else
num_prefetches equ 1
endif
PREFETCH_OP macro offs
cur_offset = 7 * 4 ; it's average offset in 64-bytes cache line.
; cur_offset = 0 ; we can use zero offset, if we are sure that array is aligned for 64-bytes.
rept num_prefetches
if 1
prefetcht0 byte ptr [p + offs + cur_offset]
else
mov pref_x, dword ptr [p + offs + cur_offset]
endif
cur_offset = cur_offset + 64
endm
endm
PREFETCH_MY macro
if 1
if 1
shl k, NUM_PREFETCH_LEVELS + 3
else
; we delay prefetch instruction to improve main loads
shl k, NUM_PREFETCH_LEVELS
shl k, 3
; shl k, 0
endif
PREFETCH_OP k
elseif 1
shl k, 3
PREFETCH_OP k * (1 SHL NUM_PREFETCH_LEVELS) ; change it
endif
endm
STEP_1 macro exit_label, prefetch_macro
use_cmov_1 equ 1 ; set 1 for cmov, but it's slower in some cases
; set 0 for LOAD after adc s, 0
cmp t0, t1
if use_cmov_1
cmovb t0, t1
; STORE t0, k
endif
adc s, 0
if use_cmov_1 eq 0
LOAD t0, s
endif
cmp qq, t0
jae exit_label
if 1 ; use_cmov_1 eq 0
STORE t0, k
endif
prefetch_macro
mov t0, [p + s * 8]
mov t1, [p + s * 8 + 4]
mov k, s
add s, s ; slower for some cpus
; lea s, dword ptr [s + s] ; slower for some cpus
; shl s, 1 ; faster for some cpus
; lea s, dword ptr [s * 2] ; faster for some cpus
rept 0 ; 1000 for debug : 0 for normal
; number of calls in generate_stage : ~0.6 of number of items
shl k, 0
endm
endm
STEP_2 macro exit_label, prefetch_macro
use_cmov_2 equ 0 ; set 1 for cmov, but it's slower in some cases
; set 0 for LOAD after adc s, 0
cmp t0, t1
if use_cmov_2
mov t2, t0
cmovb t2, t1
; STORE t2, k
endif
mov t0, [p + s * 8]
mov t1, [p + s * 8 + 4]
cmovb t0, [p + s * 8 + 8]
cmovb t1, [p + s * 8 + 12]
adc s, 0
if use_cmov_2 eq 0
LOAD t2, s
endif
cmp qq, t2
jae exit_label
if 1 ; use_cmov_2 eq 0
STORE t2, k
endif
prefetch_macro
mov k, s
; add s, s
; lea s, [s + s]
shl s, 1
; lea s, [s * 2]
endm
MOVE_SMALLEST_UP macro STEP, use_prefetch, num_unrolls
LOCAL exit_1, exit_2, leaves, opt_loop, last_nodes
; s == k * 2
; t0 == (p)[s]
; t1 == (p)[s + 1]
cmp k, next4_lim
jae leaves
rept num_unrolls
STEP exit_2
cmp k, next4_lim
jae leaves
endm
if use_prefetch
prefetch_macro equ PREFETCH_MY
pref_lim_2 equ pref_lim
; lea pref_lim, dword ptr [num_last + 1]
; shr pref_lim, NUM_PREFETCH_LEVELS + 1
cmp k, pref_lim_2
jae last_nodes
else
prefetch_macro equ
pref_lim_2 equ next4_lim
endif
MY_ALIGN_16
opt_loop:
STEP exit_2, prefetch_macro
cmp k, pref_lim_2
jb opt_loop
last_nodes:
; k >= pref_lim_2
; 2 cases are possible:
; case-1: num_after_prefetch_levels == 0 && next4_lim = pref_lim_2
; case-2: num_after_prefetch_levels == NUM_PREFETCH_LEVELS - 1 &&
; next4_lim = pref_lim_2 / (NUM_PREFETCH_LEVELS - 1)
if use_prefetch
yyy = NUM_PREFETCH_LEVELS - 1
while yyy
yyy = yyy - 1
STEP exit_2
if yyy
cmp k, next4_lim
jae leaves
endif
endm
endif
leaves:
; k >= next4_lim == (num_last + 1) / 4 must be provided by previous code.
; we have 2 nodes in (s) level : always
; we can have some nodes in (s * 2) level : low probability case
; we have no nodes in (s * 4) level
; s == k * 2
; t0 == (p)[s]
; t1 == (p)[s + 1]
cmp t0, t1
cmovb t0, t1
adc s, 0
STORE t0, k
; t0 == (p)[s]
; s / 2 == k : (s) is index of max item from (p)[k * 2], (p)[k * 2 + 1]
; we have 3 possible cases here:
; s * 2 > num_last : (s) node has no childs
; s * 2 == num_last : (s) node has 1 leaf child that is last item of array
; s * 2 < num_last : (s) node has 2 leaf childs. We provide (s * 4 > num_last)
; we check for (s * 2 > num_last) before "cmp qq, t0" check, because
; we will replace conditional jump with cmov instruction later.
lea t1_r, dword ptr [s + s]
cmp t1_r, num_last
ja exit_1 ; if (s * 2 > num_last), we have no childs : it's high probability branch
; it's low probability branch
; s * 2 <= num_last
cmp qq, t0
jae exit_2
; qq < t0, so we go to next level
; we check 1 or 2 childs in next level
mov t0, [p + s * 8]
mov k, s
mov s, t1_r
cmp t1_r, num_last
je @F ; (s == num_last) means that we have single child in tree
; (s < num_last) : so we must read both childs and select max of them.
mov t1, [p + k * 8 + 4]
cmp t0, t1
cmovb t0, t1
adc s, 0
@@:
STORE t0, k
exit_1:
; t0 == (p)[s], s / 2 == k : (s) is index of max item from (p)[k * 2], (p)[k * 2 + 1]
cmp qq, t0
cmovb k, s
exit_2:
STORE qq, k
endm
ifdef Z7_SORT_ASM_USE_SEGMENT
; MY_ALIGN_64
else
MY_ALIGN_16
endif
MY_PROC HeapSort, 2
if (IS_LINUX gt 0)
mov p, REG_ABI_PARAM_0 ; r1 <- r7 : linux
endif
mov num_last, REG_ABI_PARAM_1 ; r10 <- r6 : linux
; r10 <- r2 : win64
cmp num_last, 2
jb end_1
; MY_PUSH_PRESERVED_ABI_REGS
MY_PUSH_PRESERVED_ABI_REGS_UP_TO_INCLUDING_R11
push r12
cmp num_last, 4
ja sort_5
LOAD a0, 0
LOAD a1, 1
SORT a0, a1
cmp num_last, 3
jb end_2
LOAD a2, 2
je sort_3
LOAD a3, 3
SORT a2, a3
SORT a1, a3
STORE a3, 3
sort_3:
SORT a0, a2
SORT a1, a2
STORE a2, 2
jmp end_2
sort_5:
; (num_last > 4) is required here
; if (num_last >= 6) : we will use optimized loop for leaf nodes loop_down_1
mov next4_lim, num_last
shr next4_lim, 2
dec num_last
mov k, num_last
shr k, 1
mov i, num_last
shr i, 2
test num_last, 1
jnz size_even
; ODD number of items. So we compare parent with single child
LOAD t1, num_last
LOAD t0, k
SORT_2_WITH_TEMP_REG t1, t0, t2
STORE t1, num_last
STORE t0, k
dec k
size_even:
cmp k, i
jbe loop_down ; jump for num_last == 4 case
if 0 ; 1 for debug
mov r15, k
mov r14d, 1 ; 100
loop_benchmark:
endif
; optimized loop for leaf nodes:
mov t0, [p + k * 8]
mov t1, [p + k * 8 + 4]
MY_ALIGN_16
loop_down_1:
; we compare parent with max of childs:
; lea s, dword ptr [2 * k]
mov s, k
cmp t0, t1
cmovb t0, t1
adc s, s
LOAD t2, k
STORE t0, k
cmp t2, t0
cmovae s, k
dec k
; we preload next items before STORE operation for calculated address
mov t0, [p + k * 8]
mov t1, [p + k * 8 + 4]
STORE t2, s
cmp k, i
jne loop_down_1
if 0 ; 1 for debug
mov k, r15
dec r14d
jnz loop_benchmark
; jmp end_debug
endif
MY_ALIGN_16
loop_down:
mov t0, [p + i * 8]
mov t1, [p + i * 8 + 4]
LOAD qq, i
mov k, i
lea s, dword ptr [i + i]
; jmp end_debug
DOWN_use_prefetch equ 0
DOWN_num_unrolls equ 0
MOVE_SMALLEST_UP STEP_1, DOWN_use_prefetch, DOWN_num_unrolls
sub i, 1
jnb loop_down
; jmp end_debug
LOAD e0, 0
LOAD e1, 1
LEVEL_3_LIMIT equ 8 ; 8 is default, but 7 also can work
cmp num_last, LEVEL_3_LIMIT + 1
jb main_loop_sort_5
MY_ALIGN_16
main_loop_sort:
; num_last > LEVEL_3_LIMIT
; p[size--] = p[0];
LOAD qq, num_last
STORE e0, num_last
mov e0, e1
mov next4_lim, num_last
shr next4_lim, 2
mov pref_lim, num_last
shr pref_lim, NUM_PREFETCH_LEVELS + 1
dec num_last
if 0 ; 1 for debug
; that optional optimization can improve the performance, if there are identical items in array
; 3 times improvement : if all items in array are identical
; 20% improvement : if items are different for 1 bit only
; 1-10% improvement : if items are different for (2+) bits
; no gain : if items are different
cmp qq, e1
jae next_iter_main
endif
LOAD e1, 2
LOAD t0, 3
mov k_x, 2
cmp e1, t0
cmovb e1, t0
mov t0, [p + 4 * (4 + 0)]
mov t1, [p + 4 * (4 + 1)]
cmovb t0, [p + 4 * (4 + 2)]
cmovb t1, [p + 4 * (4 + 3)]
adc k_x, 0
; (qq <= e1), because the tree is correctly sorted
; also here we could check (qq >= e1) or (qq == e1) for faster exit
lea s, dword ptr [k + k]
MAIN_use_prefetch equ 1
MAIN_num_unrolls equ 0
MOVE_SMALLEST_UP STEP_2, MAIN_use_prefetch, MAIN_num_unrolls
next_iter_main:
cmp num_last, LEVEL_3_LIMIT
jne main_loop_sort
; num_last == LEVEL_3_LIMIT
main_loop_sort_5:
; 4 <= num_last <= LEVEL_3_LIMIT
; p[size--] = p[0];
LOAD qq, num_last
STORE e0, num_last
mov e0, e1
dec num_last_x
LOAD e1, 2
LOAD t0, 3
mov k_x, 2
cmp e1, t0
cmovb e1, t0
adc k_x, 0
lea s_x, dword ptr [k * 2]
cmp s_x, num_last_x
ja exit_2
mov t0, [p + k * 8]
je exit_1
; s < num_last
mov t1, [p + k * 8 + 4]
cmp t0, t1
cmovb t0, t1
adc s_x, 0
exit_1:
STORE t0, k
cmp qq, t0
cmovb k_x, s_x
exit_2:
STORE qq, k
cmp num_last_x, 3
jne main_loop_sort_5
; num_last == 3 (real_size == 4)
LOAD a0, 2
LOAD a1, 3
STORE e1, 2
STORE e0, 3
SORT a0, a1
end_2:
STORE a0, 0
STORE a1, 1
; end_debug:
; MY_POP_PRESERVED_ABI_REGS
pop r12
MY_POP_PRESERVED_ABI_REGS_UP_TO_INCLUDING_R11
end_1:
MY_ENDP
else
; ------------ x86 32-bit ------------
ifdef x64
IS_CDECL = 0
endif
acc equ x0
k equ r0
k_x equ acc
p equ r1
num_last equ r2
num_last_x equ x2
a0 equ x3
t0 equ a0
a3 equ x5
i equ r5
e0 equ a3
a1 equ x6
qq equ a1
a2 equ x7
s equ r7
s_x equ a2
SORT macro b0, b1
cmp b1, b0
jae @F
if 1
xchg b0, b1
else
mov acc, b0
mov b0, b1 ; min
mov b1, acc ; max
endif
@@:
endm
LOAD macro dest:req, index:req
mov dest, [p + 4 * index]
endm
STORE macro reg:req, index:req
mov [p + 4 * index], reg
endm
STEP_1 macro exit_label
mov t0, [p + k * 8]
cmp t0, [p + k * 8 + 4]
adc s, 0
LOAD t0, s
STORE t0, k ; we lookahed stooring for most expected branch
cmp qq, t0
jae exit_label
; STORE t0, k ; use if
mov k, s
add s, s
; lea s, dword ptr [s + s]
; shl s, 1
; lea s, dword ptr [s * 2]
endm
STEP_BRANCH macro exit_label
mov t0, [p + k * 8]
cmp t0, [p + k * 8 + 4]
jae @F
inc s
mov t0, [p + k * 8 + 4]
@@:
cmp qq, t0
jae exit_label
STORE t0, k
mov k, s
add s, s
endm
MOVE_SMALLEST_UP macro STEP, num_unrolls, exit_2
LOCAL leaves, opt_loop, single
; s == k * 2
rept num_unrolls
cmp s, num_last
jae leaves
STEP_1 exit_2
endm
cmp s, num_last
jb opt_loop
leaves:
; (s >= num_last)
jne exit_2
single:
; (s == num_last)
mov t0, [p + k * 8]
cmp qq, t0
jae exit_2
STORE t0, k
mov k, s
jmp exit_2
MY_ALIGN_16
opt_loop:
STEP exit_2
cmp s, num_last
jb opt_loop
je single
exit_2:
STORE qq, k
endm
ifdef Z7_SORT_ASM_USE_SEGMENT
; MY_ALIGN_64
else
MY_ALIGN_16
endif
MY_PROC HeapSort, 2
ifdef x64
if (IS_LINUX gt 0)
mov num_last, REG_ABI_PARAM_1 ; r2 <- r6 : linux
mov p, REG_ABI_PARAM_0 ; r1 <- r7 : linux
endif
elseif (IS_CDECL gt 0)
mov num_last, [r4 + REG_SIZE * 2]
mov p, [r4 + REG_SIZE * 1]
endif
cmp num_last, 2
jb end_1
MY_PUSH_PRESERVED_ABI_REGS_UP_TO_INCLUDING_R11
cmp num_last, 4
ja sort_5
LOAD a0, 0
LOAD a1, 1
SORT a0, a1
cmp num_last, 3
jb end_2
LOAD a2, 2
je sort_3
LOAD a3, 3
SORT a2, a3
SORT a1, a3
STORE a3, 3
sort_3:
SORT a0, a2
SORT a1, a2
STORE a2, 2
jmp end_2
sort_5:
; num_last > 4
lea i, dword ptr [num_last - 2]
dec num_last
test i, 1
jz loop_down
; single child
mov t0, [p + num_last * 4]
mov qq, [p + num_last * 2]
dec i
cmp qq, t0
jae loop_down
mov [p + num_last * 2], t0
mov [p + num_last * 4], qq
MY_ALIGN_16
loop_down:
mov t0, [p + i * 4]
cmp t0, [p + i * 4 + 4]
mov k, i
mov qq, [p + i * 2]
adc k, 0
LOAD t0, k
cmp qq, t0
jae down_next
mov [p + i * 2], t0
lea s, dword ptr [k + k]
DOWN_num_unrolls equ 0
MOVE_SMALLEST_UP STEP_1, DOWN_num_unrolls, down_exit_label
down_next:
sub i, 2
jnb loop_down
; jmp end_debug
LOAD e0, 0
MY_ALIGN_16
main_loop_sort:
; num_last > 3
mov t0, [p + 2 * 4]
cmp t0, [p + 3 * 4]
LOAD qq, num_last
STORE e0, num_last
LOAD e0, 1
mov s_x, 2
mov k_x, 1
adc s, 0
LOAD t0, s
dec num_last
cmp qq, t0
jae main_exit_label
STORE t0, 1
mov k, s
add s, s
if 1
; for branch data prefetch mode :
; it's faster for large arrays : larger than (1 << 13) items.
MAIN_num_unrolls equ 10
STEP_LOOP equ STEP_BRANCH
else
MAIN_num_unrolls equ 0
STEP_LOOP equ STEP_1
endif
MOVE_SMALLEST_UP STEP_LOOP, MAIN_num_unrolls, main_exit_label
; jmp end_debug
cmp num_last, 3
jne main_loop_sort
; num_last == 3 (real_size == 4)
LOAD a0, 2
LOAD a1, 3
LOAD a2, 1
STORE e0, 3 ; e0 is alias for a3
STORE a2, 2
SORT a0, a1
end_2:
STORE a0, 0
STORE a1, 1
; end_debug:
MY_POP_PRESERVED_ABI_REGS_UP_TO_INCLUDING_R11
end_1:
MY_ENDP
endif
ifdef Z7_SORT_ASM_USE_SEGMENT
_TEXT$Z7_SORT ENDS
endif
if 0
LEA_IS_D8 (R64) [R2 * 4 + 16]
Lat : TP
2 : 1 : adl-e
2 : 3 p056 adl-p
1 : 2 : p15 hsw-rocket
1 : 2 : p01 snb-ivb
1 : 1 : p1 conroe-wsm
1 : 4 : zen3,zen4
2 : 4 : zen1,zen2
LEA_B_IS (R64) [R2 + R3 * 4]
Lat : TP
1 : 1 : adl-e
2 : 3 p056 adl-p
1 : 2 : p15 hsw-rocket
1 : 2 : p01 snb-ivb
1 : 1 : p1 nhm-wsm
1 : 1 : p0 conroe-wsm
1 : 4 : zen3,zen4
2 :2,4 : zen1,zen2
LEA_B_IS_D8 (R64) [R2 + R3 * 4 + 16]
Lat : TP
2 : 1 : adl-e
2 : 3 p056 adl-p
1 : 2 : p15 ice-rocket
3 : 1 : p1/p15 hsw-rocket
3 : 1 : p01 snb-ivb
1 : 1 : p1 nhm-wsm
1 : 1 : p0 conroe-wsm
2,1 : 2 : zen3,zen4
2 : 2 : zen1,zen2
CMOVB (R64, R64)
Lat : TP
1,2 : 2 : adl-e
1 : 2 p06 adl-p
1 : 2 : p06 bwd-rocket
1,2 : 2 : p0156+p06 hsw
1,2 :1.5 : p015+p05 snb-ivb
1,2 : 1 : p015+p05 nhm
1 : 1 : 2*p015 conroe
1 : 2 : zen3,zen4
1 : 4 : zen1,zen2
ADC (R64, 0)
Lat : TP
1,2 : 2 : adl-e
1 : 2 p06 adl-p
1 : 2 : p06 bwd-rocket
1 :1.5 : p0156+p06 hsw
1 :1.5 : p015+p05 snb-ivb
2 : 1 : 2*p015 conroe-wstm
1 : 2 : zen1,zen2,zen3,zen4
PREFETCHNTA : fetch data into non-temporal cache close to the processor, minimizing cache pollution.
L1 : Pentium3
L2 : NetBurst
L1, not L2: Core duo, Core 2, Atom processors
L1, not L2, may fetch into L3 with fast replacement: Nehalem, Westmere, Sandy Bridge, ...
NEHALEM: Fills L1/L3, L1 LRU is not updated
L3 with fast replacement: Xeon Processors based on Nehalem, Westmere, Sandy Bridge, ...
PREFETCHT0 : fetch data into all cache levels.
PREFETCHT1 : fetch data into L2 and L3
endif
end

10
C/7zVersion.h
View file

@ -1,7 +1,7 @@
#define MY_VER_MAJOR 24
#define MY_VER_MINOR 9
#define MY_VER_MAJOR 26
#define MY_VER_MINOR 0
#define MY_VER_BUILD 0
#define MY_VERSION_NUMBERS "24.09"
#define MY_VERSION_NUMBERS "26.00"
#define MY_VERSION MY_VERSION_NUMBERS
#ifdef MY_CPU_NAME
@ -10,12 +10,12 @@
#define MY_VERSION_CPU MY_VERSION
#endif
#define MY_DATE "2024-11-29"
#define MY_DATE "2026-02-12"
#undef MY_COPYRIGHT
#undef MY_VERSION_COPYRIGHT_DATE
#define MY_AUTHOR_NAME "Igor Pavlov"
#define MY_COPYRIGHT_PD "Igor Pavlov : Public domain"
#define MY_COPYRIGHT_CR "Copyright (c) 1999-2024 Igor Pavlov"
#define MY_COPYRIGHT_CR "Copyright (c) 1999-2026 Igor Pavlov"
#ifdef USE_COPYRIGHT_CR
#define MY_COPYRIGHT MY_COPYRIGHT_CR

10
C/7zWindows.h
View file

@ -1,11 +1,17 @@
/* 7zWindows.h -- StdAfx
2023-04-02 : Igor Pavlov : Public domain */
/* 7zWindows.h -- Windows.h and related code
Igor Pavlov : Public domain */
#ifndef ZIP7_INC_7Z_WINDOWS_H
#define ZIP7_INC_7Z_WINDOWS_H
#ifdef _WIN32
#if defined(_MSC_VER) && _MSC_VER >= 1950 && !defined(__clang__) // VS2026
// <Windows.h> and some another windows files need that option
// VS2026: wtypesbase.h: warning C4865: 'tagCLSCTX': the underlying type will change from 'int' to 'unsigned int' when '/Zc:enumTypes' is specified on the command line
#pragma warning(disable : 4865)
#endif
#if defined(__clang__)
# pragma clang diagnostic push
#endif

2
C/7zip_gcc_c.mak
View file

@ -106,7 +106,7 @@ DEL_OBJ_EXE = -$(RM) $(O)\*.o $(O)\$(PROG).exe $(O)\$(PROG).dll
endif
LIB2 = -lOle32 -loleaut32 -luuid -ladvapi32 -lUser32 -lShell32
LIB2 = -lole32 -loleaut32 -luuid -ladvapi32 -luser32 -lshell32
CFLAGS_EXTRA = -DUNICODE -D_UNICODE
# -Wno-delete-non-virtual-dtor

470
C/BwtSort.c
View file

@ -1,5 +1,5 @@
/* BwtSort.c -- BWT block sorting
2023-04-02 : Igor Pavlov : Public domain */
: Igor Pavlov : Public domain */
#include "Precomp.h"
@ -7,6 +7,44 @@
#include "Sort.h"
/* #define BLOCK_SORT_USE_HEAP_SORT */
// #define BLOCK_SORT_USE_HEAP_SORT
#ifdef BLOCK_SORT_USE_HEAP_SORT
#define HeapSortRefDown(p, vals, n, size, temp) \
{ size_t k = n; UInt32 val = vals[temp]; for (;;) { \
size_t s = k << 1; \
if (s > size) break; \
if (s < size && vals[p[s + 1]] > vals[p[s]]) s++; \
if (val >= vals[p[s]]) break; \
p[k] = p[s]; k = s; \
} p[k] = temp; }
void HeapSortRef(UInt32 *p, UInt32 *vals, size_t size)
{
if (size <= 1)
return;
p--;
{
size_t i = size / 2;
do
{
UInt32 temp = p[i];
HeapSortRefDown(p, vals, i, size, temp);
}
while (--i != 0);
}
do
{
UInt32 temp = p[size];
p[size--] = p[1];
HeapSortRefDown(p, vals, 1, size, temp);
}
while (size > 1);
}
#endif // BLOCK_SORT_USE_HEAP_SORT
/* Don't change it !!! */
#define kNumHashBytes 2
@ -27,26 +65,27 @@
#else
#define kNumBitsMax 20
#define kIndexMask ((1 << kNumBitsMax) - 1)
#define kNumExtraBits (32 - kNumBitsMax)
#define kNumExtra0Bits (kNumExtraBits - 2)
#define kNumExtra0Mask ((1 << kNumExtra0Bits) - 1)
#define kNumBitsMax 20
#define kIndexMask (((UInt32)1 << kNumBitsMax) - 1)
#define kNumExtraBits (32 - kNumBitsMax)
#define kNumExtra0Bits (kNumExtraBits - 2)
#define kNumExtra0Mask ((1 << kNumExtra0Bits) - 1)
#define SetFinishedGroupSize(p, size) \
{ *(p) |= ((((size) - 1) & kNumExtra0Mask) << kNumBitsMax); \
{ *(p) |= ((((UInt32)(size) - 1) & kNumExtra0Mask) << kNumBitsMax); \
if ((size) > (1 << kNumExtra0Bits)) { \
*(p) |= 0x40000000; *((p) + 1) |= ((((size) - 1)>> kNumExtra0Bits) << kNumBitsMax); } } \
*(p) |= 0x40000000; \
*((p) + 1) |= (((UInt32)(size) - 1) >> kNumExtra0Bits) << kNumBitsMax; } } \
static void SetGroupSize(UInt32 *p, UInt32 size)
static void SetGroupSize(UInt32 *p, size_t size)
{
if (--size == 0)
return;
*p |= 0x80000000 | ((size & kNumExtra0Mask) << kNumBitsMax);
*p |= 0x80000000 | (((UInt32)size & kNumExtra0Mask) << kNumBitsMax);
if (size >= (1 << kNumExtra0Bits))
{
*p |= 0x40000000;
p[1] |= ((size >> kNumExtra0Bits) << kNumBitsMax);
p[1] |= (((UInt32)size >> kNumExtra0Bits) << kNumBitsMax);
}
}
@ -59,12 +98,14 @@ returns: 1 - if there are groups, 0 - no more groups
*/
static
UInt32
unsigned
Z7_FASTCALL
SortGroup(UInt32 BlockSize, UInt32 NumSortedBytes, UInt32 groupOffset, UInt32 groupSize, int NumRefBits, UInt32 *Indices
#ifndef BLOCK_SORT_USE_HEAP_SORT
, UInt32 left, UInt32 range
#endif
SortGroup(size_t BlockSize, size_t NumSortedBytes,
size_t groupOffset, size_t groupSize,
unsigned NumRefBits, UInt32 *Indices
#ifndef BLOCK_SORT_USE_HEAP_SORT
, size_t left, size_t range
#endif
)
{
UInt32 *ind2 = Indices + groupOffset;
@ -79,90 +120,93 @@ SortGroup(UInt32 BlockSize, UInt32 NumSortedBytes, UInt32 groupOffset, UInt32 gr
return 0;
}
Groups = Indices + BlockSize + BS_TEMP_SIZE;
if (groupSize <= ((UInt32)1 << NumRefBits)
#ifndef BLOCK_SORT_USE_HEAP_SORT
if (groupSize <= ((size_t)1 << NumRefBits)
#ifndef BLOCK_SORT_USE_HEAP_SORT
&& groupSize <= range
#endif
#endif
)
{
UInt32 *temp = Indices + BlockSize;
UInt32 j;
UInt32 mask, thereAreGroups, group, cg;
size_t j, group;
UInt32 mask, cg;
unsigned thereAreGroups;
{
UInt32 gPrev;
UInt32 gRes = 0;
{
UInt32 sp = ind2[0] + NumSortedBytes;
if (sp >= BlockSize) sp -= BlockSize;
size_t sp = ind2[0] + NumSortedBytes;
if (sp >= BlockSize)
sp -= BlockSize;
gPrev = Groups[sp];
temp[0] = (gPrev << NumRefBits);
temp[0] = gPrev << NumRefBits;
}
for (j = 1; j < groupSize; j++)
{
UInt32 sp = ind2[j] + NumSortedBytes;
size_t sp = ind2[j] + NumSortedBytes;
UInt32 g;
if (sp >= BlockSize) sp -= BlockSize;
if (sp >= BlockSize)
sp -= BlockSize;
g = Groups[sp];
temp[j] = (g << NumRefBits) | j;
temp[j] = (g << NumRefBits) | (UInt32)j;
gRes |= (gPrev ^ g);
}
if (gRes == 0)
{
#ifndef BLOCK_SORT_EXTERNAL_FLAGS
#ifndef BLOCK_SORT_EXTERNAL_FLAGS
SetGroupSize(ind2, groupSize);
#endif
#endif
return 1;
}
}
HeapSort(temp, groupSize);
mask = (((UInt32)1 << NumRefBits) - 1);
mask = ((UInt32)1 << NumRefBits) - 1;
thereAreGroups = 0;
group = groupOffset;
cg = (temp[0] >> NumRefBits);
cg = temp[0] >> NumRefBits;
temp[0] = ind2[temp[0] & mask];
{
#ifdef BLOCK_SORT_EXTERNAL_FLAGS
#ifdef BLOCK_SORT_EXTERNAL_FLAGS
UInt32 *Flags = Groups + BlockSize;
#else
UInt32 prevGroupStart = 0;
#endif
#else
size_t prevGroupStart = 0;
#endif
for (j = 1; j < groupSize; j++)
{
UInt32 val = temp[j];
UInt32 cgCur = (val >> NumRefBits);
const UInt32 val = temp[j];
const UInt32 cgCur = val >> NumRefBits;
if (cgCur != cg)
{
cg = cgCur;
group = groupOffset + j;
#ifdef BLOCK_SORT_EXTERNAL_FLAGS
#ifdef BLOCK_SORT_EXTERNAL_FLAGS
{
UInt32 t = group - 1;
Flags[t >> kNumFlagsBits] &= ~(1 << (t & kFlagsMask));
const size_t t = group - 1;
Flags[t >> kNumFlagsBits] &= ~((UInt32)1 << (t & kFlagsMask));
}
#else
#else
SetGroupSize(temp + prevGroupStart, j - prevGroupStart);
prevGroupStart = j;
#endif
#endif
}
else
thereAreGroups = 1;
{
UInt32 ind = ind2[val & mask];
temp[j] = ind;
Groups[ind] = group;
const UInt32 ind = ind2[val & mask];
temp[j] = ind;
Groups[ind] = (UInt32)group;
}
}
#ifndef BLOCK_SORT_EXTERNAL_FLAGS
#ifndef BLOCK_SORT_EXTERNAL_FLAGS
SetGroupSize(temp + prevGroupStart, j - prevGroupStart);
#endif
#endif
}
for (j = 0; j < groupSize; j++)
@ -172,37 +216,42 @@ SortGroup(UInt32 BlockSize, UInt32 NumSortedBytes, UInt32 groupOffset, UInt32 gr
/* Check that all strings are in one group (cannot sort) */
{
UInt32 group, j;
UInt32 sp = ind2[0] + NumSortedBytes; if (sp >= BlockSize) sp -= BlockSize;
UInt32 group;
size_t j;
size_t sp = ind2[0] + NumSortedBytes;
if (sp >= BlockSize)
sp -= BlockSize;
group = Groups[sp];
for (j = 1; j < groupSize; j++)
{
sp = ind2[j] + NumSortedBytes; if (sp >= BlockSize) sp -= BlockSize;
sp = ind2[j] + NumSortedBytes;
if (sp >= BlockSize)
sp -= BlockSize;
if (Groups[sp] != group)
break;
}
if (j == groupSize)
{
#ifndef BLOCK_SORT_EXTERNAL_FLAGS
#ifndef BLOCK_SORT_EXTERNAL_FLAGS
SetGroupSize(ind2, groupSize);
#endif
#endif
return 1;
}
}
#ifndef BLOCK_SORT_USE_HEAP_SORT
#ifndef BLOCK_SORT_USE_HEAP_SORT
{
/* ---------- Range Sort ---------- */
UInt32 i;
UInt32 mid;
size_t i;
size_t mid;
for (;;)
{
UInt32 j;
size_t j;
if (range <= 1)
{
#ifndef BLOCK_SORT_EXTERNAL_FLAGS
#ifndef BLOCK_SORT_EXTERNAL_FLAGS
SetGroupSize(ind2, groupSize);
#endif
#endif
return 1;
}
mid = left + ((range + 1) >> 1);
@ -210,7 +259,7 @@ SortGroup(UInt32 BlockSize, UInt32 NumSortedBytes, UInt32 groupOffset, UInt32 gr
i = 0;
do
{
UInt32 sp = ind2[i] + NumSortedBytes; if (sp >= BlockSize) sp -= BlockSize;
size_t sp = ind2[i] + NumSortedBytes; if (sp >= BlockSize) sp -= BlockSize;
if (Groups[sp] >= mid)
{
for (j--; j > i; j--)
@ -238,51 +287,53 @@ SortGroup(UInt32 BlockSize, UInt32 NumSortedBytes, UInt32 groupOffset, UInt32 gr
break;
}
#ifdef BLOCK_SORT_EXTERNAL_FLAGS
#ifdef BLOCK_SORT_EXTERNAL_FLAGS
{
UInt32 t = (groupOffset + i - 1);
const size_t t = groupOffset + i - 1;
UInt32 *Flags = Groups + BlockSize;
Flags[t >> kNumFlagsBits] &= ~(1 << (t & kFlagsMask));
Flags[t >> kNumFlagsBits] &= ~((UInt32)1 << (t & kFlagsMask));
}
#endif
#endif
{
UInt32 j;
size_t j;
for (j = i; j < groupSize; j++)
Groups[ind2[j]] = groupOffset + i;
Groups[ind2[j]] = (UInt32)(groupOffset + i);
}
{
UInt32 res = SortGroup(BlockSize, NumSortedBytes, groupOffset, i, NumRefBits, Indices, left, mid - left);
return res | SortGroup(BlockSize, NumSortedBytes, groupOffset + i, groupSize - i, NumRefBits, Indices, mid, range - (mid - left));
unsigned res = SortGroup(BlockSize, NumSortedBytes, groupOffset, i, NumRefBits, Indices, left, mid - left);
return res | SortGroup(BlockSize, NumSortedBytes, groupOffset + i, groupSize - i, NumRefBits, Indices, mid, range - (mid - left));
}
}
#else
#else // BLOCK_SORT_USE_HEAP_SORT
/* ---------- Heap Sort ---------- */
{
UInt32 j;
size_t j;
for (j = 0; j < groupSize; j++)
{
UInt32 sp = ind2[j] + NumSortedBytes; if (sp >= BlockSize) sp -= BlockSize;
ind2[j] = sp;
size_t sp = ind2[j] + NumSortedBytes;
if (sp >= BlockSize)
sp -= BlockSize;
ind2[j] = (UInt32)sp;
}
HeapSortRef(ind2, Groups, groupSize);
/* Write Flags */
{
UInt32 sp = ind2[0];
size_t sp = ind2[0];
UInt32 group = Groups[sp];
#ifdef BLOCK_SORT_EXTERNAL_FLAGS
#ifdef BLOCK_SORT_EXTERNAL_FLAGS
UInt32 *Flags = Groups + BlockSize;
#else
UInt32 prevGroupStart = 0;
#endif
#else
size_t prevGroupStart = 0;
#endif
for (j = 1; j < groupSize; j++)
{
@ -290,149 +341,210 @@ SortGroup(UInt32 BlockSize, UInt32 NumSortedBytes, UInt32 groupOffset, UInt32 gr
if (Groups[sp] != group)
{
group = Groups[sp];
#ifdef BLOCK_SORT_EXTERNAL_FLAGS
#ifdef BLOCK_SORT_EXTERNAL_FLAGS
{
UInt32 t = groupOffset + j - 1;
const size_t t = groupOffset + j - 1;
Flags[t >> kNumFlagsBits] &= ~(1 << (t & kFlagsMask));
}
#else
#else
SetGroupSize(ind2 + prevGroupStart, j - prevGroupStart);
prevGroupStart = j;
#endif
#endif
}
}
#ifndef BLOCK_SORT_EXTERNAL_FLAGS
#ifndef BLOCK_SORT_EXTERNAL_FLAGS
SetGroupSize(ind2 + prevGroupStart, j - prevGroupStart);
#endif
#endif
}
{
/* Write new Groups values and Check that there are groups */
UInt32 thereAreGroups = 0;
unsigned thereAreGroups = 0;
for (j = 0; j < groupSize; j++)
{
UInt32 group = groupOffset + j;
#ifndef BLOCK_SORT_EXTERNAL_FLAGS
size_t group = groupOffset + j;
#ifndef BLOCK_SORT_EXTERNAL_FLAGS
UInt32 subGroupSize = ((ind2[j] & ~0xC0000000) >> kNumBitsMax);
if ((ind2[j] & 0x40000000) != 0)
if (ind2[j] & 0x40000000)
subGroupSize += ((ind2[(size_t)j + 1] >> kNumBitsMax) << kNumExtra0Bits);
subGroupSize++;
for (;;)
{
UInt32 original = ind2[j];
UInt32 sp = original & kIndexMask;
if (sp < NumSortedBytes) sp += BlockSize; sp -= NumSortedBytes;
ind2[j] = sp | (original & ~kIndexMask);
Groups[sp] = group;
const UInt32 original = ind2[j];
size_t sp = original & kIndexMask;
if (sp < NumSortedBytes)
sp += BlockSize;
sp -= NumSortedBytes;
ind2[j] = (UInt32)sp | (original & ~kIndexMask);
Groups[sp] = (UInt32)group;
if (--subGroupSize == 0)
break;
j++;
thereAreGroups = 1;
}
#else
#else
UInt32 *Flags = Groups + BlockSize;
for (;;)
{
UInt32 sp = ind2[j]; if (sp < NumSortedBytes) sp += BlockSize; sp -= NumSortedBytes;
ind2[j] = sp;
Groups[sp] = group;
size_t sp = ind2[j];
if (sp < NumSortedBytes)
sp += BlockSize;
sp -= NumSortedBytes;
ind2[j] = (UInt32)sp;
Groups[sp] = (UInt32)group;
if ((Flags[(groupOffset + j) >> kNumFlagsBits] & (1 << ((groupOffset + j) & kFlagsMask))) == 0)
break;
j++;
thereAreGroups = 1;
}
#endif
#endif
}
return thereAreGroups;
}
}
#endif
#endif // BLOCK_SORT_USE_HEAP_SORT
}
/* conditions: blockSize > 0 */
UInt32 BlockSort(UInt32 *Indices, const Byte *data, UInt32 blockSize)
UInt32 BlockSort(UInt32 *Indices, const Byte *data, size_t blockSize)
{
UInt32 *counters = Indices + blockSize;
UInt32 i;
size_t i;
UInt32 *Groups;
#ifdef BLOCK_SORT_EXTERNAL_FLAGS
#ifdef BLOCK_SORT_EXTERNAL_FLAGS
UInt32 *Flags;
#endif
#endif
/* Radix-Sort for 2 bytes */
/* Radix-Sort for 2 bytes */
// { UInt32 yyy; for (yyy = 0; yyy < 100; yyy++) {
for (i = 0; i < kNumHashValues; i++)
counters[i] = 0;
for (i = 0; i < blockSize - 1; i++)
counters[((UInt32)data[i] << 8) | data[(size_t)i + 1]]++;
counters[((UInt32)data[i] << 8) | data[0]]++;
{
const Byte *data2 = data;
size_t a = data[(size_t)blockSize - 1];
const Byte *data_lim = data + blockSize;
if (blockSize >= 4)
{
data_lim -= 3;
do
{
size_t b;
b = data2[0]; counters[(a << 8) | b]++;
a = data2[1]; counters[(b << 8) | a]++;
b = data2[2]; counters[(a << 8) | b]++;
a = data2[3]; counters[(b << 8) | a]++;
data2 += 4;
}
while (data2 < data_lim);
data_lim += 3;
}
while (data2 != data_lim)
{
size_t b = *data2++;
counters[(a << 8) | b]++;
a = b;
}
}
// }}
Groups = counters + BS_TEMP_SIZE;
#ifdef BLOCK_SORT_EXTERNAL_FLAGS
#ifdef BLOCK_SORT_EXTERNAL_FLAGS
Flags = Groups + blockSize;
{
UInt32 numWords = (blockSize + kFlagsMask) >> kNumFlagsBits;
for (i = 0; i < numWords; i++)
Flags[i] = kAllFlags;
}
#endif
{
const size_t numWords = (blockSize + kFlagsMask) >> kNumFlagsBits;
for (i = 0; i < numWords; i++)
Flags[i] = kAllFlags;
}
#endif
{
UInt32 sum = 0;
for (i = 0; i < kNumHashValues; i++)
{
UInt32 groupSize = counters[i];
if (groupSize > 0)
const UInt32 groupSize = counters[i];
counters[i] = sum;
sum += groupSize;
#ifdef BLOCK_SORT_EXTERNAL_FLAGS
if (groupSize)
{
#ifdef BLOCK_SORT_EXTERNAL_FLAGS
UInt32 t = sum + groupSize - 1;
Flags[t >> kNumFlagsBits] &= ~(1 << (t & kFlagsMask));
#endif
sum += groupSize;
const UInt32 t = sum - 1;
Flags[t >> kNumFlagsBits] &= ~((UInt32)1 << (t & kFlagsMask));
}
counters[i] = sum - groupSize;
#endif
}
}
for (i = 0; i < blockSize - 1; i++)
Groups[i] = counters[((unsigned)data[i] << 8) | data[(size_t)i + 1]];
Groups[i] = counters[((unsigned)data[i] << 8) | data[0]];
{
#define SET_Indices(a, b, i) \
{ UInt32 c; \
a = (a << 8) | (b); \
c = counters[a]; \
Indices[c] = (UInt32)i++; \
counters[a] = c + 1; \
}
for (i = 0; i < blockSize - 1; i++)
Groups[i] = counters[((UInt32)data[i] << 8) | data[(size_t)i + 1]];
Groups[i] = counters[((UInt32)data[i] << 8) | data[0]];
for (i = 0; i < blockSize - 1; i++)
Indices[counters[((UInt32)data[i] << 8) | data[(size_t)i + 1]]++] = i;
Indices[counters[((UInt32)data[i] << 8) | data[0]]++] = i;
#ifndef BLOCK_SORT_EXTERNAL_FLAGS
size_t a = data[0];
const Byte *data_ptr = data + 1;
i = 0;
if (blockSize >= 3)
{
blockSize -= 2;
do
{
size_t b;
b = data_ptr[0]; SET_Indices(a, b, i)
a = data_ptr[1]; SET_Indices(b, a, i)
data_ptr += 2;
}
while (i < blockSize);
blockSize += 2;
}
if (i < blockSize - 1)
{
SET_Indices(a, data[(size_t)i + 1], i)
a = (Byte)a;
}
SET_Indices(a, data[0], i)
}
#ifndef BLOCK_SORT_EXTERNAL_FLAGS
{
UInt32 prev = 0;
for (i = 0; i < kNumHashValues; i++)
{
UInt32 prevGroupSize = counters[i] - prev;
const UInt32 prevGroupSize = counters[i] - prev;
if (prevGroupSize == 0)
continue;
SetGroupSize(Indices + prev, prevGroupSize);
prev = counters[i];
}
}
#endif
}
#endif
{
int NumRefBits;
UInt32 NumSortedBytes;
for (NumRefBits = 0; ((blockSize - 1) >> NumRefBits) != 0; NumRefBits++);
unsigned NumRefBits;
size_t NumSortedBytes;
for (NumRefBits = 0; ((blockSize - 1) >> NumRefBits) != 0; NumRefBits++)
{}
NumRefBits = 32 - NumRefBits;
if (NumRefBits > kNumRefBitsMax)
NumRefBits = kNumRefBitsMax;
NumRefBits = kNumRefBitsMax;
for (NumSortedBytes = kNumHashBytes; ; NumSortedBytes <<= 1)
{
#ifndef BLOCK_SORT_EXTERNAL_FLAGS
UInt32 finishedGroupSize = 0;
#endif
UInt32 newLimit = 0;
#ifndef BLOCK_SORT_EXTERNAL_FLAGS
size_t finishedGroupSize = 0;
#endif
size_t newLimit = 0;
for (i = 0; i < blockSize;)
{
UInt32 groupSize;
#ifdef BLOCK_SORT_EXTERNAL_FLAGS
size_t groupSize;
#ifdef BLOCK_SORT_EXTERNAL_FLAGS
if ((Flags[i >> kNumFlagsBits] & (1 << (i & kFlagsMask))) == 0)
{
@ -441,56 +553,56 @@ UInt32 BlockSort(UInt32 *Indices, const Byte *data, UInt32 blockSize)
}
for (groupSize = 1;
(Flags[(i + groupSize) >> kNumFlagsBits] & (1 << ((i + groupSize) & kFlagsMask))) != 0;
groupSize++);
groupSize++)
{}
groupSize++;
#else
#else
groupSize = ((Indices[i] & ~0xC0000000) >> kNumBitsMax);
groupSize = (Indices[i] & ~0xC0000000) >> kNumBitsMax;
{
BoolInt finishedGroup = ((Indices[i] & 0x80000000) == 0);
if ((Indices[i] & 0x40000000) != 0)
{
groupSize += ((Indices[(size_t)i + 1] >> kNumBitsMax) << kNumExtra0Bits);
Indices[(size_t)i + 1] &= kIndexMask;
}
Indices[i] &= kIndexMask;
groupSize++;
if (finishedGroup || groupSize == 1)
{
Indices[i - finishedGroupSize] &= kIndexMask;
if (finishedGroupSize > 1)
Indices[(size_t)(i - finishedGroupSize) + 1] &= kIndexMask;
const BoolInt finishedGroup = ((Indices[i] & 0x80000000) == 0);
if (Indices[i] & 0x40000000)
{
UInt32 newGroupSize = groupSize + finishedGroupSize;
SetFinishedGroupSize(Indices + i - finishedGroupSize, newGroupSize)
finishedGroupSize = newGroupSize;
groupSize += ((Indices[(size_t)i + 1] >> kNumBitsMax) << kNumExtra0Bits);
Indices[(size_t)i + 1] &= kIndexMask;
}
i += groupSize;
continue;
}
finishedGroupSize = 0;
Indices[i] &= kIndexMask;
groupSize++;
if (finishedGroup || groupSize == 1)
{
Indices[i - finishedGroupSize] &= kIndexMask;
if (finishedGroupSize > 1)
Indices[(size_t)(i - finishedGroupSize) + 1] &= kIndexMask;
{
const size_t newGroupSize = groupSize + finishedGroupSize;
SetFinishedGroupSize(Indices + i - finishedGroupSize, newGroupSize)
finishedGroupSize = newGroupSize;
}
i += groupSize;
continue;
}
finishedGroupSize = 0;
}
#endif
#endif
if (NumSortedBytes >= blockSize)
{
UInt32 j;
size_t j;
for (j = 0; j < groupSize; j++)
{
UInt32 t = (i + j);
size_t t = i + j;
/* Flags[t >> kNumFlagsBits] &= ~(1 << (t & kFlagsMask)); */
Groups[Indices[t]] = t;
Groups[Indices[t]] = (UInt32)t;
}
}
else
if (SortGroup(blockSize, NumSortedBytes, i, groupSize, NumRefBits, Indices
#ifndef BLOCK_SORT_USE_HEAP_SORT
, 0, blockSize
#endif
) != 0)
#ifndef BLOCK_SORT_USE_HEAP_SORT
, 0, blockSize
#endif
))
newLimit = i + groupSize;
i += groupSize;
}
@ -498,19 +610,19 @@ UInt32 BlockSort(UInt32 *Indices, const Byte *data, UInt32 blockSize)
break;
}
}
#ifndef BLOCK_SORT_EXTERNAL_FLAGS
#ifndef BLOCK_SORT_EXTERNAL_FLAGS
for (i = 0; i < blockSize;)
{
UInt32 groupSize = ((Indices[i] & ~0xC0000000) >> kNumBitsMax);
if ((Indices[i] & 0x40000000) != 0)
size_t groupSize = (Indices[i] & ~0xC0000000) >> kNumBitsMax;
if (Indices[i] & 0x40000000)
{
groupSize += ((Indices[(size_t)i + 1] >> kNumBitsMax) << kNumExtra0Bits);
groupSize += (Indices[(size_t)i + 1] >> kNumBitsMax) << kNumExtra0Bits;
Indices[(size_t)i + 1] &= kIndexMask;
}
Indices[i] &= kIndexMask;
groupSize++;
i += groupSize;
}
#endif
#endif
return Groups[0];
}

7
C/BwtSort.h
View file

@ -1,5 +1,5 @@
/* BwtSort.h -- BWT block sorting
2023-03-03 : Igor Pavlov : Public domain */
: Igor Pavlov : Public domain */
#ifndef ZIP7_INC_BWT_SORT_H
#define ZIP7_INC_BWT_SORT_H
@ -10,16 +10,17 @@ EXTERN_C_BEGIN
/* use BLOCK_SORT_EXTERNAL_FLAGS if blockSize can be > 1M */
/* #define BLOCK_SORT_EXTERNAL_FLAGS */
// #define BLOCK_SORT_EXTERNAL_FLAGS
#ifdef BLOCK_SORT_EXTERNAL_FLAGS
#define BLOCK_SORT_EXTERNAL_SIZE(blockSize) ((((blockSize) + 31) >> 5))
#define BLOCK_SORT_EXTERNAL_SIZE(blockSize) (((blockSize) + 31) >> 5)
#else
#define BLOCK_SORT_EXTERNAL_SIZE(blockSize) 0
#endif
#define BLOCK_SORT_BUF_SIZE(blockSize) ((blockSize) * 2 + BLOCK_SORT_EXTERNAL_SIZE(blockSize) + (1 << 16))
UInt32 BlockSort(UInt32 *indices, const Byte *data, UInt32 blockSize);
UInt32 BlockSort(UInt32 *indices, const Byte *data, size_t blockSize);
EXTERN_C_END

12
C/Compiler.h
View file

@ -1,5 +1,5 @@
/* Compiler.h : Compiler specific defines and pragmas
2024-01-22 : Igor Pavlov : Public domain */
: Igor Pavlov : Public domain */
#ifndef ZIP7_INC_COMPILER_H
#define ZIP7_INC_COMPILER_H
@ -183,6 +183,16 @@ typedef void (*Z7_void_Function)(void);
#define Z7_ATTRIB_NO_VECTORIZE
#endif
#if defined(Z7_MSC_VER_ORIGINAL) && (Z7_MSC_VER_ORIGINAL >= 1920)
#define Z7_PRAGMA_OPTIMIZE_FOR_CODE_SIZE _Pragma("optimize ( \"s\", on )")
#define Z7_PRAGMA_OPTIMIZE_DEFAULT _Pragma("optimize ( \"\", on )")
#else
#define Z7_PRAGMA_OPTIMIZE_FOR_CODE_SIZE
#define Z7_PRAGMA_OPTIMIZE_DEFAULT
#endif
#if defined(MY_CPU_X86_OR_AMD64) && ( \
defined(__clang__) && (__clang_major__ >= 4) \
|| defined(__GNUC__) && (__GNUC__ >= 5))

4
C/CpuArch.c
View file

@ -859,7 +859,7 @@ BoolInt CPU_IsSupported_AES (void) { return APPLE_CRYPTO_SUPPORT_VAL; }
#if defined(__GLIBC__) && (__GLIBC__ * 100 + __GLIBC_MINOR__ >= 216)
#define Z7_GETAUXV_AVAILABLE
#else
#elif !defined(__QNXNTO__)
// #pragma message("=== is not NEW GLIBC === ")
#if defined __has_include
#if __has_include (<sys/auxv.h>)
@ -877,7 +877,7 @@ BoolInt CPU_IsSupported_AES (void) { return APPLE_CRYPTO_SUPPORT_VAL; }
#ifdef USE_HWCAP
#if defined(__FreeBSD__)
#if defined(__FreeBSD__) || defined(__OpenBSD__)
static unsigned long MY_getauxval(int aux)
{
unsigned long val;

27
C/CpuArch.h
View file

@ -31,7 +31,12 @@ MY_CPU_64BIT means that processor can work with 64-bit registers.
#define MY_CPU_NAME "x32"
#define MY_CPU_SIZEOF_POINTER 4
#else
#define MY_CPU_NAME "x64"
#if defined(__APX_EGPR__) || defined(__EGPR__)
#define MY_CPU_NAME "x64-apx"
#define MY_CPU_AMD64_APX
#else
#define MY_CPU_NAME "x64"
#endif
#define MY_CPU_SIZEOF_POINTER 8
#endif
#define MY_CPU_64BIT
@ -47,6 +52,12 @@ MY_CPU_64BIT means that processor can work with 64-bit registers.
#define MY_CPU_SIZEOF_POINTER 4
#endif
#if defined(__SSE2__) \
|| defined(MY_CPU_AMD64) \
|| defined(_M_IX86_FP) && (_M_IX86_FP >= 2)
#define MY_CPU_SSE2
#endif
#if defined(_M_ARM64) \
|| defined(_M_ARM64EC) \
@ -571,10 +582,12 @@ problem-4 : performace:
#define Z7_CONV_BE_TO_NATIVE_CONST32(v) (v)
#define Z7_CONV_LE_TO_NATIVE_CONST32(v) Z7_BSWAP32_CONST(v)
#define Z7_CONV_NATIVE_TO_BE_32(v) (v)
// #define Z7_GET_NATIVE16_FROM_2_BYTES(b0, b1) ((b1) | ((b0) << 8))
#elif defined(MY_CPU_LE)
#define Z7_CONV_BE_TO_NATIVE_CONST32(v) Z7_BSWAP32_CONST(v)
#define Z7_CONV_LE_TO_NATIVE_CONST32(v) (v)
#define Z7_CONV_NATIVE_TO_BE_32(v) Z7_BSWAP32(v)
// #define Z7_GET_NATIVE16_FROM_2_BYTES(b0, b1) ((b0) | ((b1) << 8))
#else
#error Stop_Compiling_Unknown_Endian_CONV
#endif
@ -588,8 +601,20 @@ problem-4 : performace:
#define SetBe32a(p, v) { *(UInt32 *)(void *)(p) = (v); }
#define SetBe16a(p, v) { *(UInt16 *)(void *)(p) = (v); }
// gcc and clang for powerpc can transform load byte access to load reverse word access.
// sp we can use byte access instead of word access. Z7_BSWAP64 cab be slow
#if 1 && defined(Z7_CPU_FAST_BSWAP_SUPPORTED) && defined(MY_CPU_64BIT)
#define GetUi64a(p) Z7_BSWAP64 (*(const UInt64 *)(const void *)(p))
#else
#define GetUi64a(p) GetUi64(p)
#endif
#if 1 && defined(Z7_CPU_FAST_BSWAP_SUPPORTED)
#define GetUi32a(p) Z7_BSWAP32 (*(const UInt32 *)(const void *)(p))
#else
#define GetUi32a(p) GetUi32(p)
#endif
#define GetUi16a(p) GetUi16(p)
#define SetUi32a(p, v) SetUi32(p, v)
#define SetUi16a(p, v) SetUi16(p, v)

397
C/HuffEnc.c
View file

@ -1,60 +1,126 @@
/* HuffEnc.c -- functions for Huffman encoding
2023-09-07 : Igor Pavlov : Public domain */
Igor Pavlov : Public domain */
#include "Precomp.h"
#include <string.h>
#include "HuffEnc.h"
#include "Sort.h"
#include "CpuArch.h"
#define kMaxLen 16
#define NUM_BITS 10
#define MASK ((1u << NUM_BITS) - 1)
#define kMaxLen Z7_HUFFMAN_LEN_MAX
#define NUM_BITS 10
#define MASK ((1u << NUM_BITS) - 1)
#define FREQ_MASK (~(UInt32)MASK)
#define NUM_COUNTERS (104 * 2) // (80 * 2) or (128 * 2) : ((prime_number + 1) * 2) for smaller code.
#define NUM_COUNTERS 64
#if 1 && (defined(MY_CPU_LE) || defined(MY_CPU_BE))
#if defined(MY_CPU_LE)
#define HI_HALF_OFFSET 1
#else
#define HI_HALF_OFFSET 0
#endif
#define LOAD_PARENT(p) ((unsigned)*((const UInt16 *)(p) + HI_HALF_OFFSET))
#define STORE_PARENT(p, fb, val) *((UInt16 *)(p) + HI_HALF_OFFSET) = (UInt16)(val);
#define STORE_PARENT_DIRECT(p, fb, hi) STORE_PARENT(p, fb, hi)
#define UPDATE_E(eHi) eHi++;
#else
#define LOAD_PARENT(p) ((unsigned)(*(p) >> NUM_BITS))
#define STORE_PARENT_DIRECT(p, fb, hi) *(p) = ((fb) & MASK) | (hi); // set parent field
#define STORE_PARENT(p, fb, val) STORE_PARENT_DIRECT(p, fb, ((UInt32)(val) << NUM_BITS))
#define UPDATE_E(eHi) eHi += 1 << NUM_BITS;
#endif
#define HUFFMAN_SPEED_OPT
void Huffman_Generate(const UInt32 *freqs, UInt32 *p, Byte *lens, UInt32 numSymbols, UInt32 maxLen)
void Huffman_Generate(const UInt32 *freqs, UInt32 *p, Byte *lens, unsigned numSymbols, unsigned maxLen)
{
UInt32 num = 0;
/* if (maxLen > 10) maxLen = 10; */
#if NUM_COUNTERS > 2
unsigned counters[NUM_COUNTERS];
#endif
#if 1 && NUM_COUNTERS > (kMaxLen + 4) * 2
#define lenCounters (counters)
#define codes (counters + kMaxLen + 4)
#else
unsigned lenCounters[kMaxLen + 1];
UInt32 codes[kMaxLen + 1];
#endif
unsigned num;
{
UInt32 i;
unsigned i;
// UInt32 sum = 0;
#if NUM_COUNTERS > 2
#ifdef HUFFMAN_SPEED_OPT
UInt32 counters[NUM_COUNTERS];
#define CTR_ITEM_FOR_FREQ(freq) \
counters[(freq) >= NUM_COUNTERS - 1 ? NUM_COUNTERS - 1 : (unsigned)(freq)]
for (i = 0; i < NUM_COUNTERS; i++)
counters[i] = 0;
for (i = 0; i < numSymbols; i++)
memset(lens, 0, numSymbols);
{
UInt32 freq = freqs[i];
counters[(freq < NUM_COUNTERS - 1) ? freq : NUM_COUNTERS - 1]++;
const UInt32 *fp = freqs + numSymbols;
#define NUM_UNROLLS 1
#if NUM_UNROLLS > 1 // use 1 if odd (numSymbols) is possisble
if (numSymbols & 1)
{
UInt32 f;
f = *--fp; CTR_ITEM_FOR_FREQ(f)++;
// sum += f;
}
#endif
do
{
UInt32 f;
fp -= NUM_UNROLLS;
f = fp[0]; CTR_ITEM_FOR_FREQ(f)++;
// sum += f;
#if NUM_UNROLLS > 1
f = fp[1]; CTR_ITEM_FOR_FREQ(f)++;
// sum += f;
#endif
}
while (fp != freqs);
}
for (i = 1; i < NUM_COUNTERS; i++)
#if 0
printf("\nsum=%8u numSymbols =%3u ctrs:", sum, numSymbols);
{
UInt32 temp = counters[i];
counters[i] = num;
num += temp;
unsigned k = 0;
for (k = 0; k < NUM_COUNTERS; k++)
printf(" %u", counters[k]);
}
for (i = 0; i < numSymbols; i++)
#endif
num = counters[1];
counters[1] = 0;
for (i = 2; i != NUM_COUNTERS; i += 2)
{
UInt32 freq = freqs[i];
if (freq == 0)
lens[i] = 0;
else
p[counters[((freq < NUM_COUNTERS - 1) ? freq : NUM_COUNTERS - 1)]++] = i | (freq << NUM_BITS);
const unsigned c0 = (counters )[i];
const unsigned c1 = (counters + 1)[i];
(counters )[i] = num; num += c0;
(counters + 1)[i] = num; num += c1;
}
counters[0] = num; // we want to write (freq==0) symbols to the end of (p) array
{
i = 0;
do
{
const UInt32 f = freqs[i];
#if 0
if (f == 0) lens[i] = 0; else
#endif
p[CTR_ITEM_FOR_FREQ(f)++] = i | (f << NUM_BITS);
}
while (++i != numSymbols);
}
counters[0] = 0;
HeapSort(p + counters[NUM_COUNTERS - 2], counters[NUM_COUNTERS - 1] - counters[NUM_COUNTERS - 2]);
#else
#else // NUM_COUNTERS <= 2
num = 0;
for (i = 0; i < numSymbols; i++)
{
UInt32 freq = freqs[i];
const UInt32 freq = freqs[i];
if (freq == 0)
lens[i] = 0;
else
@ -62,17 +128,27 @@ void Huffman_Generate(const UInt32 *freqs, UInt32 *p, Byte *lens, UInt32 numSymb
}
HeapSort(p, num);
#endif
#endif
}
if (num < 2)
if (num <= 2)
{
unsigned minCode = 0;
unsigned maxCode = 1;
if (num == 1)
if (num)
{
maxCode = (unsigned)p[0] & MASK;
if (maxCode == 0)
maxCode = (unsigned)p[(size_t)num - 1] & MASK;
if (num == 2)
{
minCode = (unsigned)p[0] & MASK;
if (minCode > maxCode)
{
const unsigned temp = minCode;
minCode = maxCode;
maxCode = temp;
}
}
else if (maxCode == 0)
maxCode++;
}
p[minCode] = 0;
@ -80,75 +156,206 @@ void Huffman_Generate(const UInt32 *freqs, UInt32 *p, Byte *lens, UInt32 numSymb
lens[minCode] = lens[maxCode] = 1;
return;
}
{
UInt32 b, e, i;
i = b = e = 0;
do
unsigned i;
for (i = 0; i <= kMaxLen; i++)
lenCounters[i] = 0;
lenCounters[1] = 2; // by default root node has 2 child leaves at level 1.
}
// if (num != 2)
{
// num > 2
// the binary tree will contain (num - 1) internal nodes.
// p[num - 2] will be root node of binary tree.
UInt32 *b;
UInt32 *n;
// first node will have two leaf childs: p[0] and p[1]:
// p[0] += p[1] & FREQ_MASK; // set frequency sum of child leafs
// if (pi == n) exit(0);
// if (pi != n)
{
UInt32 n, m, freq;
n = (i != num && (b == e || (p[i] >> NUM_BITS) <= (p[b] >> NUM_BITS))) ? i++ : b++;
freq = (p[n] & ~MASK);
p[n] = (p[n] & MASK) | (e << NUM_BITS);
m = (i != num && (b == e || (p[i] >> NUM_BITS) <= (p[b] >> NUM_BITS))) ? i++ : b++;
freq += (p[m] & ~MASK);
p[m] = (p[m] & MASK) | (e << NUM_BITS);
p[e] = (p[e] & MASK) | freq;
e++;
}
while (num - e > 1);
{
UInt32 lenCounters[kMaxLen + 1];
for (i = 0; i <= kMaxLen; i++)
lenCounters[i] = 0;
p[--e] &= MASK;
lenCounters[1] = 2;
while (e != 0)
UInt32 fb = (p[1] & FREQ_MASK) + p[0];
UInt32 f = p[2] & FREQ_MASK;
const UInt32 *pi = p + 2;
UInt32 *e = p;
UInt32 eHi = 0;
n = p + num;
b = p;
// p[0] = fb;
for (;;)
{
UInt32 len = (p[p[--e] >> NUM_BITS] >> NUM_BITS) + 1;
p[e] = (p[e] & MASK) | (len << NUM_BITS);
if (len >= maxLen)
for (len = maxLen - 1; lenCounters[len] == 0; len--);
lenCounters[len]--;
lenCounters[(size_t)len + 1] += 2;
}
{
UInt32 len;
i = 0;
for (len = maxLen; len != 0; len--)
{
UInt32 k;
for (k = lenCounters[len]; k != 0; k--)
lens[p[i++] & MASK] = (Byte)len;
}
}
{
UInt32 nextCodes[kMaxLen + 1];
{
UInt32 code = 0;
UInt32 len;
for (len = 1; len <= kMaxLen; len++)
nextCodes[len] = code = (code + lenCounters[(size_t)len - 1]) << 1;
}
/* if (code + lenCounters[kMaxLen] - 1 != (1 << kMaxLen) - 1) throw 1; */
// (b <= e)
UInt32 sum;
e++;
UPDATE_E(eHi)
// (b < e)
// p range : high bits
// [0, b) : parent : processed nodes that have parent and childs
// [b, e) : FREQ : non-processed nodes that have no parent but have childs
// [e, pi) : FREQ : processed leaves for which parent node was created
// [pi, n) : FREQ : non-processed leaves for which parent node was not created
// first child
// note : (*b < f) is same result as ((*b & FREQ_MASK) < f)
if (fb < f)
{
UInt32 k;
for (k = 0; k < numSymbols; k++)
p[k] = nextCodes[lens[k]]++;
// node freq is smaller
sum = fb & FREQ_MASK;
STORE_PARENT_DIRECT (b, fb, eHi)
b++;
fb = *b;
if (b == e)
{
if (++pi == n)
break;
sum += f;
fb &= MASK;
fb |= sum;
*e = fb;
f = *pi & FREQ_MASK;
continue;
}
}
else if (++pi == n)
{
STORE_PARENT_DIRECT (b, fb, eHi)
b++;
break;
}
else
{
sum = f;
f = *pi & FREQ_MASK;
}
// (b < e)
// second child
if (fb < f)
{
sum += fb;
sum &= FREQ_MASK;
STORE_PARENT_DIRECT (b, fb, eHi)
b++;
*e = (*e & MASK) | sum; // set frequency sum
// (b <= e) is possible here
fb = *b;
}
else if (++pi == n)
break;
else
{
sum += f;
f = *pi & FREQ_MASK;
*e = (*e & MASK) | sum; // set frequency sum
}
}
}
// printf("\nnum-e=%3u, numSymbols=%3u, num=%3u, b=%3u", n - e, numSymbols, n - p, b - p);
{
n -= 2;
*n &= MASK; // root node : we clear high bits (zero bits mean level == 0)
if (n != b)
{
// We go here, if we have some number of non-created nodes up to root.
// We process them in simplified code:
// position of parent for each pair of nodes is known.
// n[-2], n[-1] : current pair of child nodes
// (p1) : parent node for current pair.
UInt32 *p1 = n;
do
{
const unsigned len = LOAD_PARENT(p1) + 1;
p1--;
(lenCounters )[len] -= 2; // we remove 2 leaves from level (len)
(lenCounters + 1)[len] += 2 * 2; // we add 4 leaves at level (len + 1)
n -= 2;
STORE_PARENT (n , n[0], len)
STORE_PARENT (n + 1, n[1], len)
}
while (n != b);
}
}
if (b != p)
{
// we detect level of each node (realtive to root),
// and update lenCounters[].
// We process only intermediate nodes and we don't process leaves.
do
{
// if (ii < b) : parent_bits_of (p[ii]) == index of parent node : ii < (p[ii])
// if (ii >= b) : parent_bits_of (p[ii]) == level of this (ii) node in tree
unsigned len;
b--;
len = (unsigned)LOAD_PARENT(p + LOAD_PARENT(b)) + 1;
STORE_PARENT (b, *b, len)
if (len >= maxLen)
{
// We are not allowed to create node at level (maxLen) and higher,
// because all leaves must be placed to level (maxLen) or lower.
// We find nearest allowed leaf and place current node to level of that leaf:
for (len = maxLen - 1; lenCounters[len] == 0; len--) {}
}
lenCounters[len]--; // we remove 1 leaf from level (len)
(lenCounters + 1)[len] += 2; // we add 2 leaves at level (len + 1)
}
while (b != p);
}
}
{
{
unsigned len = maxLen;
const UInt32 *p2 = p;
do
{
unsigned k = lenCounters[len];
if (k)
do
lens[(unsigned)*p2++ & MASK] = (Byte)len;
while (--k);
}
while (--len);
}
codes[0] = 0; // we don't want garbage values to be written to p[] array.
// codes[1] = 0;
{
UInt32 code = 0;
unsigned len;
for (len = 0; len < kMaxLen; len++)
(codes + 1)[len] = code = (code + lenCounters[len]) << 1;
}
/* if (code + lenCounters[kMaxLen] - 1 != (1 << kMaxLen) - 1) throw 1; */
{
const Byte * const limit = lens + numSymbols;
do
{
unsigned len;
UInt32 c;
len = lens[0]; c = codes[len]; p[0] = c; codes[len] = c + 1;
// len = lens[1]; c = codes[len]; p[1] = c; codes[len] = c + 1;
p += 1;
lens += 1;
}
while (lens != limit);
}
}
}
#undef kMaxLen
#undef NUM_BITS
#undef MASK
#undef FREQ_MASK
#undef NUM_COUNTERS
#undef HUFFMAN_SPEED_OPT
#undef CTR_ITEM_FOR_FREQ
#undef LOAD_PARENT
#undef STORE_PARENT
#undef STORE_PARENT_DIRECT
#undef UPDATE_E
#undef HI_HALF_OFFSET
#undef NUM_UNROLLS
#undef lenCounters
#undef codes

10
C/HuffEnc.h
View file

@ -1,5 +1,5 @@
/* HuffEnc.h -- Huffman encoding
2023-03-05 : Igor Pavlov : Public domain */
Igor Pavlov : Public domain */
#ifndef ZIP7_INC_HUFF_ENC_H
#define ZIP7_INC_HUFF_ENC_H
@ -8,15 +8,15 @@
EXTERN_C_BEGIN
#define Z7_HUFFMAN_LEN_MAX 16
/*
Conditions:
num <= 1024 = 2 ^ NUM_BITS
2 <= num <= 1024 = 2 ^ NUM_BITS
Sum(freqs) < 4M = 2 ^ (32 - NUM_BITS)
maxLen <= 16 = kMaxLen
1 <= maxLen <= 16 = Z7_HUFFMAN_LEN_MAX
Num_Items(p) >= HUFFMAN_TEMP_SIZE(num)
*/
void Huffman_Generate(const UInt32 *freqs, UInt32 *p, Byte *lens, UInt32 num, UInt32 maxLen);
void Huffman_Generate(const UInt32 *freqs, UInt32 *p, Byte *lens, unsigned num, unsigned maxLen);
EXTERN_C_END

26
C/LzFind.c
View file

@ -1,5 +1,5 @@
/* LzFind.c -- Match finder for LZ algorithms
2024-03-01 : Igor Pavlov : Public domain */
: Igor Pavlov : Public domain */
#include "Precomp.h"
@ -404,7 +404,7 @@ int MatchFinder_Create(CMatchFinder *p, UInt32 historySize,
const unsigned nbMax =
(p->numHashBytes == 2 ? 16 :
(p->numHashBytes == 3 ? 24 : 32));
if (numBits > nbMax)
if (numBits >= nbMax)
numBits = nbMax;
if (numBits >= 32)
hs = (UInt32)0 - 1;
@ -416,14 +416,14 @@ int MatchFinder_Create(CMatchFinder *p, UInt32 historySize,
hs |= (256 << kLzHash_CrcShift_2) - 1;
{
const UInt32 hs2 = MatchFinder_GetHashMask2(p, historySize);
if (hs > hs2)
if (hs >= hs2)
hs = hs2;
}
hsCur = hs;
if (p->expectedDataSize < historySize)
{
const UInt32 hs2 = MatchFinder_GetHashMask2(p, (UInt32)p->expectedDataSize);
if (hsCur > hs2)
if (hsCur >= hs2)
hsCur = hs2;
}
}
@ -434,7 +434,7 @@ int MatchFinder_Create(CMatchFinder *p, UInt32 historySize,
if (p->expectedDataSize < historySize)
{
hsCur = MatchFinder_GetHashMask(p, (UInt32)p->expectedDataSize);
if (hsCur > hs) // is it possible?
if (hsCur >= hs) // is it possible?
hsCur = hs;
}
}
@ -598,7 +598,7 @@ void MatchFinder_Init(void *_p)
#ifdef MY_CPU_X86_OR_AMD64
#if defined(__clang__) && (__clang_major__ >= 4) \
|| defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40701)
|| defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40900)
// || defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1900)
#define USE_LZFIND_SATUR_SUB_128
@ -890,7 +890,7 @@ static UInt32 * Hc_GetMatchesSpec(size_t lenLimit, UInt32 curMatch, UInt32 pos,
return d;
{
const Byte *pb = cur - delta;
curMatch = son[_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)];
curMatch = son[_cyclicBufferPos - delta + (_cyclicBufferPos < delta ? _cyclicBufferSize : 0)];
if (pb[maxLen] == cur[maxLen] && *pb == *cur)
{
UInt32 len = 0;
@ -925,7 +925,7 @@ static UInt32 * Hc_GetMatchesSpec(size_t lenLimit, UInt32 curMatch, UInt32 pos,
break;
{
ptrdiff_t diff;
curMatch = son[_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)];
curMatch = son[_cyclicBufferPos - delta + (_cyclicBufferPos < delta ? _cyclicBufferSize : 0)];
diff = (ptrdiff_t)0 - (ptrdiff_t)delta;
if (cur[maxLen] == cur[(ptrdiff_t)maxLen + diff])
{
@ -972,7 +972,7 @@ UInt32 * GetMatchesSpec1(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byt
// if (curMatch >= pos) { *ptr0 = *ptr1 = kEmptyHashValue; return NULL; }
cmCheck = (UInt32)(pos - _cyclicBufferSize);
if ((UInt32)pos <= _cyclicBufferSize)
if ((UInt32)pos < _cyclicBufferSize)
cmCheck = 0;
if (cmCheck < curMatch)
@ -980,7 +980,7 @@ UInt32 * GetMatchesSpec1(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byt
{
const UInt32 delta = pos - curMatch;
{
CLzRef *pair = son + ((size_t)(_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1);
CLzRef *pair = son + ((size_t)(_cyclicBufferPos - delta + (_cyclicBufferPos < delta ? _cyclicBufferSize : 0)) << 1);
const Byte *pb = cur - delta;
unsigned len = (len0 < len1 ? len0 : len1);
const UInt32 pair0 = pair[0];
@ -1039,7 +1039,7 @@ static void SkipMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const
UInt32 cmCheck;
cmCheck = (UInt32)(pos - _cyclicBufferSize);
if ((UInt32)pos <= _cyclicBufferSize)
if ((UInt32)pos < _cyclicBufferSize)
cmCheck = 0;
if (// curMatch >= pos || // failure
@ -1048,7 +1048,7 @@ static void SkipMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const
{
const UInt32 delta = pos - curMatch;
{
CLzRef *pair = son + ((size_t)(_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1);
CLzRef *pair = son + ((size_t)(_cyclicBufferPos - delta + (_cyclicBufferPos < delta ? _cyclicBufferSize : 0)) << 1);
const Byte *pb = cur - delta;
unsigned len = (len0 < len1 ? len0 : len1);
if (pb[len] == cur[len])
@ -1595,7 +1595,7 @@ static void Bt5_MatchFinder_Skip(void *_p, UInt32 num)
UInt32 pos = p->pos; \
UInt32 num2 = num; \
/* (p->pos == p->posLimit) is not allowed here !!! */ \
{ const UInt32 rem = p->posLimit - pos; if (num2 > rem) num2 = rem; } \
{ const UInt32 rem = p->posLimit - pos; if (num2 >= rem) num2 = rem; } \
num -= num2; \
{ const UInt32 cycPos = p->cyclicBufferPos; \
son = p->son + cycPos; \

10
C/LzFindMt.c
View file

@ -1,5 +1,5 @@
/* LzFindMt.c -- multithreaded Match finder for LZ algorithms
2024-01-22 : Igor Pavlov : Public domain */
: Igor Pavlov : Public domain */
#include "Precomp.h"
@ -82,6 +82,8 @@ extern UInt64 g_NumIters_Bytes;
Z7_NO_INLINE
static void MtSync_Construct(CMtSync *p)
{
p->affinityGroup = -1;
p->affinityInGroup = 0;
p->affinity = 0;
p->wasCreated = False;
p->csWasInitialized = False;
@ -259,6 +261,12 @@ static WRes MtSync_Create_WRes(CMtSync *p, THREAD_FUNC_TYPE startAddress, void *
// return ERROR_TOO_MANY_POSTS; // for debug
// return EINVAL; // for debug
#ifdef _WIN32
if (p->affinityGroup >= 0)
wres = Thread_Create_With_Group(&p->thread, startAddress, obj,
(unsigned)(UInt32)p->affinityGroup, (CAffinityMask)p->affinityInGroup);
else
#endif
if (p->affinity != 0)
wres = Thread_Create_With_Affinity(&p->thread, startAddress, obj, (CAffinityMask)p->affinity);
else

6
C/LzFindMt.h
View file

@ -1,5 +1,5 @@
/* LzFindMt.h -- multithreaded Match finder for LZ algorithms
2024-01-22 : Igor Pavlov : Public domain */
: Igor Pavlov : Public domain */
#ifndef ZIP7_INC_LZ_FIND_MT_H
#define ZIP7_INC_LZ_FIND_MT_H
@ -12,8 +12,10 @@ EXTERN_C_BEGIN
typedef struct
{
UInt32 numProcessedBlocks;
CThread thread;
Int32 affinityGroup;
UInt64 affinityInGroup;
UInt64 affinity;
CThread thread;
BoolInt wasCreated;
BoolInt needStart;

4
C/Lzma2Enc.c
View file

@ -1,5 +1,5 @@
/* Lzma2Enc.c -- LZMA2 Encoder
2023-04-13 : Igor Pavlov : Public domain */
: Igor Pavlov : Public domain */
#include "Precomp.h"
@ -235,6 +235,7 @@ void Lzma2EncProps_Init(CLzma2EncProps *p)
p->numBlockThreads_Reduced = -1;
p->numBlockThreads_Max = -1;
p->numTotalThreads = -1;
p->numThreadGroups = 0;
}
void Lzma2EncProps_Normalize(CLzma2EncProps *p)
@ -781,6 +782,7 @@ SRes Lzma2Enc_Encode2(CLzma2EncHandle p,
}
p->mtCoder.numThreadsMax = (unsigned)p->props.numBlockThreads_Max;
p->mtCoder.numThreadGroups = p->props.numThreadGroups;
p->mtCoder.expectedDataSize = p->expectedDataSize;
{

1
C/Lzma2Enc.h
View file

@ -18,6 +18,7 @@ typedef struct
int numBlockThreads_Reduced;
int numBlockThreads_Max;
int numTotalThreads;
unsigned numThreadGroups; // 0 : no groups
} CLzma2EncProps;
void Lzma2EncProps_Init(CLzma2EncProps *p);

6
C/LzmaEnc.c
View file

@ -62,7 +62,9 @@ void LzmaEncProps_Init(CLzmaEncProps *p)
p->lc = p->lp = p->pb = p->algo = p->fb = p->btMode = p->numHashBytes = p->numThreads = -1;
p->numHashOutBits = 0;
p->writeEndMark = 0;
p->affinityGroup = -1;
p->affinity = 0;
p->affinityInGroup = 0;
}
void LzmaEncProps_Normalize(CLzmaEncProps *p)
@ -598,6 +600,10 @@ SRes LzmaEnc_SetProps(CLzmaEncHandle p, const CLzmaEncProps *props2)
p->multiThread = (props.numThreads > 1);
p->matchFinderMt.btSync.affinity =
p->matchFinderMt.hashSync.affinity = props.affinity;
p->matchFinderMt.btSync.affinityGroup =
p->matchFinderMt.hashSync.affinityGroup = props.affinityGroup;
p->matchFinderMt.btSync.affinityInGroup =
p->matchFinderMt.hashSync.affinityInGroup = props.affinityInGroup;
#endif
return SZ_OK;

4
C/LzmaEnc.h
View file

@ -1,5 +1,5 @@
/* LzmaEnc.h -- LZMA Encoder
2023-04-13 : Igor Pavlov : Public domain */
: Igor Pavlov : Public domain */
#ifndef ZIP7_INC_LZMA_ENC_H
#define ZIP7_INC_LZMA_ENC_H
@ -29,11 +29,13 @@ typedef struct
int numThreads; /* 1 or 2, default = 2 */
// int _pad;
Int32 affinityGroup;
UInt64 reduceSize; /* estimated size of data that will be compressed. default = (UInt64)(Int64)-1.
Encoder uses this value to reduce dictionary size */
UInt64 affinity;
UInt64 affinityInGroup;
} CLzmaEncProps;
void LzmaEncProps_Init(CLzmaEncProps *p);

61
C/MtCoder.c
View file

@ -1,5 +1,5 @@
/* MtCoder.c -- Multi-thread Coder
2023-09-07 : Igor Pavlov : Public domain */
: Igor Pavlov : Public domain */
#include "Precomp.h"
@ -39,14 +39,28 @@ void MtProgressThunk_CreateVTable(CMtProgressThunk *p)
static THREAD_FUNC_DECL ThreadFunc(void *pp);
static SRes MtCoderThread_CreateAndStart(CMtCoderThread *t)
static SRes MtCoderThread_CreateAndStart(CMtCoderThread *t
#ifdef _WIN32
, CMtCoder * const mtc
#endif
)
{
WRes wres = AutoResetEvent_OptCreate_And_Reset(&t->startEvent);
// printf("\n====== MtCoderThread_CreateAndStart : \n");
if (wres == 0)
{
t->stop = False;
if (!Thread_WasCreated(&t->thread))
wres = Thread_Create(&t->thread, ThreadFunc, t);
{
#ifdef _WIN32
if (mtc->numThreadGroups)
wres = Thread_Create_With_Group(&t->thread, ThreadFunc, t,
ThreadNextGroup_GetNext(&mtc->nextGroup), // group
0); // affinityMask
else
#endif
wres = Thread_Create(&t->thread, ThreadFunc, t);
}
if (wres == 0)
wres = Event_Set(&t->startEvent);
}
@ -56,6 +70,7 @@ static SRes MtCoderThread_CreateAndStart(CMtCoderThread *t)
}
Z7_FORCE_INLINE
static void MtCoderThread_Destruct(CMtCoderThread *t)
{
if (Thread_WasCreated(&t->thread))
@ -85,7 +100,7 @@ static void MtCoderThread_Destruct(CMtCoderThread *t)
static SRes ThreadFunc2(CMtCoderThread *t)
{
CMtCoder *mtc = t->mtCoder;
CMtCoder * const mtc = t->mtCoder;
for (;;)
{
@ -185,7 +200,11 @@ static SRes ThreadFunc2(CMtCoderThread *t)
if (mtc->numStartedThreads < mtc->numStartedThreadsLimit
&& mtc->expectedDataSize != readProcessed)
{
res = MtCoderThread_CreateAndStart(&mtc->threads[mtc->numStartedThreads]);
res = MtCoderThread_CreateAndStart(&mtc->threads[mtc->numStartedThreads]
#ifdef _WIN32
, mtc
#endif
);
if (res == SZ_OK)
mtc->numStartedThreads++;
else
@ -221,7 +240,7 @@ static SRes ThreadFunc2(CMtCoderThread *t)
}
{
CMtCoderBlock *block = &mtc->blocks[bi];
CMtCoderBlock * const block = &mtc->blocks[bi];
block->res = res;
block->bufIndex = bufIndex;
block->finished = finished;
@ -311,7 +330,7 @@ static SRes ThreadFunc2(CMtCoderThread *t)
static THREAD_FUNC_DECL ThreadFunc(void *pp)
{
CMtCoderThread *t = (CMtCoderThread *)pp;
CMtCoderThread * const t = (CMtCoderThread *)pp;
for (;;)
{
if (Event_Wait(&t->startEvent) != 0)
@ -319,7 +338,7 @@ static THREAD_FUNC_DECL ThreadFunc(void *pp)
if (t->stop)
return 0;
{
SRes res = ThreadFunc2(t);
const SRes res = ThreadFunc2(t);
CMtCoder *mtc = t->mtCoder;
if (res != SZ_OK)
{
@ -328,7 +347,7 @@ static THREAD_FUNC_DECL ThreadFunc(void *pp)
#ifndef MTCODER_USE_WRITE_THREAD
{
unsigned numFinished = (unsigned)InterlockedIncrement(&mtc->numFinishedThreads);
const unsigned numFinished = (unsigned)InterlockedIncrement(&mtc->numFinishedThreads);
if (numFinished == mtc->numStartedThreads)
if (Event_Set(&mtc->finishedEvent) != 0)
return (THREAD_FUNC_RET_TYPE)SZ_ERROR_THREAD;
@ -346,6 +365,7 @@ void MtCoder_Construct(CMtCoder *p)
p->blockSize = 0;
p->numThreadsMax = 0;
p->numThreadGroups = 0;
p->expectedDataSize = (UInt64)(Int64)-1;
p->inStream = NULL;
@ -429,6 +449,8 @@ SRes MtCoder_Code(CMtCoder *p)
unsigned i;
SRes res = SZ_OK;
// printf("\n====== MtCoder_Code : \n");
if (numThreads > MTCODER_THREADS_MAX)
numThreads = MTCODER_THREADS_MAX;
numBlocksMax = MTCODER_GET_NUM_BLOCKS_FROM_THREADS(numThreads);
@ -492,11 +514,22 @@ SRes MtCoder_Code(CMtCoder *p)
p->numStartedThreadsLimit = numThreads;
p->numStartedThreads = 0;
ThreadNextGroup_Init(&p->nextGroup, p->numThreadGroups, 0); // startGroup
// for (i = 0; i < numThreads; i++)
{
// here we create new thread for first block.
// And each new thread will create another new thread after block reading
// until numStartedThreadsLimit is reached.
CMtCoderThread *nextThread = &p->threads[p->numStartedThreads++];
RINOK(MtCoderThread_CreateAndStart(nextThread))
{
const SRes res2 = MtCoderThread_CreateAndStart(nextThread
#ifdef _WIN32
, p
#endif
);
RINOK(res2)
}
}
RINOK_THREAD(Event_Set(&p->readEvent))
@ -513,9 +546,9 @@ SRes MtCoder_Code(CMtCoder *p)
RINOK_THREAD(Event_Wait(&p->writeEvents[bi]))
{
const CMtCoderBlock *block = &p->blocks[bi];
unsigned bufIndex = block->bufIndex;
BoolInt finished = block->finished;
const CMtCoderBlock * const block = &p->blocks[bi];
const unsigned bufIndex = block->bufIndex;
const BoolInt finished = block->finished;
if (res == SZ_OK && block->res != SZ_OK)
res = block->res;
@ -545,7 +578,7 @@ SRes MtCoder_Code(CMtCoder *p)
}
#else
{
WRes wres = Event_Wait(&p->finishedEvent);
const WRes wres = Event_Wait(&p->finishedEvent);
res = MY_SRes_HRESULT_FROM_WRes(wres);
}
#endif

7
C/MtCoder.h
View file

@ -1,5 +1,5 @@
/* MtCoder.h -- Multi-thread Coder
2023-04-13 : Igor Pavlov : Public domain */
: Igor Pavlov : Public domain */
#ifndef ZIP7_INC_MT_CODER_H
#define ZIP7_INC_MT_CODER_H
@ -16,7 +16,7 @@ EXTERN_C_BEGIN
#ifndef Z7_ST
#define MTCODER_GET_NUM_BLOCKS_FROM_THREADS(numThreads) ((numThreads) + (numThreads) / 8 + 1)
#define MTCODER_THREADS_MAX 64
#define MTCODER_THREADS_MAX 256
#define MTCODER_BLOCKS_MAX (MTCODER_GET_NUM_BLOCKS_FROM_THREADS(MTCODER_THREADS_MAX) + 3)
#else
#define MTCODER_THREADS_MAX 1
@ -77,6 +77,7 @@ typedef struct CMtCoder_
size_t blockSize; /* size of input block */
unsigned numThreadsMax;
unsigned numThreadGroups;
UInt64 expectedDataSize;
ISeqInStreamPtr inStream;
@ -125,6 +126,8 @@ typedef struct CMtCoder_
CMtProgress mtProgress;
CMtCoderBlock blocks[MTCODER_BLOCKS_MAX];
CMtCoderThread threads[MTCODER_THREADS_MAX];
CThreadNextGroup nextGroup;
} CMtCoder;

169
C/Sha512.c
View file

@ -439,26 +439,78 @@ void Sha512_Final(CSha512 *p, Byte *digest, unsigned digestSize)
// #define Z7_SHA512_PROBE_DEBUG // for debug
#if defined(_WIN32) && defined(Z7_COMPILER_SHA512_SUPPORTED) \
&& defined(MY_CPU_ARM64) // we can disable this check to debug in x64
#if defined(Z7_SHA512_PROBE_DEBUG) || defined(Z7_COMPILER_SHA512_SUPPORTED)
#if 1 // 0 for debug
#include "7zWindows.h"
// #include <stdio.h>
#if 0 && defined(MY_CPU_X86_OR_AMD64)
#include <intrin.h> // for debug : for __ud2()
#if defined(Z7_SHA512_PROBE_DEBUG) \
|| defined(_WIN32) && defined(MY_CPU_ARM64)
#ifndef Z7_SHA512_USE_PROBE
#define Z7_SHA512_USE_PROBE
#endif
#endif
BoolInt CPU_IsSupported_SHA512(void)
#ifdef Z7_SHA512_USE_PROBE
#ifdef Z7_SHA512_PROBE_DEBUG
#include <stdio.h>
#define PRF(x) x
#else
#define PRF(x)
#endif
#if 0 || !defined(_MSC_VER) // 1 || : for debug LONGJMP mode
// MINGW doesn't support __try. So we use signal() / longjmp().
// Note: signal() / longjmp() probably is not thread-safe.
// So we must call Sha512Prepare() from main thread at program start.
#ifndef Z7_SHA512_USE_LONGJMP
#define Z7_SHA512_USE_LONGJMP
#endif
#endif
#ifdef Z7_SHA512_USE_LONGJMP
#include <signal.h>
#include <setjmp.h>
static jmp_buf g_Sha512_jmp_buf;
// static int g_Sha512_Unsupported;
#if defined(__GNUC__) && (__GNUC__ >= 8) \
|| defined(__clang__) && (__clang_major__ >= 3)
__attribute__((noreturn))
#endif
static void Z7_CDECL Sha512_signal_Handler(int v)
{
PRF(printf("======== Sha512_signal_Handler = %x\n", (unsigned)v);)
// g_Sha512_Unsupported = 1;
longjmp(g_Sha512_jmp_buf, 1);
}
#endif // Z7_SHA512_USE_LONGJMP
#if defined(_WIN32)
#include "7zWindows.h"
#endif
#if defined(MY_CPU_ARM64)
// #define Z7_SHA512_USE_SIMPLIFIED_PROBE // for debug
#endif
#ifdef Z7_SHA512_USE_SIMPLIFIED_PROBE
#include <arm_neon.h>
#if defined(__clang__)
__attribute__((__target__("sha3")))
#elif !defined(_MSC_VER)
__attribute__((__target__("arch=armv8.2-a+sha3")))
#endif
#endif
static BoolInt CPU_IsSupported_SHA512_Probe(void)
{
PRF(printf("\n== CPU_IsSupported_SHA512_Probe\n");)
#if defined(_WIN32) && defined(MY_CPU_ARM64)
// we have no SHA512 flag for IsProcessorFeaturePresent() still.
if (!CPU_IsSupported_CRYPTO())
return False;
#endif
// printf("\nCPU_IsSupported_SHA512\n");
PRF(printf("==== Registry check\n");)
{
// we can't read ID_AA64ISAR0_EL1 register from application.
// but ID_AA64ISAR0_EL1 register is mapped to "CP 4030" registry value.
@ -486,6 +538,7 @@ BoolInt CPU_IsSupported_SHA512(void)
// 2 : SHA256 and SHA512 implemented
}
}
#endif // defined(_WIN32) && defined(MY_CPU_ARM64)
#if 1 // 0 for debug to disable SHA512 PROBE code
@ -509,59 +562,97 @@ Does this PROBE code work in native Windows-arm64 (with/without sha512 hw instru
Are there any ways to fix the problems with arm64-wine and x64-SDE cases?
*/
// printf("\n========== CPU_IsSupported_SHA512 PROBE ========\n");
PRF(printf("==== CPU_IsSupported_SHA512 PROBE\n");)
{
BoolInt isSupported = False;
#ifdef Z7_SHA512_USE_LONGJMP
void (Z7_CDECL *signal_prev)(int);
/*
if (g_Sha512_Unsupported)
{
PRF(printf("==== g_Sha512_Unsupported\n");)
return False;
}
*/
printf("====== signal(SIGILL)\n");
signal_prev = signal(SIGILL, Sha512_signal_Handler);
if (signal_prev == SIG_ERR)
{
PRF(printf("====== signal fail\n");)
return False;
}
// PRF(printf("==== signal_prev = %p\n", (void *)signal_prev);)
// docs: Before the specified function is executed,
// the value of func is set to SIG_DFL.
// So we can exit if (setjmp(g_Sha512_jmp_buf) != 0).
PRF(printf("====== setjmp\n");)
if (!setjmp(g_Sha512_jmp_buf))
#else // Z7_SHA512_USE_LONGJMP
#ifdef _MSC_VER
#ifdef __clang_major__
#pragma GCC diagnostic ignored "-Wlanguage-extension-token"
#endif
__try
#endif
#endif // Z7_SHA512_USE_LONGJMP
{
#if 0 // 1 : for debug (reduced version to detect sha512)
#if defined(Z7_COMPILER_SHA512_SUPPORTED)
#ifdef Z7_SHA512_USE_SIMPLIFIED_PROBE
// simplified sha512 check for arm64:
const uint64x2_t a = vdupq_n_u64(1);
const uint64x2_t b = vsha512hq_u64(a, a, a);
PRF(printf("======== vsha512hq_u64 probe\n");)
if ((UInt32)vgetq_lane_u64(b, 0) == 0x11800002)
return True;
#else
MY_ALIGN(16)
UInt64 temp[SHA512_NUM_DIGEST_WORDS + SHA512_NUM_BLOCK_WORDS];
memset(temp, 0x5a, sizeof(temp));
#if 0 && defined(MY_CPU_X86_OR_AMD64)
__ud2(); // for debug : that exception is not problem for SDE
#endif
#if 1
PRF(printf("======== Sha512_UpdateBlocks_HW\n");)
Sha512_UpdateBlocks_HW(temp,
(const Byte *)(const void *)(temp + SHA512_NUM_DIGEST_WORDS), 1);
// printf("\n==== t = %x\n", (UInt32)temp[0]);
// PRF(printf("======== t = %x\n", (UInt32)temp[0]);)
if ((UInt32)temp[0] == 0xa33cfdf7)
#endif
{
// printf("\n=== PROBE SHA512: SHA512 supported\n");
return True;
PRF(printf("======== PROBE SHA512: SHA512 is supported\n");)
isSupported = True;
}
#else // Z7_COMPILER_SHA512_SUPPORTED
// for debug : we generate bad instrction or raise exception.
// __except() doesn't catch raise() calls.
#ifdef Z7_SHA512_USE_LONGJMP
PRF(printf("====== raise(SIGILL)\n");)
raise(SIGILL);
#else
#if defined(_MSC_VER) && defined(MY_CPU_X86)
__asm ud2
#endif
#endif
#endif // Z7_SHA512_USE_LONGJMP
#endif // Z7_COMPILER_SHA512_SUPPORTED
}
#ifdef Z7_SHA512_USE_LONGJMP
PRF(printf("====== restore signal SIGILL\n");)
signal(SIGILL, signal_prev);
#elif _MSC_VER
__except (EXCEPTION_EXECUTE_HANDLER)
{
// printf("\n==== CPU_IsSupported_SHA512 EXCEPTION_EXECUTE_HANDLER\n");
PRF(printf("==== CPU_IsSupported_SHA512 __except(EXCEPTION_EXECUTE_HANDLER)\n");)
}
#endif
PRF(printf("== return (sha512 supported) = %d\n", isSupported);)
return isSupported;
}
return False;
#else
// without SHA512 PROBE code
return True;
#endif
}
#else
BoolInt CPU_IsSupported_SHA512(void)
{
return False;
}
#endif
#endif // WIN32 arm64
#endif // Z7_SHA512_USE_PROBE
#endif // defined(Z7_SHA512_PROBE_DEBUG) || defined(Z7_COMPILER_SHA512_SUPPORTED)
void Sha512Prepare(void)
@ -570,10 +661,10 @@ void Sha512Prepare(void)
SHA512_FUNC_UPDATE_BLOCKS f, f_hw;
f = Sha512_UpdateBlocks;
f_hw = NULL;
#ifdef MY_CPU_X86_OR_AMD64
if (CPU_IsSupported_SHA512()
&& CPU_IsSupported_AVX2()
)
#ifdef Z7_SHA512_USE_PROBE
if (CPU_IsSupported_SHA512_Probe())
#elif defined(MY_CPU_X86_OR_AMD64)
if (CPU_IsSupported_SHA512() && CPU_IsSupported_AVX2())
#else
if (CPU_IsSupported_SHA512())
#endif
@ -583,6 +674,8 @@ void Sha512Prepare(void)
}
g_SHA512_FUNC_UPDATE_BLOCKS = f;
g_SHA512_FUNC_UPDATE_BLOCKS_HW = f_hw;
#elif defined(Z7_SHA512_PROBE_DEBUG)
CPU_IsSupported_SHA512_Probe(); // for debug
#endif
}

367
C/Sort.c
View file

@ -1,141 +1,268 @@
/* Sort.c -- Sort functions
2014-04-05 : Igor Pavlov : Public domain */
: Igor Pavlov : Public domain */
#include "Precomp.h"
#include "Sort.h"
#include "CpuArch.h"
#define HeapSortDown(p, k, size, temp) \
{ for (;;) { \
size_t s = (k << 1); \
if (s > size) break; \
if (s < size && p[s + 1] > p[s]) s++; \
if (temp >= p[s]) break; \
p[k] = p[s]; k = s; \
} p[k] = temp; }
#if ( (defined(__GNUC__) && (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1))) \
|| (defined(__clang__) && Z7_has_builtin(__builtin_prefetch)) \
)
// the code with prefetch is slow for small arrays on x86.
// So we disable prefetch for x86.
#ifndef MY_CPU_X86
// #pragma message("Z7_PREFETCH : __builtin_prefetch")
#define Z7_PREFETCH(a) __builtin_prefetch((a))
#endif
void HeapSort(UInt32 *p, size_t size)
{
if (size <= 1)
return;
p--;
{
size_t i = size / 2;
do
{
UInt32 temp = p[i];
size_t k = i;
HeapSortDown(p, k, size, temp)
}
while (--i != 0);
}
/*
do
{
size_t k = 1;
UInt32 temp = p[size];
p[size--] = p[1];
HeapSortDown(p, k, size, temp)
}
while (size > 1);
*/
while (size > 3)
{
UInt32 temp = p[size];
size_t k = (p[3] > p[2]) ? 3 : 2;
p[size--] = p[1];
p[1] = p[k];
HeapSortDown(p, k, size, temp)
}
{
UInt32 temp = p[size];
p[size] = p[1];
if (size > 2 && p[2] < temp)
{
p[1] = p[2];
p[2] = temp;
}
else
p[1] = temp;
}
}
#elif defined(_WIN32) // || defined(_MSC_VER) && (_MSC_VER >= 1200)
void HeapSort64(UInt64 *p, size_t size)
{
if (size <= 1)
return;
p--;
{
size_t i = size / 2;
do
{
UInt64 temp = p[i];
size_t k = i;
HeapSortDown(p, k, size, temp)
}
while (--i != 0);
}
/*
do
{
size_t k = 1;
UInt64 temp = p[size];
p[size--] = p[1];
HeapSortDown(p, k, size, temp)
}
while (size > 1);
*/
while (size > 3)
{
UInt64 temp = p[size];
size_t k = (p[3] > p[2]) ? 3 : 2;
p[size--] = p[1];
p[1] = p[k];
HeapSortDown(p, k, size, temp)
}
{
UInt64 temp = p[size];
p[size] = p[1];
if (size > 2 && p[2] < temp)
{
p[1] = p[2];
p[2] = temp;
}
else
p[1] = temp;
}
}
#include "7zWindows.h"
// NOTE: CLANG/GCC/MSVC can define different values for _MM_HINT_T0 / PF_TEMPORAL_LEVEL_1.
// For example, clang-cl can generate "prefetcht2" instruction for
// PreFetchCacheLine(PF_TEMPORAL_LEVEL_1) call.
// But we want to generate "prefetcht0" instruction.
// So for CLANG/GCC we must use __builtin_prefetch() in code branch above
// instead of PreFetchCacheLine() / _mm_prefetch().
// New msvc-x86 compiler generates "prefetcht0" instruction for PreFetchCacheLine() call.
// But old x86 cpus don't support "prefetcht0".
// So we will use PreFetchCacheLine(), only if we are sure that
// generated instruction is supported by all cpus of that isa.
#if defined(MY_CPU_AMD64) \
|| defined(MY_CPU_ARM64) \
|| defined(MY_CPU_IA64)
// we need to use additional braces for (a) in PreFetchCacheLine call, because
// PreFetchCacheLine macro doesn't use braces:
// #define PreFetchCacheLine(l, a) _mm_prefetch((CHAR CONST *) a, l)
// #pragma message("Z7_PREFETCH : PreFetchCacheLine")
#define Z7_PREFETCH(a) PreFetchCacheLine(PF_TEMPORAL_LEVEL_1, (a))
#endif
#endif // _WIN32
#define PREFETCH_NO(p,k,s,size)
#ifndef Z7_PREFETCH
#define SORT_PREFETCH(p,k,s,size)
#else
// #define PREFETCH_LEVEL 2 // use it if cache line is 32-bytes
#define PREFETCH_LEVEL 3 // it is fast for most cases (64-bytes cache line prefetch)
// #define PREFETCH_LEVEL 4 // it can be faster for big array (128-bytes prefetch)
#if PREFETCH_LEVEL == 0
#define SORT_PREFETCH(p,k,s,size)
#else // PREFETCH_LEVEL != 0
/*
#define HeapSortRefDown(p, vals, n, size, temp) \
{ size_t k = n; UInt32 val = vals[temp]; for (;;) { \
size_t s = (k << 1); \
if (s > size) break; \
if (s < size && vals[p[s + 1]] > vals[p[s]]) s++; \
if (val >= vals[p[s]]) break; \
p[k] = p[s]; k = s; \
} p[k] = temp; }
if defined(USE_PREFETCH_FOR_ALIGNED_ARRAY)
we prefetch one value per cache line.
Use it if array is aligned for cache line size (64 bytes)
or if array is small (less than L1 cache size).
void HeapSortRef(UInt32 *p, UInt32 *vals, size_t size)
if !defined(USE_PREFETCH_FOR_ALIGNED_ARRAY)
we perfetch all cache lines that can be required.
it can be faster for big unaligned arrays.
*/
#define USE_PREFETCH_FOR_ALIGNED_ARRAY
// s == k * 2
#if 0 && PREFETCH_LEVEL <= 3 && defined(MY_CPU_X86_OR_AMD64)
// x86 supports (lea r1*8+offset)
#define PREFETCH_OFFSET(k,s) ((s) << PREFETCH_LEVEL)
#else
#define PREFETCH_OFFSET(k,s) ((k) << (PREFETCH_LEVEL + 1))
#endif
#if 1 && PREFETCH_LEVEL <= 3 && defined(USE_PREFETCH_FOR_ALIGNED_ARRAY)
#define PREFETCH_ADD_OFFSET 0
#else
// last offset that can be reqiured in PREFETCH_LEVEL step:
#define PREFETCH_RANGE ((2 << PREFETCH_LEVEL) - 1)
#define PREFETCH_ADD_OFFSET PREFETCH_RANGE / 2
#endif
#if PREFETCH_LEVEL <= 3
#ifdef USE_PREFETCH_FOR_ALIGNED_ARRAY
#define SORT_PREFETCH(p,k,s,size) \
{ const size_t s2 = PREFETCH_OFFSET(k,s) + PREFETCH_ADD_OFFSET; \
if (s2 <= size) { \
Z7_PREFETCH((p + s2)); \
}}
#else /* for unaligned array */
#define SORT_PREFETCH(p,k,s,size) \
{ const size_t s2 = PREFETCH_OFFSET(k,s) + PREFETCH_RANGE; \
if (s2 <= size) { \
Z7_PREFETCH((p + s2 - PREFETCH_RANGE)); \
Z7_PREFETCH((p + s2)); \
}}
#endif
#else // PREFETCH_LEVEL > 3
#ifdef USE_PREFETCH_FOR_ALIGNED_ARRAY
#define SORT_PREFETCH(p,k,s,size) \
{ const size_t s2 = PREFETCH_OFFSET(k,s) + PREFETCH_RANGE - 16 / 2; \
if (s2 <= size) { \
Z7_PREFETCH((p + s2 - 16)); \
Z7_PREFETCH((p + s2)); \
}}
#else /* for unaligned array */
#define SORT_PREFETCH(p,k,s,size) \
{ const size_t s2 = PREFETCH_OFFSET(k,s) + PREFETCH_RANGE; \
if (s2 <= size) { \
Z7_PREFETCH((p + s2 - PREFETCH_RANGE)); \
Z7_PREFETCH((p + s2 - PREFETCH_RANGE / 2)); \
Z7_PREFETCH((p + s2)); \
}}
#endif
#endif // PREFETCH_LEVEL > 3
#endif // PREFETCH_LEVEL != 0
#endif // Z7_PREFETCH
#if defined(MY_CPU_ARM64) \
/* || defined(MY_CPU_AMD64) */ \
/* || defined(MY_CPU_ARM) && !defined(_MSC_VER) */
// we want to use cmov, if cmov is very fast:
// - this cmov version is slower for clang-x64.
// - this cmov version is faster for gcc-arm64 for some fast arm64 cpus.
#define Z7_FAST_CMOV_SUPPORTED
#endif
#ifdef Z7_FAST_CMOV_SUPPORTED
// we want to use cmov here, if cmov is fast: new arm64 cpus.
// we want the compiler to use conditional move for this branch
#define GET_MAX_VAL(n0, n1, max_val_slow) if (n0 < n1) n0 = n1;
#else
// use this branch, if cpu doesn't support fast conditional move.
// it uses slow array access reading:
#define GET_MAX_VAL(n0, n1, max_val_slow) n0 = max_val_slow;
#endif
#define HeapSortDown(p, k, size, temp, macro_prefetch) \
{ \
for (;;) { \
UInt32 n0, n1; \
size_t s = k * 2; \
if (s >= size) { \
if (s == size) { \
n0 = p[s]; \
p[k] = n0; \
if (temp < n0) k = s; \
} \
break; \
} \
n0 = p[k * 2]; \
n1 = p[k * 2 + 1]; \
s += n0 < n1; \
GET_MAX_VAL(n0, n1, p[s]) \
if (temp >= n0) break; \
macro_prefetch(p, k, s, size) \
p[k] = n0; \
k = s; \
} \
p[k] = temp; \
}
/*
stage-1 : O(n) :
we generate intermediate partially sorted binary tree:
p[0] : it's additional item for better alignment of tree structure in memory.
p[1]
p[2] p[3]
p[4] p[5] p[6] p[7]
...
p[x] >= p[x * 2]
p[x] >= p[x * 2 + 1]
stage-2 : O(n)*log2(N):
we move largest item p[0] from head of tree to the end of array
and insert last item to sorted binary tree.
*/
// (p) must be aligned for cache line size (64-bytes) for best performance
void Z7_FASTCALL HeapSort(UInt32 *p, size_t size)
{
if (size <= 1)
if (size < 2)
return;
p--;
if (size == 2)
{
size_t i = size / 2;
const UInt32 a0 = p[0];
const UInt32 a1 = p[1];
const unsigned k = a1 < a0;
p[k] = a0;
p[k ^ 1] = a1;
return;
}
{
// stage-1 : O(n)
// we transform array to partially sorted binary tree.
size_t i = --size / 2;
// (size) now is the index of the last item in tree,
// if (i)
{
do
{
const UInt32 temp = p[i];
size_t k = i;
HeapSortDown(p, k, size, temp, PREFETCH_NO)
}
while (--i);
}
{
const UInt32 temp = p[0];
const UInt32 a1 = p[1];
if (temp < a1)
{
size_t k = 1;
p[0] = a1;
HeapSortDown(p, k, size, temp, PREFETCH_NO)
}
}
}
if (size < 3)
{
// size == 2
const UInt32 a0 = p[0];
p[0] = p[2];
p[2] = a0;
return;
}
if (size != 3)
{
// stage-2 : O(size) * log2(size):
// we move largest item p[0] from head to the end of array,
// and insert last item to sorted binary tree.
do
{
UInt32 temp = p[i];
HeapSortRefDown(p, vals, i, size, temp);
const UInt32 temp = p[size];
size_t k = p[2] < p[3] ? 3 : 2;
p[size--] = p[0];
p[0] = p[1];
p[1] = p[k];
HeapSortDown(p, k, size, temp, SORT_PREFETCH) // PREFETCH_NO
}
while (--i != 0);
while (size != 3);
}
do
{
UInt32 temp = p[size];
p[size--] = p[1];
HeapSortRefDown(p, vals, 1, size, temp);
const UInt32 a2 = p[2];
const UInt32 a3 = p[3];
const size_t k = a2 < a3;
p[2] = p[1];
p[3] = p[0];
p[k] = a3;
p[k ^ 1] = a2;
}
while (size > 1);
}
*/

7
C/Sort.h
View file

@ -1,5 +1,5 @@
/* Sort.h -- Sort functions
2023-03-05 : Igor Pavlov : Public domain */
: Igor Pavlov : Public domain */
#ifndef ZIP7_INC_SORT_H
#define ZIP7_INC_SORT_H
@ -8,10 +8,7 @@
EXTERN_C_BEGIN
void HeapSort(UInt32 *p, size_t size);
void HeapSort64(UInt64 *p, size_t size);
/* void HeapSortRef(UInt32 *p, UInt32 *vals, size_t size); */
void Z7_FASTCALL HeapSort(UInt32 *p, size_t size);
EXTERN_C_END

237
C/Threads.c
View file

@ -1,5 +1,5 @@
/* Threads.c -- multithreading library
2024-03-28 : Igor Pavlov : Public domain */
: Igor Pavlov : Public domain */
#include "Precomp.h"
@ -59,6 +59,100 @@ WRes Thread_Wait_Close(CThread *p)
return (res != 0 ? res : res2);
}
typedef struct MY_PROCESSOR_NUMBER {
WORD Group;
BYTE Number;
BYTE Reserved;
} MY_PROCESSOR_NUMBER, *MY_PPROCESSOR_NUMBER;
typedef struct MY_GROUP_AFFINITY {
#if defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION < 100000)
// KAFFINITY is not defined in old mingw
ULONG_PTR
#else
KAFFINITY
#endif
Mask;
WORD Group;
WORD Reserved[3];
} MY_GROUP_AFFINITY, *MY_PGROUP_AFFINITY;
typedef BOOL (WINAPI *Func_SetThreadGroupAffinity)(
HANDLE hThread,
CONST MY_GROUP_AFFINITY *GroupAffinity,
MY_PGROUP_AFFINITY PreviousGroupAffinity);
typedef BOOL (WINAPI *Func_GetThreadGroupAffinity)(
HANDLE hThread,
MY_PGROUP_AFFINITY GroupAffinity);
typedef BOOL (WINAPI *Func_GetProcessGroupAffinity)(
HANDLE hProcess,
PUSHORT GroupCount,
PUSHORT GroupArray);
Z7_DIAGNOSTIC_IGNORE_CAST_FUNCTION
#if 0
#include <stdio.h>
#define PRF(x) x
/*
--
before call of SetThreadGroupAffinity()
GetProcessGroupAffinity return one group.
after call of SetThreadGroupAffinity():
GetProcessGroupAffinity return more than group,
if SetThreadGroupAffinity() was to another group.
--
GetProcessAffinityMask MS DOCs:
{
If the calling process contains threads in multiple groups,
the function returns zero for both affinity masks.
}
but tests in win10 with 2 groups (less than 64 cores total):
GetProcessAffinityMask() still returns non-zero affinity masks
even after SetThreadGroupAffinity() calls.
*/
static void PrintProcess_Info()
{
{
const
Func_GetProcessGroupAffinity fn_GetProcessGroupAffinity =
(Func_GetProcessGroupAffinity) Z7_CAST_FUNC_C GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")),
"GetProcessGroupAffinity");
if (fn_GetProcessGroupAffinity)
{
unsigned i;
USHORT GroupCounts[64];
USHORT GroupCount = Z7_ARRAY_SIZE(GroupCounts);
BOOL boolRes = fn_GetProcessGroupAffinity(GetCurrentProcess(),
&GroupCount, GroupCounts);
printf("\n====== GetProcessGroupAffinity : "
"boolRes=%u GroupCounts = %u :",
boolRes, (unsigned)GroupCount);
for (i = 0; i < GroupCount; i++)
printf(" %u", GroupCounts[i]);
printf("\n");
}
}
{
DWORD_PTR processAffinityMask, systemAffinityMask;
if (GetProcessAffinityMask(GetCurrentProcess(), &processAffinityMask, &systemAffinityMask))
{
PRF(printf("\n====== GetProcessAffinityMask : "
": processAffinityMask=%x, systemAffinityMask=%x\n",
(UInt32)processAffinityMask, (UInt32)systemAffinityMask);)
}
else
printf("\n==GetProcessAffinityMask FAIL");
}
}
#else
#ifndef USE_THREADS_CreateThread
// #define PRF(x)
#endif
#endif
WRes Thread_Create(CThread *p, THREAD_FUNC_TYPE func, LPVOID param)
{
/* Windows Me/98/95: threadId parameter may not be NULL in _beginthreadex/CreateThread functions */
@ -72,7 +166,43 @@ WRes Thread_Create(CThread *p, THREAD_FUNC_TYPE func, LPVOID param)
unsigned threadId;
*p = (HANDLE)(_beginthreadex(NULL, 0, func, param, 0, &threadId));
#if 0 // 1 : for debug
{
DWORD_PTR prevMask;
DWORD_PTR affinity = 1 << 0;
prevMask = SetThreadAffinityMask(*p, (DWORD_PTR)affinity);
prevMask = prevMask;
}
#endif
#if 0 // 1 : for debug
{
/* win10: new thread will be created in same group that is assigned to parent thread
but affinity mask will contain all allowed threads of that group,
even if affinity mask of parent group is not full
win11: what group it will be created, if we have set
affinity of parent thread with ThreadGroupAffinity?
*/
const
Func_GetThreadGroupAffinity fn =
(Func_GetThreadGroupAffinity) Z7_CAST_FUNC_C GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")),
"GetThreadGroupAffinity");
if (fn)
{
// BOOL wres2;
MY_GROUP_AFFINITY groupAffinity;
memset(&groupAffinity, 0, sizeof(groupAffinity));
/* wres2 = */ fn(*p, &groupAffinity);
PRF(printf("\n==Thread_Create cur = %6u GetThreadGroupAffinity(): "
"wres2_BOOL = %u, group=%u mask=%x\n",
GetCurrentThreadId(),
wres2,
groupAffinity.Group,
(UInt32)groupAffinity.Mask);)
}
}
#endif
#endif
/* maybe we must use errno here, but probably GetLastError() is also OK. */
@ -110,7 +240,84 @@ WRes Thread_Create_With_Affinity(CThread *p, THREAD_FUNC_TYPE func, LPVOID param
*/
}
{
DWORD prevSuspendCount = ResumeThread(h);
const DWORD prevSuspendCount = ResumeThread(h);
/* ResumeThread() returns:
0 : was_not_suspended
1 : was_resumed
-1 : error
*/
if (prevSuspendCount == (DWORD)-1)
wres = GetError();
}
}
/* maybe we must use errno here, but probably GetLastError() is also OK. */
return wres;
#endif
}
WRes Thread_Create_With_Group(CThread *p, THREAD_FUNC_TYPE func, LPVOID param, unsigned group, CAffinityMask affinityMask)
{
#ifdef USE_THREADS_CreateThread
UNUSED_VAR(group)
UNUSED_VAR(affinityMask)
return Thread_Create(p, func, param);
#else
/* Windows Me/98/95: threadId parameter may not be NULL in _beginthreadex/CreateThread functions */
HANDLE h;
WRes wres;
unsigned threadId;
h = (HANDLE)(_beginthreadex(NULL, 0, func, param, CREATE_SUSPENDED, &threadId));
*p = h;
wres = HandleToWRes(h);
if (h)
{
// PrintProcess_Info();
{
const
Func_SetThreadGroupAffinity fn =
(Func_SetThreadGroupAffinity) Z7_CAST_FUNC_C GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")),
"SetThreadGroupAffinity");
if (fn)
{
// WRes wres2;
MY_GROUP_AFFINITY groupAffinity, prev_groupAffinity;
memset(&groupAffinity, 0, sizeof(groupAffinity));
// groupAffinity.Mask must use only bits that supported by current group
// (groupAffinity.Mask = 0) means all allowed bits
groupAffinity.Mask = affinityMask;
groupAffinity.Group = (WORD)group;
// wres2 =
fn(h, &groupAffinity, &prev_groupAffinity);
/*
if (groupAffinity.Group == prev_groupAffinity.Group)
wres2 = wres2;
else
wres2 = wres2;
if (wres2 == 0)
{
wres2 = GetError();
PRF(printf("\n==SetThreadGroupAffinity error: %u\n", wres2);)
}
else
{
PRF(printf("\n==Thread_Create_With_Group::SetThreadGroupAffinity()"
" threadId = %6u"
" group=%u mask=%x\n",
threadId,
prev_groupAffinity.Group,
(UInt32)prev_groupAffinity.Mask);)
}
*/
}
}
{
const DWORD prevSuspendCount = ResumeThread(h);
/* ResumeThread() returns:
0 : was_not_suspended
1 : was_resumed
@ -297,6 +504,13 @@ WRes Thread_Create(CThread *p, THREAD_FUNC_TYPE func, LPVOID param)
return Thread_Create_With_CpuSet(p, func, param, NULL);
}
/*
WRes Thread_Create_With_Group(CThread *p, THREAD_FUNC_TYPE func, LPVOID param, unsigned group, CAffinityMask affinity)
{
UNUSED_VAR(group)
return Thread_Create_With_Affinity(p, func, param, affinity);
}
*/
WRes Thread_Create_With_Affinity(CThread *p, THREAD_FUNC_TYPE func, LPVOID param, CAffinityMask affinity)
{
@ -577,5 +791,22 @@ WRes AutoResetEvent_OptCreate_And_Reset(CAutoResetEvent *p)
return AutoResetEvent_CreateNotSignaled(p);
}
void ThreadNextGroup_Init(CThreadNextGroup *p, UInt32 numGroups, UInt32 startGroup)
{
// printf("\n====== ThreadNextGroup_Init numGroups = %x: startGroup=%x\n", numGroups, startGroup);
if (numGroups == 0)
numGroups = 1;
p->NumGroups = numGroups;
p->NextGroup = startGroup % numGroups;
}
UInt32 ThreadNextGroup_GetNext(CThreadNextGroup *p)
{
const UInt32 next = p->NextGroup;
p->NextGroup = (next + 1) % p->NumGroups;
return next;
}
#undef PRF
#undef Print

12
C/Threads.h
View file

@ -1,5 +1,5 @@
/* Threads.h -- multithreading library
2024-03-28 : Igor Pavlov : Public domain */
: Igor Pavlov : Public domain */
#ifndef ZIP7_INC_THREADS_H
#define ZIP7_INC_THREADS_H
@ -140,12 +140,22 @@ WRes Thread_Create_With_Affinity(CThread *p, THREAD_FUNC_TYPE func, LPVOID param
WRes Thread_Wait_Close(CThread *p);
#ifdef _WIN32
WRes Thread_Create_With_Group(CThread *p, THREAD_FUNC_TYPE func, LPVOID param, unsigned group, CAffinityMask affinityMask);
#define Thread_Create_With_CpuSet(p, func, param, cs) \
Thread_Create_With_Affinity(p, func, param, *cs)
#else
WRes Thread_Create_With_CpuSet(CThread *p, THREAD_FUNC_TYPE func, LPVOID param, const CCpuSet *cpuSet);
#endif
typedef struct
{
unsigned NumGroups;
unsigned NextGroup;
} CThreadNextGroup;
void ThreadNextGroup_Init(CThreadNextGroup *p, unsigned numGroups, unsigned startGroup);
unsigned ThreadNextGroup_GetNext(CThreadNextGroup *p);
#ifdef _WIN32

4
C/Util/Lzma/LzmaUtil.dsp
View file

@ -122,6 +122,10 @@ SOURCE=..\..\Compiler.h
# End Source File
# Begin Source File
SOURCE=..\..\CpuArch.c
# End Source File
# Begin Source File
SOURCE=..\..\CpuArch.h
# End Source File
# Begin Source File

8
C/Util/LzmaLib/LzmaLib.dsp
View file

@ -43,7 +43,7 @@ RSC=rc.exe
# PROP Ignore_Export_Lib 0
# PROP Target_Dir ""
# ADD BASE CPP /nologo /MT /W3 /GX /O2 /D "WIN32" /D "NDEBUG" /D "_WINDOWS" /D "_MBCS" /D "_USRDLL" /D "LZMALIB_EXPORTS" /YX /FD /c
# ADD CPP /nologo /Gr /MT /W3 /O2 /D "NDEBUG" /D "WIN32" /D "_WINDOWS" /D "_MBCS" /D "_USRDLL" /D "LZMALIB_EXPORTS" /FD /c
# ADD CPP /nologo /Gr /MT /W4 /WX /O2 /D "NDEBUG" /D "WIN32" /D "_WINDOWS" /D "_MBCS" /D "_USRDLL" /D "LZMALIB_EXPORTS" /FD /c
# SUBTRACT CPP /YX
# ADD BASE MTL /nologo /D "NDEBUG" /mktyplib203 /win32
# ADD MTL /nologo /D "NDEBUG" /mktyplib203 /win32
@ -71,7 +71,7 @@ LINK32=link.exe
# PROP Ignore_Export_Lib 0
# PROP Target_Dir ""
# ADD BASE CPP /nologo /MTd /W3 /Gm /GX /ZI /Od /D "WIN32" /D "_DEBUG" /D "_WINDOWS" /D "_MBCS" /D "_USRDLL" /D "LZMALIB_EXPORTS" /YX /FD /GZ /c
# ADD CPP /nologo /MTd /W3 /Gm /ZI /Od /D "_DEBUG" /D "WIN32" /D "_WINDOWS" /D "_MBCS" /D "_USRDLL" /D "LZMALIB_EXPORTS" /D "COMPRESS_MF_MT" /FD /GZ /c
# ADD CPP /nologo /MTd /W4 /WX /Gm /ZI /Od /D "_DEBUG" /D "WIN32" /D "_WINDOWS" /D "_MBCS" /D "_USRDLL" /D "LZMALIB_EXPORTS" /D "COMPRESS_MF_MT" /FD /GZ /c
# SUBTRACT CPP /YX
# ADD BASE MTL /nologo /D "_DEBUG" /mktyplib203 /win32
# ADD MTL /nologo /D "_DEBUG" /mktyplib203 /win32
@ -128,6 +128,10 @@ SOURCE=..\..\Compiler.h
# End Source File
# Begin Source File
SOURCE=..\..\CpuArch.c
# End Source File
# Begin Source File
SOURCE=..\..\CpuArch.h
# End Source File
# Begin Source File

98
C/Xxh64.c
View file

@ -1,6 +1,6 @@
/* Xxh64.c -- XXH64 hash calculation
original code: Copyright (c) Yann Collet.
2023-08-18 : modified by Igor Pavlov.
modified by Igor Pavlov.
This source code is licensed under BSD 2-Clause License.
*/
@ -27,6 +27,14 @@ void Xxh64State_Init(CXxh64State *p)
#if !defined(MY_CPU_64BIT) && defined(MY_CPU_X86) && defined(_MSC_VER)
#define Z7_XXH64_USE_ASM
#elif !defined(MY_CPU_LE_UNALIGN_64) // && defined (MY_CPU_LE)
#define Z7_XXH64_USE_ALIGNED
#endif
#ifdef Z7_XXH64_USE_ALIGNED
#define Xxh64State_UpdateBlocks_Unaligned_Select Xxh64State_UpdateBlocks_Unaligned
#else
#define Xxh64State_UpdateBlocks_Unaligned_Select Xxh64State_UpdateBlocks
#endif
#if !defined(MY_CPU_64BIT) && defined(MY_CPU_X86) \
@ -188,32 +196,76 @@ Xxh64State_UpdateBlocks(CXxh64State *p, const void *data, const void *end)
#else
#ifdef Z7_XXH64_USE_ALIGNED
static
#endif
void
Z7_NO_INLINE
Z7_FASTCALL
Xxh64State_UpdateBlocks(CXxh64State *p, const void *_data, const void *end)
Xxh64State_UpdateBlocks_Unaligned_Select(CXxh64State *p, const void *_data, const void *end)
{
const Byte *data = (const Byte *)_data;
UInt64 v[4];
v[0] = p->v[0];
v[1] = p->v[1];
v[2] = p->v[2];
v[3] = p->v[3];
UInt64 v0, v1, v2, v3;
v0 = p->v[0];
v1 = p->v[1];
v2 = p->v[2];
v3 = p->v[3];
do
{
v[0] = Xxh64_Round(v[0], GetUi64(data)); data += 8;
v[1] = Xxh64_Round(v[1], GetUi64(data)); data += 8;
v[2] = Xxh64_Round(v[2], GetUi64(data)); data += 8;
v[3] = Xxh64_Round(v[3], GetUi64(data)); data += 8;
v0 = Xxh64_Round(v0, GetUi64(data)); data += 8;
v1 = Xxh64_Round(v1, GetUi64(data)); data += 8;
v2 = Xxh64_Round(v2, GetUi64(data)); data += 8;
v3 = Xxh64_Round(v3, GetUi64(data)); data += 8;
}
while (data != end);
p->v[0] = v[0];
p->v[1] = v[1];
p->v[2] = v[2];
p->v[3] = v[3];
p->v[0] = v0;
p->v[1] = v1;
p->v[2] = v2;
p->v[3] = v3;
}
#endif
#ifdef Z7_XXH64_USE_ALIGNED
static
void
Z7_NO_INLINE
Z7_FASTCALL
Xxh64State_UpdateBlocks_Aligned(CXxh64State *p, const void *_data, const void *end)
{
const Byte *data = (const Byte *)_data;
UInt64 v0, v1, v2, v3;
v0 = p->v[0];
v1 = p->v[1];
v2 = p->v[2];
v3 = p->v[3];
do
{
v0 = Xxh64_Round(v0, GetUi64a(data)); data += 8;
v1 = Xxh64_Round(v1, GetUi64a(data)); data += 8;
v2 = Xxh64_Round(v2, GetUi64a(data)); data += 8;
v3 = Xxh64_Round(v3, GetUi64a(data)); data += 8;
}
while (data != end);
p->v[0] = v0;
p->v[1] = v1;
p->v[2] = v2;
p->v[3] = v3;
}
void
Z7_NO_INLINE
Z7_FASTCALL
Xxh64State_UpdateBlocks(CXxh64State *p, const void *data, const void *end)
{
if (((unsigned)(ptrdiff_t)data & 7) == 0)
Xxh64State_UpdateBlocks_Aligned(p, data, end);
else
Xxh64State_UpdateBlocks_Unaligned(p, data, end);
}
#endif // Z7_XXH64_USE_ALIGNED
#endif // Z7_XXH64_USE_ASM
UInt64 Xxh64State_Digest(const CXxh64State *p, const void *_data, UInt64 count)
{
@ -306,12 +358,22 @@ void Xxh64_Update(CXxh64 *p, const void *_data, size_t size)
while (--rem);
if (cnt != 32)
return;
Xxh64State_UpdateBlocks(&p->state, p->buf64, &p->buf64[4]);
#ifdef Z7_XXH64_USE_ALIGNED
Xxh64State_UpdateBlocks_Aligned
#else
Xxh64State_UpdateBlocks_Unaligned_Select
#endif
(&p->state, p->buf64, &p->buf64[4]);
}
if (size &= ~(size_t)31)
{
Xxh64State_UpdateBlocks(&p->state, data, data + size);
#ifdef Z7_XXH64_USE_ALIGNED
if (((unsigned)(ptrdiff_t)data & 7) == 0)
Xxh64State_UpdateBlocks_Aligned(&p->state, data, data + size);
else
#endif
Xxh64State_UpdateBlocks_Unaligned_Select(&p->state, data, data + size);
data += size;
}

12
C/Xz.h
View file

@ -1,5 +1,5 @@
/* Xz.h - Xz interface
2024-01-26 : Igor Pavlov : Public domain */
Igor Pavlov : Public domain */
#ifndef ZIP7_INC_XZ_H
#define ZIP7_INC_XZ_H
@ -121,6 +121,7 @@ typedef struct
UInt64 startOffset;
} CXzStream;
#define Xz_CONSTRUCT(p) { (p)->numBlocks = 0; (p)->blocks = NULL; (p)->flags = 0; }
void Xz_Construct(CXzStream *p);
void Xz_Free(CXzStream *p, ISzAllocPtr alloc);
@ -136,8 +137,13 @@ typedef struct
CXzStream *streams;
} CXzs;
#define Xzs_CONSTRUCT(p) { (p)->num = 0; (p)->numAllocated = 0; (p)->streams = NULL; }
void Xzs_Construct(CXzs *p);
void Xzs_Free(CXzs *p, ISzAllocPtr alloc);
/*
Xzs_ReadBackward() must be called for empty CXzs object.
Xzs_ReadBackward() can return non empty object with (p->num != 0) even in case of error.
*/
SRes Xzs_ReadBackward(CXzs *p, ILookInStreamPtr inStream, Int64 *startOffset, ICompressProgressPtr progress, ISzAllocPtr alloc);
UInt64 Xzs_GetNumBlocks(const CXzs *p);
@ -268,8 +274,8 @@ typedef struct
size_t outBufSize;
size_t outDataWritten; // the size of data in (outBuf) that were fully unpacked
Byte shaDigest[SHA256_DIGEST_SIZE];
Byte buf[XZ_BLOCK_HEADER_SIZE_MAX];
UInt32 shaDigest32[SHA256_DIGEST_SIZE / 4];
Byte buf[XZ_BLOCK_HEADER_SIZE_MAX]; // it must be aligned for 4-bytes
} CXzUnpacker;
/* alloc : aligned for cache line allocation is better */

4
C/XzCrc64Opt.c
View file

@ -1,5 +1,5 @@
/* XzCrc64Opt.c -- CRC64 calculation (optimized functions)
2023-12-08 : Igor Pavlov : Public domain */
: Igor Pavlov : Public domain */
#include "Precomp.h"
@ -235,7 +235,7 @@ CRC64_FUNC_PRE_BE(Z7_CRC64_NUM_TABLES_USE)
v = Q32BE(1, w1) ^ Q32BE(0, w0);
v ^= Q32BE(3, d1) ^ Q32BE(2, d0);
#endif
#elif
#else
#error Stop_Compiling_Bad_CRC64_NUM_TABLES
#endif
p += Z7_CRC64_NUM_TABLES_USE;

29
C/XzDec.c
View file

@ -1,5 +1,5 @@
/* XzDec.c -- Xz Decode
2024-03-01 : Igor Pavlov : Public domain */
: Igor Pavlov : Public domain */
#include "Precomp.h"
@ -59,7 +59,7 @@ unsigned Xz_ReadVarInt(const Byte *p, size_t maxSize, UInt64 *value)
for (i = 0; i < limit;)
{
Byte b = p[i];
const unsigned b = p[i];
*value |= (UInt64)(b & 0x7F) << (7 * i++);
if ((b & 0x80) == 0)
return (b == 0 && i != 1) ? 0 : i;
@ -796,11 +796,10 @@ SRes Xz_ParseHeader(CXzStreamFlags *p, const Byte *buf)
static BoolInt Xz_CheckFooter(CXzStreamFlags flags, UInt64 indexSize, const Byte *buf)
{
return indexSize == (((UInt64)GetUi32(buf + 4) + 1) << 2)
&& GetUi32(buf) == CrcCalc(buf + 4, 6)
&& flags == GetBe16(buf + 8)
&& buf[10] == XZ_FOOTER_SIG_0
&& buf[11] == XZ_FOOTER_SIG_1;
return indexSize == (((UInt64)GetUi32a(buf + 4) + 1) << 2)
&& GetUi32a(buf) == CrcCalc(buf + 4, 6)
&& flags == GetBe16a(buf + 8)
&& GetUi16a(buf + 10) == (XZ_FOOTER_SIG_0 | (XZ_FOOTER_SIG_1 << 8));
}
#define READ_VARINT_AND_CHECK(buf, pos, size, res) \
@ -1166,7 +1165,7 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen,
p->indexPreSize = 1 + Xz_WriteVarInt(p->buf + 1, p->numBlocks);
p->indexPos = p->indexPreSize;
p->indexSize += p->indexPreSize;
Sha256_Final(&p->sha, p->shaDigest);
Sha256_Final(&p->sha, (Byte *)(void *)p->shaDigest32);
Sha256_Init(&p->sha);
p->crc = CrcUpdate(CRC_INIT_VAL, p->buf, p->indexPreSize);
p->state = XZ_STATE_STREAM_INDEX;
@ -1241,10 +1240,10 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen,
break;
}
{
Byte digest[XZ_CHECK_SIZE_MAX];
UInt32 digest32[XZ_CHECK_SIZE_MAX / 4];
p->state = XZ_STATE_BLOCK_HEADER;
p->pos = 0;
if (XzCheck_Final(&p->check, digest) && memcmp(digest, p->buf, checkSize) != 0)
if (XzCheck_Final(&p->check, (void *)digest32) && memcmp(digest32, p->buf, checkSize) != 0)
return SZ_ERROR_CRC;
if (p->decodeOnlyOneBlock)
{
@ -1289,12 +1288,12 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen,
}
else
{
Byte digest[SHA256_DIGEST_SIZE];
UInt32 digest32[SHA256_DIGEST_SIZE / 4];
p->state = XZ_STATE_STREAM_INDEX_CRC;
p->indexSize += 4;
p->pos = 0;
Sha256_Final(&p->sha, digest);
if (memcmp(digest, p->shaDigest, SHA256_DIGEST_SIZE) != 0)
Sha256_Final(&p->sha, (void *)digest32);
if (memcmp(digest32, p->shaDigest32, SHA256_DIGEST_SIZE) != 0)
return SZ_ERROR_CRC;
}
}
@ -1313,7 +1312,7 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen,
const Byte *ptr = p->buf;
p->state = XZ_STATE_STREAM_FOOTER;
p->pos = 0;
if (CRC_GET_DIGEST(p->crc) != GetUi32(ptr))
if (CRC_GET_DIGEST(p->crc) != GetUi32a(ptr))
return SZ_ERROR_CRC;
}
break;
@ -1343,7 +1342,7 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen,
{
if (*src != 0)
{
if (((UInt32)p->padSize & 3) != 0)
if ((unsigned)p->padSize & 3)
return SZ_ERROR_NO_ARCHIVE;
p->pos = 0;
p->state = XZ_STATE_STREAM_HEADER;

8
C/XzEnc.c
View file

@ -1,5 +1,5 @@
/* XzEnc.c -- Xz Encode
2024-03-01 : Igor Pavlov : Public domain */
: Igor Pavlov : Public domain */
#include "Precomp.h"
@ -411,6 +411,7 @@ static SRes SeqInFilter_Read(ISeqInStreamPtr pp, void *data, size_t *size)
}
}
Z7_FORCE_INLINE
static void SeqInFilter_Construct(CSeqInFilter *p)
{
p->buf = NULL;
@ -418,6 +419,7 @@ static void SeqInFilter_Construct(CSeqInFilter *p)
p->vt.Read = SeqInFilter_Read;
}
Z7_FORCE_INLINE
static void SeqInFilter_Free(CSeqInFilter *p, ISzAllocPtr alloc)
{
if (p->StateCoder.p)
@ -507,6 +509,7 @@ void XzFilterProps_Init(CXzFilterProps *p)
void XzProps_Init(CXzProps *p)
{
p->checkId = XZ_CHECK_CRC32;
p->numThreadGroups = 0;
p->blockSize = XZ_PROPS_BLOCK_SIZE_AUTO;
p->numBlockThreads_Reduced = -1;
p->numBlockThreads_Max = -1;
@ -689,6 +692,7 @@ typedef struct
} CLzma2WithFilters;
Z7_FORCE_INLINE
static void Lzma2WithFilters_Construct(CLzma2WithFilters *p)
{
p->lzma2 = NULL;
@ -712,6 +716,7 @@ static SRes Lzma2WithFilters_Create(CLzma2WithFilters *p, ISzAllocPtr alloc, ISz
}
Z7_FORCE_INLINE
static void Lzma2WithFilters_Free(CLzma2WithFilters *p, ISzAllocPtr alloc)
{
#ifdef USE_SUBBLOCK
@ -1236,6 +1241,7 @@ SRes XzEnc_Encode(CXzEncHandle p, ISeqOutStreamPtr outStream, ISeqInStreamPtr in
}
p->mtCoder.numThreadsMax = (unsigned)props->numBlockThreads_Max;
p->mtCoder.numThreadGroups = props->numThreadGroups;
p->mtCoder.expectedDataSize = p->expectedDataSize;
RINOK(MtCoder_Code(&p->mtCoder))

3
C/XzEnc.h
View file

@ -1,5 +1,5 @@
/* XzEnc.h -- Xz Encode
2023-04-13 : Igor Pavlov : Public domain */
: Igor Pavlov : Public domain */
#ifndef ZIP7_INC_XZ_ENC_H
#define ZIP7_INC_XZ_ENC_H
@ -31,6 +31,7 @@ typedef struct
CLzma2EncProps lzma2Props;
CXzFilterProps filterProps;
unsigned checkId;
unsigned numThreadGroups; // 0 : no groups
UInt64 blockSize;
int numBlockThreads_Reduced;
int numBlockThreads_Max;

265
C/XzIn.c
View file

@ -1,38 +1,39 @@
/* XzIn.c - Xz input
2023-09-07 : Igor Pavlov : Public domain */
: Igor Pavlov : Public domain */
#include "Precomp.h"
#include <string.h>
#include "7zCrc.h"
#include "CpuArch.h"
#include "Xz.h"
#include "CpuArch.h"
/*
#define XZ_FOOTER_SIG_CHECK(p) (memcmp((p), XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE) == 0)
*/
#define XZ_FOOTER_SIG_CHECK(p) ((p)[0] == XZ_FOOTER_SIG_0 && (p)[1] == XZ_FOOTER_SIG_1)
#define XZ_FOOTER_12B_ALIGNED16_SIG_CHECK(p) \
(GetUi16a((const Byte *)(const void *)(p) + 10) == \
(XZ_FOOTER_SIG_0 | (XZ_FOOTER_SIG_1 << 8)))
SRes Xz_ReadHeader(CXzStreamFlags *p, ISeqInStreamPtr inStream)
{
Byte sig[XZ_STREAM_HEADER_SIZE];
UInt32 data32[XZ_STREAM_HEADER_SIZE / 4];
size_t processedSize = XZ_STREAM_HEADER_SIZE;
RINOK(SeqInStream_ReadMax(inStream, sig, &processedSize))
RINOK(SeqInStream_ReadMax(inStream, data32, &processedSize))
if (processedSize != XZ_STREAM_HEADER_SIZE
|| memcmp(sig, XZ_SIG, XZ_SIG_SIZE) != 0)
|| memcmp(data32, XZ_SIG, XZ_SIG_SIZE) != 0)
return SZ_ERROR_NO_ARCHIVE;
return Xz_ParseHeader(p, sig);
return Xz_ParseHeader(p, (const Byte *)(const void *)data32);
}
#define READ_VARINT_AND_CHECK(buf, pos, size, res) \
{ const unsigned s = Xz_ReadVarInt(buf + pos, size - pos, res); \
#define READ_VARINT_AND_CHECK(buf, size, res) \
{ const unsigned s = Xz_ReadVarInt(buf, size, res); \
if (s == 0) return SZ_ERROR_ARCHIVE; \
pos += s; }
size -= s; \
buf += s; \
}
SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStreamPtr inStream, BoolInt *isIndex, UInt32 *headerSizeRes)
{
MY_ALIGN(4)
Byte header[XZ_BLOCK_HEADER_SIZE_MAX];
unsigned headerSize;
*headerSizeRes = 0;
@ -57,8 +58,12 @@ SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStreamPtr inStream, BoolInt *isIndex,
return XzBlock_Parse(p, header);
}
#define ADD_SIZE_CHECK(size, val) \
{ const UInt64 newSize = size + (val); if (newSize < size) return XZ_SIZE_OVERFLOW; size = newSize; }
{ const UInt64 newSize = size + (val); \
if (newSize < size) return XZ_SIZE_OVERFLOW; \
size = newSize; \
}
UInt64 Xz_GetUnpackSize(const CXzStream *p)
{
@ -82,76 +87,85 @@ UInt64 Xz_GetPackSize(const CXzStream *p)
return size;
}
/*
SRes XzBlock_ReadFooter(CXzBlock *p, CXzStreamFlags f, ISeqInStreamPtr inStream)
{
return SeqInStream_Read(inStream, p->check, XzFlags_GetCheckSize(f));
}
*/
static SRes Xz_ReadIndex2(CXzStream *p, const Byte *buf, size_t size, ISzAllocPtr alloc)
// input;
// CXzStream (p) is empty object.
// size != 0
// (size & 3) == 0
// (buf) is aligned for at least 4 bytes.
// output:
// p->numBlocks is number of allocated items in p->blocks
// p->blocks[*] values must be ignored, if function returns error.
static SRes Xz_ParseIndex(CXzStream *p, const Byte *buf, size_t size, ISzAllocPtr alloc)
{
size_t numBlocks, pos = 1;
UInt32 crc;
size_t numBlocks;
if (size < 5 || buf[0] != 0)
return SZ_ERROR_ARCHIVE;
size -= 4;
crc = CrcCalc(buf, size);
if (crc != GetUi32(buf + size))
return SZ_ERROR_ARCHIVE;
{
const UInt32 crc = CrcCalc(buf, size);
if (crc != GetUi32a(buf + size))
return SZ_ERROR_ARCHIVE;
}
buf++;
size--;
{
UInt64 numBlocks64;
READ_VARINT_AND_CHECK(buf, pos, size, &numBlocks64)
READ_VARINT_AND_CHECK(buf, size, &numBlocks64)
// (numBlocks64) is 63-bit value, so we can calculate (numBlocks64 * 2):
if (numBlocks64 * 2 > size)
return SZ_ERROR_ARCHIVE;
if (numBlocks64 >= ((size_t)1 << (sizeof(size_t) * 8 - 1)) / sizeof(CXzBlockSizes))
return SZ_ERROR_MEM; // SZ_ERROR_ARCHIVE
numBlocks = (size_t)numBlocks64;
if (numBlocks != numBlocks64 || numBlocks * 2 > size)
return SZ_ERROR_ARCHIVE;
}
Xz_Free(p, alloc);
if (numBlocks != 0)
// Xz_Free(p, alloc); // it's optional, because (p) is empty already
if (numBlocks)
{
size_t i;
p->numBlocks = numBlocks;
p->blocks = (CXzBlockSizes *)ISzAlloc_Alloc(alloc, sizeof(CXzBlockSizes) * numBlocks);
if (!p->blocks)
CXzBlockSizes *blocks = (CXzBlockSizes *)ISzAlloc_Alloc(alloc, sizeof(CXzBlockSizes) * numBlocks);
if (!blocks)
return SZ_ERROR_MEM;
for (i = 0; i < numBlocks; i++)
p->blocks = blocks;
p->numBlocks = numBlocks;
// the caller will call Xz_Free() in case of error
do
{
CXzBlockSizes *block = &p->blocks[i];
READ_VARINT_AND_CHECK(buf, pos, size, &block->totalSize)
READ_VARINT_AND_CHECK(buf, pos, size, &block->unpackSize)
if (block->totalSize == 0)
READ_VARINT_AND_CHECK(buf, size, &blocks->totalSize)
READ_VARINT_AND_CHECK(buf, size, &blocks->unpackSize)
if (blocks->totalSize == 0)
return SZ_ERROR_ARCHIVE;
blocks++;
}
while (--numBlocks);
}
while ((pos & 3) != 0)
if (buf[pos++] != 0)
if (size >= 4)
return SZ_ERROR_ARCHIVE;
while (size)
if (buf[--size])
return SZ_ERROR_ARCHIVE;
return (pos == size) ? SZ_OK : SZ_ERROR_ARCHIVE;
return SZ_OK;
}
/*
static SRes Xz_ReadIndex(CXzStream *p, ILookInStreamPtr stream, UInt64 indexSize, ISzAllocPtr alloc)
{
SRes res;
size_t size;
Byte *buf;
if (indexSize > ((UInt32)1 << 31))
return SZ_ERROR_UNSUPPORTED;
if (indexSize >= ((size_t)1 << (sizeof(size_t) * 8 - 1)))
return SZ_ERROR_MEM; // SZ_ERROR_ARCHIVE
size = (size_t)indexSize;
if (size != indexSize)
return SZ_ERROR_UNSUPPORTED;
buf = (Byte *)ISzAlloc_Alloc(alloc, size);
if (!buf)
return SZ_ERROR_MEM;
res = LookInStream_Read2(stream, buf, size, SZ_ERROR_UNSUPPORTED);
if (res == SZ_OK)
res = Xz_ReadIndex2(p, buf, size, alloc);
res = Xz_ParseIndex(p, buf, size, alloc);
ISzAlloc_Free(alloc, buf);
return res;
}
*/
static SRes LookInStream_SeekRead_ForArc(ILookInStreamPtr stream, UInt64 offset, void *buf, size_t size)
{
@ -160,84 +174,102 @@ static SRes LookInStream_SeekRead_ForArc(ILookInStreamPtr stream, UInt64 offset,
/* return LookInStream_Read2(stream, buf, size, SZ_ERROR_NO_ARCHIVE); */
}
/*
in:
(*startOffset) is position in (stream) where xz_stream must be finished.
out:
if returns SZ_OK, then (*startOffset) is position in stream that shows start of xz_stream.
*/
static SRes Xz_ReadBackward(CXzStream *p, ILookInStreamPtr stream, Int64 *startOffset, ISzAllocPtr alloc)
{
UInt64 indexSize;
Byte buf[XZ_STREAM_FOOTER_SIZE];
#define TEMP_BUF_SIZE (1 << 10)
UInt32 buf32[TEMP_BUF_SIZE / 4];
UInt64 pos = (UInt64)*startOffset;
if ((pos & 3) != 0 || pos < XZ_STREAM_FOOTER_SIZE)
if ((pos & 3) || pos < XZ_STREAM_FOOTER_SIZE)
return SZ_ERROR_NO_ARCHIVE;
pos -= XZ_STREAM_FOOTER_SIZE;
RINOK(LookInStream_SeekRead_ForArc(stream, pos, buf, XZ_STREAM_FOOTER_SIZE))
RINOK(LookInStream_SeekRead_ForArc(stream, pos, buf32, XZ_STREAM_FOOTER_SIZE))
if (!XZ_FOOTER_SIG_CHECK(buf + 10))
if (!XZ_FOOTER_12B_ALIGNED16_SIG_CHECK(buf32))
{
UInt32 total = 0;
pos += XZ_STREAM_FOOTER_SIZE;
for (;;)
{
size_t i;
#define TEMP_BUF_SIZE (1 << 10)
Byte temp[TEMP_BUF_SIZE];
i = (pos > TEMP_BUF_SIZE) ? TEMP_BUF_SIZE : (size_t)pos;
// pos != 0
// (pos & 3) == 0
size_t i = pos >= TEMP_BUF_SIZE ? TEMP_BUF_SIZE : (size_t)pos;
pos -= i;
RINOK(LookInStream_SeekRead_ForArc(stream, pos, temp, i))
total += (UInt32)i;
for (; i != 0; i--)
if (temp[i - 1] != 0)
RINOK(LookInStream_SeekRead_ForArc(stream, pos, buf32, i))
i /= 4;
do
if (buf32[i - 1] != 0)
break;
if (i != 0)
{
if ((i & 3) != 0)
return SZ_ERROR_NO_ARCHIVE;
pos += i;
break;
}
if (pos < XZ_STREAM_FOOTER_SIZE || total > (1 << 16))
while (--i);
pos += i * 4;
#define XZ_STREAM_BACKWARD_READING_PAD_MAX (1 << 16)
// here we don't support rare case with big padding for xz stream.
// so we have padding limit for backward reading.
if ((UInt64)*startOffset - pos > XZ_STREAM_BACKWARD_READING_PAD_MAX)
return SZ_ERROR_NO_ARCHIVE;
if (i)
break;
}
// we try to open xz stream after skipping zero padding.
// ((UInt64)*startOffset == pos) is possible here!
if (pos < XZ_STREAM_FOOTER_SIZE)
return SZ_ERROR_NO_ARCHIVE;
pos -= XZ_STREAM_FOOTER_SIZE;
RINOK(LookInStream_SeekRead_ForArc(stream, pos, buf, XZ_STREAM_FOOTER_SIZE))
if (!XZ_FOOTER_SIG_CHECK(buf + 10))
RINOK(LookInStream_SeekRead_ForArc(stream, pos, buf32, XZ_STREAM_FOOTER_SIZE))
if (!XZ_FOOTER_12B_ALIGNED16_SIG_CHECK(buf32))
return SZ_ERROR_NO_ARCHIVE;
}
p->flags = (CXzStreamFlags)GetBe16(buf + 8);
p->flags = (CXzStreamFlags)GetBe16a(buf32 + 2);
if (!XzFlags_IsSupported(p->flags))
return SZ_ERROR_UNSUPPORTED;
{
/* to eliminate GCC 6.3 warning:
dereferencing type-punned pointer will break strict-aliasing rules */
const Byte *buf_ptr = buf;
if (GetUi32(buf_ptr) != CrcCalc(buf + 4, 6))
const UInt32 *buf_ptr = buf32;
if (GetUi32a(buf_ptr) != CrcCalc(buf32 + 1, 6))
return SZ_ERROR_ARCHIVE;
}
indexSize = ((UInt64)GetUi32(buf + 4) + 1) << 2;
if (pos < indexSize)
return SZ_ERROR_ARCHIVE;
pos -= indexSize;
RINOK(LookInStream_SeekTo(stream, pos))
RINOK(Xz_ReadIndex(p, stream, indexSize, alloc))
{
UInt64 totalSize = Xz_GetPackSize(p);
if (totalSize == XZ_SIZE_OVERFLOW
|| totalSize >= ((UInt64)1 << 63)
|| pos < totalSize + XZ_STREAM_HEADER_SIZE)
const UInt64 indexSize = ((UInt64)GetUi32a(buf32 + 1) + 1) << 2;
if (pos < indexSize)
return SZ_ERROR_ARCHIVE;
pos -= (totalSize + XZ_STREAM_HEADER_SIZE);
pos -= indexSize;
// v25.00: relaxed indexSize check. We allow big index table.
// if (indexSize > ((UInt32)1 << 31))
if (indexSize >= ((size_t)1 << (sizeof(size_t) * 8 - 1)))
return SZ_ERROR_MEM; // SZ_ERROR_ARCHIVE
RINOK(LookInStream_SeekTo(stream, pos))
// RINOK(Xz_ReadIndex(p, stream, indexSize, alloc))
{
SRes res;
const size_t size = (size_t)indexSize;
// if (size != indexSize) return SZ_ERROR_UNSUPPORTED;
Byte *buf = (Byte *)ISzAlloc_Alloc(alloc, size);
if (!buf)
return SZ_ERROR_MEM;
res = LookInStream_Read2(stream, buf, size, SZ_ERROR_UNSUPPORTED);
if (res == SZ_OK)
res = Xz_ParseIndex(p, buf, size, alloc);
ISzAlloc_Free(alloc, buf);
RINOK(res)
}
}
{
UInt64 total = Xz_GetPackSize(p);
if (total == XZ_SIZE_OVERFLOW || total >= ((UInt64)1 << 63))
return SZ_ERROR_ARCHIVE;
total += XZ_STREAM_HEADER_SIZE;
if (pos < total)
return SZ_ERROR_ARCHIVE;
pos -= total;
RINOK(LookInStream_SeekTo(stream, pos))
*startOffset = (Int64)pos;
}
@ -246,7 +278,6 @@ static SRes Xz_ReadBackward(CXzStream *p, ILookInStreamPtr stream, Int64 *startO
CSecToRead secToRead;
SecToRead_CreateVTable(&secToRead);
secToRead.realStream = stream;
RINOK(Xz_ReadHeader(&headerFlags, &secToRead.vt))
return (p->flags == headerFlags) ? SZ_OK : SZ_ERROR_ARCHIVE;
}
@ -257,8 +288,7 @@ static SRes Xz_ReadBackward(CXzStream *p, ILookInStreamPtr stream, Int64 *startO
void Xzs_Construct(CXzs *p)
{
p->num = p->numAllocated = 0;
p->streams = 0;
Xzs_CONSTRUCT(p)
}
void Xzs_Free(CXzs *p, ISzAllocPtr alloc)
@ -268,7 +298,7 @@ void Xzs_Free(CXzs *p, ISzAllocPtr alloc)
Xz_Free(&p->streams[i], alloc);
ISzAlloc_Free(alloc, p->streams);
p->num = p->numAllocated = 0;
p->streams = 0;
p->streams = NULL;
}
UInt64 Xzs_GetNumBlocks(const CXzs *p)
@ -307,34 +337,49 @@ UInt64 Xzs_GetPackSize(const CXzs *p)
SRes Xzs_ReadBackward(CXzs *p, ILookInStreamPtr stream, Int64 *startOffset, ICompressProgressPtr progress, ISzAllocPtr alloc)
{
Int64 endOffset = 0;
// it's supposed that CXzs object is empty here.
// if CXzs object is not empty, it will add new streams to that non-empty object.
// Xzs_Free(p, alloc); // it's optional call to empty CXzs object.
RINOK(ILookInStream_Seek(stream, &endOffset, SZ_SEEK_END))
*startOffset = endOffset;
for (;;)
{
CXzStream st;
SRes res;
Xz_Construct(&st);
Xz_CONSTRUCT(&st)
res = Xz_ReadBackward(&st, stream, startOffset, alloc);
// if (res == SZ_OK), then (*startOffset) is start offset of new stream if
// if (res != SZ_OK), then (*startOffset) is unchend or it's expected start offset of stream with error
st.startOffset = (UInt64)*startOffset;
RINOK(res)
// we must store (st) object to array, or we must free (st) local object.
if (res != SZ_OK)
{
Xz_Free(&st, alloc);
return res;
}
if (p->num == p->numAllocated)
{
const size_t newNum = p->num + p->num / 4 + 1;
void *data = ISzAlloc_Alloc(alloc, newNum * sizeof(CXzStream));
if (!data)
{
Xz_Free(&st, alloc);
return SZ_ERROR_MEM;
}
p->numAllocated = newNum;
if (p->num != 0)
memcpy(data, p->streams, p->num * sizeof(CXzStream));
ISzAlloc_Free(alloc, p->streams);
p->streams = (CXzStream *)data;
}
// we use direct copying of raw data from local variable (st) to object in array.
// so we don't need to call Xz_Free(&st, alloc) after copying and after p->num++
p->streams[p->num++] = st;
if (*startOffset == 0)
break;
RINOK(LookInStream_SeekTo(stream, (UInt64)*startOffset))
return SZ_OK;
// seek operation is optional:
// RINOK(LookInStream_SeekTo(stream, (UInt64)*startOffset))
if (progress && ICompressProgress_Progress(progress, (UInt64)(endOffset - *startOffset), (UInt64)(Int64)-1) != SZ_OK)
return SZ_ERROR_PROGRESS;
}
return SZ_OK;
}

10
CPP/7zip/7zip_gcc.mak
View file

@ -142,8 +142,8 @@ MY_MKDIR=mkdir
DEL_OBJ_EXE = -$(RM) $(O)\*.o $(O)\$(PROG).exe $(O)\$(PROG).dll
endif
LIB2_GUI = -lOle32 -lGdi32 -lComctl32 -lComdlg32 -lShell32 $(LIB_HTMLHELP)
LIB2 = -loleaut32 -luuid -ladvapi32 -lUser32 $(LIB2_GUI)
LIB2_GUI = -lole32 -lgdi32 -lcomctl32 -lcomdlg32 -lshell32 $(LIB_HTMLHELP)
LIB2 = -loleaut32 -luuid -ladvapi32 -luser32 $(LIB2_GUI)
# v24.00: -DUNICODE and -D_UNICODE are defined in precompilation header files
# CXXFLAGS_EXTRA = -DUNICODE -D_UNICODE
@ -1245,8 +1245,6 @@ $O/Sha512.o: ../../../../C/Sha512.c
$(CC) $(CFLAGS) $<
$O/Sha512Opt.o: ../../../../C/Sha512Opt.c
$(CC) $(CFLAGS) $<
$O/Sort.o: ../../../../C/Sort.c
$(CC) $(CFLAGS) $<
$O/SwapBytes.o: ../../../../C/SwapBytes.c
$(CC) $(CFLAGS) $<
$O/Xxh64.o: ../../../../C/Xxh64.c
@ -1285,6 +1283,8 @@ $O/Sha1Opt.o: ../../../../Asm/x86/Sha1Opt.asm
$(MY_ASM) $(AFLAGS) $<
$O/Sha256Opt.o: ../../../../Asm/x86/Sha256Opt.asm
$(MY_ASM) $(AFLAGS) $<
$O/Sort.o: ../../../../Asm/x86/Sort.asm
$(MY_ASM) $(AFLAGS) $<
ifndef USE_JWASM
USE_X86_ASM_AES=1
@ -1299,6 +1299,8 @@ $O/Sha1Opt.o: ../../../../C/Sha1Opt.c
$(CC) $(CFLAGS) $<
$O/Sha256Opt.o: ../../../../C/Sha256Opt.c
$(CC) $(CFLAGS) $<
$O/Sort.o: ../../../../C/Sort.c
$(CC) $(CFLAGS) $<
endif

2
CPP/7zip/Archive/7z/7zCompressionMode.h
View file

@ -59,6 +59,7 @@ struct CCompressionMethodMode
bool NumThreads_WasForced;
bool MultiThreadMixer;
UInt32 NumThreads;
UInt32 NumThreadGroups;
#endif
UString Password; // _Wipe
@ -74,6 +75,7 @@ struct CCompressionMethodMode
, NumThreads_WasForced(false)
, MultiThreadMixer(true)
, NumThreads(1)
, NumThreadGroups(0)
#endif
, MemoryUsageLimit((UInt64)1 << 30)
{}

30
CPP/7zip/Archive/7z/7zHandlerOut.cpp
View file

@ -111,8 +111,8 @@ HRESULT CHandler::SetMainMethod(CCompressionMethodMode &methodMode)
}
}
const UInt64 kSolidBytes_Min = (1 << 24);
const UInt64 kSolidBytes_Max = ((UInt64)1 << 32);
const UInt64 kSolidBytes_Min = 1 << 24;
const UInt64 kSolidBytes_Max = (UInt64)1 << 32; // for non-LZMA2 methods
bool needSolid = false;
@ -122,22 +122,24 @@ HRESULT CHandler::SetMainMethod(CCompressionMethodMode &methodMode)
SetGlobalLevelTo(oneMethodInfo);
#ifndef Z7_ST
#ifndef Z7_ST
const bool numThreads_WasSpecifiedInMethod = (oneMethodInfo.Get_NumThreads() >= 0);
if (!numThreads_WasSpecifiedInMethod)
{
// here we set the (NCoderPropID::kNumThreads) property in each method, only if there is no such property already
CMultiMethodProps::SetMethodThreadsTo_IfNotFinded(oneMethodInfo, methodMode.NumThreads);
}
#endif
if (methodMode.NumThreadGroups > 1)
CMultiMethodProps::Set_Method_NumThreadGroups_IfNotFinded(oneMethodInfo, methodMode.NumThreadGroups);
#endif
CMethodFull &methodFull = methodMode.Methods.AddNew();
RINOK(PropsMethod_To_FullMethod(methodFull, oneMethodInfo))
#ifndef Z7_ST
#ifndef Z7_ST
methodFull.Set_NumThreads = true;
methodFull.NumThreads = methodMode.NumThreads;
#endif
#endif
if (methodFull.Id != k_Copy)
needSolid = true;
@ -217,19 +219,18 @@ HRESULT CHandler::SetMainMethod(CCompressionMethodMode &methodMode)
// here we get real chunkSize
cs = oneMethodInfo.Get_Xz_BlockSize();
if (dicSize > cs)
dicSize = cs;
dicSize = cs;
const UInt64 kSolidBytes_Lzma2_Max = ((UInt64)1 << 34);
const UInt64 kSolidBytes_Lzma2_Max = (UInt64)1 << 34;
if (numSolidBytes > kSolidBytes_Lzma2_Max)
numSolidBytes = kSolidBytes_Lzma2_Max;
numSolidBytes = kSolidBytes_Lzma2_Max;
methodFull.Set_NumThreads = false; // we don't use ICompressSetCoderMt::SetNumberOfThreads() for LZMA2 encoder
#ifndef Z7_ST
if (!numThreads_WasSpecifiedInMethod
&& !methodMode.NumThreads_WasForced
&& methodMode.MemoryUsageLimit_WasSet
)
&& methodMode.MemoryUsageLimit_WasSet)
{
const UInt32 lzmaThreads = oneMethodInfo.Get_Lzma_NumThreads();
const UInt32 numBlockThreads_Original = methodMode.NumThreads / lzmaThreads;
@ -273,14 +274,14 @@ HRESULT CHandler::SetMainMethod(CCompressionMethodMode &methodMode)
{
numSolidBytes = (UInt64)dicSize << 7;
if (numSolidBytes > kSolidBytes_Max)
numSolidBytes = kSolidBytes_Max;
numSolidBytes = kSolidBytes_Max;
}
if (_numSolidBytesDefined)
continue;
if (numSolidBytes < kSolidBytes_Min)
numSolidBytes = kSolidBytes_Min;
numSolidBytes = kSolidBytes_Min;
_numSolidBytes = numSolidBytes;
_numSolidBytesDefined = true;
}
@ -704,6 +705,9 @@ Z7_COM7F_IMF(CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt
methodMode.NumThreads = numThreads;
methodMode.NumThreads_WasForced = _numThreads_WasForced;
methodMode.MultiThreadMixer = _useMultiThreadMixer;
#ifdef _WIN32
methodMode.NumThreadGroups = _numThreadGroups; // _change it
#endif
// headerMethod.NumThreads = 1;
headerMethod.MultiThreadMixer = _useMultiThreadMixer;
}

2
CPP/7zip/Archive/7z/7zUpdate.cpp
View file

@ -721,7 +721,7 @@ static int CompareEmptyItems(const unsigned *p1, const unsigned *p2, void *param
return (u1.IsDir && u1.IsAnti) ? -n : n;
}
static const char *g_Exts =
static const char * const g_Exts =
" 7z xz lzma ace arc arj bz tbz bz2 tbz2 cab deb gz tgz ha lha lzh lzo lzx pak rar rpm sit zoo"
" zip jar ear war msi"
" 3gp avi mov mpeg mpg mpe wmv"

14
CPP/7zip/Archive/ArHandler.cpp
View file

@ -325,7 +325,7 @@ HRESULT CHandler::ParseLongNames(IInStream *stream)
{
unsigned i;
for (i = 0; i < _items.Size(); i++)
if (_items[i].Name == "//")
if (_items[i].Name.IsEqualTo("//"))
break;
if (i == _items.Size())
return S_OK;
@ -378,7 +378,7 @@ void CHandler::ChangeDuplicateNames()
if (item.Name[0] == '/')
continue;
CItem &prev = _items[i - 1];
if (item.Name == prev.Name)
if (item.Name.IsEqualTo(prev.Name))
{
if (prev.SameNameIndex < 0)
prev.SameNameIndex = 0;
@ -448,9 +448,9 @@ static UInt32 Get32(const Byte *p, unsigned be) { if (be) return GetBe32(p); ret
HRESULT CHandler::ParseLibSymbols(IInStream *stream, unsigned fileIndex)
{
CItem &item = _items[fileIndex];
if (item.Name != "/" &&
item.Name != "__.SYMDEF" &&
item.Name != "__.SYMDEF SORTED")
if (!item.Name.IsEqualTo("/") &&
!item.Name.IsEqualTo("__.SYMDEF") &&
!item.Name.IsEqualTo("__.SYMDEF SORTED"))
return S_OK;
if (item.Size > ((UInt32)1 << 30) ||
item.Size < 4)
@ -462,7 +462,7 @@ HRESULT CHandler::ParseLibSymbols(IInStream *stream, unsigned fileIndex)
size_t pos = 0;
if (item.Name != "/")
if (!item.Name.IsEqualTo("/"))
{
// "__.SYMDEF" parsing (BSD)
unsigned be;
@ -603,7 +603,7 @@ Z7_COM7F_IMF(CHandler::Open(IInStream *stream,
if (_longNames_FileIndex >= 0)
_items.Delete((unsigned)_longNames_FileIndex);
if (!_items.IsEmpty() && _items[0].Name == "debian-binary")
if (!_items.IsEmpty() && _items[0].Name.IsEqualTo("debian-binary"))
{
_type = kType_Deb;
_items.DeleteFrontal(1);

8
CPP/7zip/Archive/Bz2Handler.cpp
View file

@ -427,9 +427,13 @@ Z7_COM7F_IMF(CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt
}
CMethodProps props2 = _props;
#ifndef Z7_ST
#ifndef Z7_ST
props2.AddProp_NumThreads(_props._numThreads);
#endif
#ifdef _WIN32
if (_props._numThreadGroups > 1)
props2.AddProp32(NCoderPropID::kNumThreadGroups, _props._numThreadGroups);
#endif
#endif
return UpdateArchive(size, outStream, props2, updateCallback);
}

1374
CPP/7zip/Archive/ComHandler.cpp
View file

File diff suppressed because it is too large Load diff

8
CPP/7zip/Archive/Common/HandlerOut.cpp
View file

@ -4,8 +4,6 @@
#include "../../../Common/StringToInt.h"
#include "../Common/ParseProperties.h"
#include "HandlerOut.h"
namespace NArchive {
@ -82,6 +80,7 @@ bool ParseSizeString(const wchar_t *s, const PROPVARIANT &prop, UInt64 percentsB
return true;
}
bool CCommonMethodProps::SetCommonProperty(const UString &name, const PROPVARIANT &value, HRESULT &hres)
{
hres = S_OK;
@ -151,6 +150,11 @@ void CMultiMethodProps::SetMethodThreadsTo_Replace(CMethodProps &oneMethodInfo,
SetMethodProp32_Replace(oneMethodInfo, NCoderPropID::kNumThreads, numThreads);
}
void CMultiMethodProps::Set_Method_NumThreadGroups_IfNotFinded(CMethodProps &oneMethodInfo, UInt32 numThreadGroups)
{
SetMethodProp32(oneMethodInfo, NCoderPropID::kNumThreadGroups, numThreadGroups);
}
#endif // Z7_ST

33
CPP/7zip/Archive/Common/HandlerOut.h
View file

@ -17,11 +17,21 @@ protected:
void InitCommon()
{
// _Write_MTime = true;
#ifndef Z7_ST
_numProcessors = _numThreads = NWindows::NSystem::GetNumberOfProcessors();
_numThreads_WasForced = false;
#endif
{
#ifndef Z7_ST
_numThreads_WasForced = false;
UInt32 numThreads;
#ifdef _WIN32
NWindows::NSystem::CProcessAffinity aff;
numThreads = aff.Load_and_GetNumberOfThreads();
_numThreadGroups = aff.IsGroupMode ? aff.Groups.GroupSizes.Size() : 0;
#else
numThreads = NWindows::NSystem::GetNumberOfProcessors();
#endif // _WIN32
_numProcessors = _numThreads = numThreads;
#endif // Z7_ST
}
size_t memAvail = (size_t)sizeof(size_t) << 28;
_memAvail = memAvail;
_memUsage_Compress = memAvail;
@ -46,11 +56,14 @@ protected:
}
public:
#ifndef Z7_ST
#ifndef Z7_ST
UInt32 _numThreads;
UInt32 _numProcessors;
#ifdef _WIN32
UInt32 _numThreadGroups;
#endif
bool _numThreads_WasForced;
#endif
#endif
bool _memUsage_WasSet;
UInt64 _memUsage_Compress;
@ -80,10 +93,12 @@ public:
void SetGlobalLevelTo(COneMethodInfo &oneMethodInfo) const;
#ifndef Z7_ST
#ifndef Z7_ST
static void SetMethodThreadsTo_IfNotFinded(CMethodProps &props, UInt32 numThreads);
static void SetMethodThreadsTo_Replace(CMethodProps &props, UInt32 numThreads);
#endif
static void Set_Method_NumThreadGroups_IfNotFinded(CMethodProps &props, UInt32 numThreadGroups);
#endif
unsigned GetNumEmptyMethods() const

35
CPP/7zip/Archive/Common/ItemNameUtils.cpp
View file

@ -47,6 +47,25 @@ UString GetOsPath_Remove_TailSlash(const UString &name)
}
#if WCHAR_PATH_SEPARATOR != L'/'
void ReplaceToWinSlashes(UString &name, bool useBackslashReplacement)
{
// name.Replace(kUnixPathSepar, kOsPathSepar);
const unsigned len = name.Len();
for (unsigned i = 0; i < len; i++)
{
wchar_t c = name[i];
if (c == L'/')
c = WCHAR_PATH_SEPARATOR;
else if (useBackslashReplacement && c == L'\\')
c = WCHAR_IN_FILE_NAME_BACKSLASH_REPLACEMENT; // WSL scheme
else
continue;
name.ReplaceOneCharAtPos(i, c);
}
}
#endif
void ReplaceToOsSlashes_Remove_TailSlash(UString &name, bool
#if WCHAR_PATH_SEPARATOR != L'/'
useBackslashReplacement
@ -57,21 +76,7 @@ void ReplaceToOsSlashes_Remove_TailSlash(UString &name, bool
return;
#if WCHAR_PATH_SEPARATOR != L'/'
{
// name.Replace(kUnixPathSepar, kOsPathSepar);
const unsigned len = name.Len();
for (unsigned i = 0; i < len; i++)
{
wchar_t c = name[i];
if (c == L'/')
c = WCHAR_PATH_SEPARATOR;
else if (useBackslashReplacement && c == L'\\')
c = WCHAR_IN_FILE_NAME_BACKSLASH_REPLACEMENT; // WSL scheme
else
continue;
name.ReplaceOneCharAtPos(i, c);
}
}
ReplaceToWinSlashes(name, useBackslashReplacement);
#endif
if (name.Back() == kOsPathSepar)

3
CPP/7zip/Archive/Common/ItemNameUtils.h
View file

@ -13,6 +13,9 @@ void ReplaceSlashes_OsToUnix(UString &name);
UString GetOsPath(const UString &name);
UString GetOsPath_Remove_TailSlash(const UString &name);
#if WCHAR_PATH_SEPARATOR != L'/'
void ReplaceToWinSlashes(UString &name, bool useBackslashReplacement);
#endif
void ReplaceToOsSlashes_Remove_TailSlash(UString &name, bool useBackslashReplacement = false);
void NormalizeSlashes_in_FileName_for_OsPath(wchar_t *s, unsigned len);
void NormalizeSlashes_in_FileName_for_OsPath(UString &name);

42
CPP/7zip/Archive/CpioHandler.cpp
View file

@ -437,7 +437,14 @@ HRESULT CInArchive::GetNextItem()
return S_OK;
/* v23.02: we have disabled rDevMinor check because real file
from Apple contains rDevMinor==255 by some unknown reason */
from Apple contains rDevMinor==255 by some unknown reason
cpio 2.13 and older versions: it copies stat::st_rdev to archive.
and stat::st_rdev can be non-zero for some old linux/filesystems cases for regular files.
cpio 2.14 (2023) copies st_rdev to archive only if (S_ISBLK (st->st_mode) || S_ISCHR (st->st_mode))
v25.00: we have disabled RDevMajor check here to support some rare case created by cpio 2.13- with old linux.
But we still keep full check in IsArc_Cpio() to reduce false cpio detection cases.
*/
#if 0 // 0 : to disable check to support some old linux cpio archives.
if (item.RDevMajor != 0
// || item.RDevMinor != 0
)
@ -446,6 +453,7 @@ HRESULT CInArchive::GetNextItem()
!MY_LIN_S_ISBLK(item.Mode))
return S_OK;
}
#endif
// Size must be 0 for FIFOs and directories
if (item.IsDir() || MY_LIN_S_ISFIFO(item.Mode))
@ -873,17 +881,13 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val
{
case kpidPath:
{
UString res;
bool needConvert = true;
#ifdef _WIN32
// if (
ConvertUTF8ToUnicode(item.Name, res);
// )
needConvert = false;
#endif
if (needConvert)
res = MultiByteToUnicodeString(item.Name, CP_OEMCP);
prop = NItemName::GetOsPath(res);
#ifdef _WIN32
UString u;
ConvertUTF8ToUnicode(item.Name, u);
#else
const UString u = MultiByteToUnicodeString(item.Name, CP_OEMCP);
#endif
prop = NItemName::GetOsPath(u);
break;
}
case kpidIsDir: prop = item.IsDir(); break;
@ -921,16 +925,12 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val
s.SetFrom_CalcLen((const char *)(const void *)(const Byte *)item.Data, (unsigned)item.Data.Size());
if (s.Len() == item.Data.Size())
{
#ifdef _WIN32
UString u;
bool needConvert = true;
#ifdef _WIN32
// if (
ConvertUTF8ToUnicode(item.Name, u);
// )
needConvert = false;
#endif
if (needConvert)
u = MultiByteToUnicodeString(s, CP_OEMCP);
ConvertUTF8ToUnicode(s, u);
#else
const UString u = MultiByteToUnicodeString(s, CP_OEMCP);
#endif
prop = u;
}
}

6
CPP/7zip/Archive/DmgHandler.cpp
View file

@ -444,7 +444,7 @@ const char *Find_Apple_FS_Ext(const AString &name)
{
const CAppleName &a = k_Names[i];
if (a.Ext)
if (name == a.AppleName)
if (name.IsEqualTo(a.AppleName))
return a.Ext;
}
return NULL;
@ -784,7 +784,7 @@ static const CXmlItem *FindKeyPair(const CXmlItem &item, const char *key, const
for (unsigned i = 0; i + 1 < item.SubItems.Size(); i++)
{
const CXmlItem &si = item.SubItems[i];
if (si.IsTagged("key") && si.GetSubString() == key)
if (si.IsTagged("key") && si.GetSubString().IsEqualTo(key))
{
const CXmlItem *si_1 = &item.SubItems[i + 1];
if (si_1->IsTagged(nextTag))
@ -1251,7 +1251,7 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *openArchiveCall
#endif
}
if (xml.Root.Name != "plist")
if (!xml.Root.Name.IsEqualTo("plist"))
return S_FALSE;
const CXmlItem *dictItem = xml.Root.FindSubTag_GetPtr("dict");

817
CPP/7zip/Archive/FatHandler.cpp
View file

File diff suppressed because it is too large Load diff

8
CPP/7zip/Archive/Nsis/NsisIn.cpp
View file

@ -4005,7 +4005,7 @@ HRESULT CInArchive::ReadEntries(const CBlockHeader &bh)
AddParam_Var(params[0]);
AString temp;
ReadString2(temp, params[1]);
if (temp != "$TEMP")
if (!temp.IsEqualTo("$TEMP"))
SpaceQuStr(temp);
break;
}
@ -4410,7 +4410,7 @@ HRESULT CInArchive::ReadEntries(const CBlockHeader &bh)
}
else
{
if (func == "DllUnregisterServer")
if (func.IsEqualTo("DllUnregisterServer"))
{
s += "UnRegDLL";
printFunc = false;
@ -4418,7 +4418,7 @@ HRESULT CInArchive::ReadEntries(const CBlockHeader &bh)
else
{
s += "RegDLL";
if (func == "DllRegisterServer")
if (func.IsEqualTo("DllRegisterServer"))
printFunc = false;
}
AddParam(params[0]);
@ -4886,7 +4886,7 @@ HRESULT CInArchive::ReadEntries(const CBlockHeader &bh)
AddParam_Var(params[1]);
AddParam(params[2]);
AddParam(params[4]);
// if (params[2] == "0") AddCommentAndString("GetWinVer");
// if (params[2].IsEqualTo("0")) AddCommentAndString("GetWinVer");
}
else
s += "GetOsInfo";

2
CPP/7zip/Archive/NtfsHandler.cpp
View file

@ -1907,7 +1907,7 @@ HRESULT CDatabase::Open()
for (i = 0; i < SecurityAttrs.Size(); i++)
{
const CAttr &attr = SecurityAttrs[i];
if (attr.Name == L"$SII")
if (attr.Name.IsEqualTo("$SII"))
{
if (attr.Type == ATTR_TYPE_INDEX_ROOT)
{

4
CPP/7zip/Archive/PeHandler.cpp
View file

@ -2638,7 +2638,7 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *callback)
{
const CSection &sect = _sections[i];
if (IsOpt())
if (_parseResources && sect.Name == ".rsrc")
if (_parseResources && sect.Name.IsEqualTo(".rsrc"))
{
// 20.01: we try to parse only first copy of .rsrc section.
_parseResources = false;
@ -2727,7 +2727,7 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *callback)
for (i = 0; i < _mixItems.Size(); i++)
{
const CMixItem &mixItem = _mixItems[i];
if (mixItem.StringIndex < 0 && mixItem.ResourceIndex < 0 && _sections[mixItem.SectionIndex].Name == "_winzip_")
if (mixItem.StringIndex < 0 && mixItem.ResourceIndex < 0 && _sections[mixItem.SectionIndex].Name.IsEqualTo("_winzip_"))
{
_mainSubfile = (Int32)(int)i;
break;

5
CPP/7zip/Archive/QcowHandler.cpp
View file

@ -482,6 +482,10 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *openCallback)
if (_phySize < headerSize)
_phySize = headerSize;
// we use 32 MiB limit for L1 size, as QEMU with QCOW_MAX_L1_SIZE limit.
if (l1Size > (1u << 22)) // if (l1Size > (1u << (sizeof(size_t) * 8 - 4)))
return S_FALSE;
_isArc = true;
{
const UInt64 backOffset = Get64((const Byte *)(const void *)buf64 + 8);
@ -519,7 +523,6 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *openCallback)
}
CObjArray<UInt64> table64(l1Size);
{
// if ((t1SizeBytes >> 3) != l1Size) return S_FALSE;
RINOK(InStream_SeekSet(stream, l1Offset))
RINOK(ReadStream_FALSE(stream, table64, t1SizeBytes))
}

24
CPP/7zip/Archive/Rar/Rar5Handler.cpp
View file

@ -8,6 +8,7 @@
#include "../../../Common/ComTry.h"
#include "../../../Common/IntToString.h"
#include "../../../Common/MyBuffer2.h"
#include "../../../Common/MyLinux.h"
#include "../../../Common/UTFConvert.h"
#include "../../../Windows/PropVariantUtils.h"
@ -393,6 +394,7 @@ void CItem::Link_to_Prop(unsigned linkType, NWindows::NCOM::CPropVariant &prop)
if (!FindExtra_Link(link))
return;
bool isWindows = (HostOS == kHost_Windows);
if (link.Type != linkType)
{
if (linkType != NLinkType::kUnixSymLink)
@ -400,8 +402,11 @@ void CItem::Link_to_Prop(unsigned linkType, NWindows::NCOM::CPropVariant &prop)
switch ((unsigned)link.Type)
{
case NLinkType::kUnixSymLink:
isWindows = false;
break;
case NLinkType::kWinSymLink:
case NLinkType::kWinJunction:
isWindows = true;
break;
default: return;
}
@ -409,10 +414,15 @@ void CItem::Link_to_Prop(unsigned linkType, NWindows::NCOM::CPropVariant &prop)
AString s;
s.SetFrom_CalcLen((const char *)(Extra + link.NameOffset), link.NameLen);
UString unicode;
ConvertUTF8ToUnicode(s, unicode);
prop = NItemName::GetOsPath(unicode);
// rar5.0 used '\\' separator for windows symlinks and \??\ prefix for abs paths.
// rar5.1+ uses '/' separator for windows symlinks and /??/ prefix for abs paths.
// v25.00: we convert Windows slashes to Linux slashes:
if (isWindows)
unicode.Replace(L'\\', L'/');
prop = unicode;
// prop = NItemName::GetOsPath(unicode);
}
bool CItem::GetAltStreamName(AString &name) const
@ -1175,7 +1185,15 @@ HRESULT CUnpacker::Code(const CItem &item, const CItem &lastItem, UInt64 packSiz
const UInt64 processedSize = outStream->GetPos();
if (res == S_OK && !lastItem.Is_UnknownSize() && processedSize != lastItem.Size)
res = S_FALSE;
{
// rar_v7.13-: linux archive contains symLink with (packSize == 0 && lastItem.Size != 0)
// v25.02: we ignore such record in rar headers:
if (packSize != 0
|| method != 0
|| lastItem.HostOS != kHost_Unix
|| !MY_LIN_S_ISLNK(lastItem.Attrib))
res = S_FALSE;
}
// if (res == S_OK)
{

8
CPP/7zip/Archive/Rar/Rar5Handler.h
View file

@ -286,10 +286,10 @@ struct CItem
bool IsService() const { return RecordType == NHeaderType::kService; }
bool Is_STM() const { return IsService() && Name == "STM"; }
bool Is_CMT() const { return IsService() && Name == "CMT"; }
bool Is_ACL() const { return IsService() && Name == "ACL"; }
// bool Is_QO() const { return IsService() && Name == "QO"; }
bool Is_STM() const { return IsService() && Name.IsEqualTo("STM"); }
bool Is_CMT() const { return IsService() && Name.IsEqualTo("CMT"); }
bool Is_ACL() const { return IsService() && Name.IsEqualTo("ACL"); }
// bool Is_QO() const { return IsService() && Name.IsEqualTo("QO"); }
int FindExtra(unsigned extraID, unsigned &recordDataSize) const;
void PrintInfo(AString &s) const;

26
CPP/7zip/Archive/Rar/RarHandler.cpp
View file

@ -7,6 +7,7 @@
#include "../../../Common/ComTry.h"
#include "../../../Common/IntToString.h"
#include "../../../Common/MyBuffer2.h"
#include "../../../Common/MyLinux.h"
#include "../../../Common/UTFConvert.h"
#include "../../../Windows/PropVariantUtils.h"
@ -70,8 +71,14 @@ bool CItem::IsDir() const
case NHeader::NFile::kHostMSDOS:
case NHeader::NFile::kHostOS2:
case NHeader::NFile::kHostWin32:
if ((Attrib & FILE_ATTRIBUTE_DIRECTORY) != 0)
if (Attrib & FILE_ATTRIBUTE_DIRECTORY)
return true;
break;
case NHeader::NFile::kHostUnix:
case NHeader::NFile::kHostBeOS:
if (MY_LIN_S_ISDIR(Attrib))
return true;
break;
}
return false;
}
@ -86,11 +93,20 @@ UInt32 CItem::GetWinAttrib() const
case NHeader::NFile::kHostWin32:
a = Attrib;
break;
case NHeader::NFile::kHostUnix:
case NHeader::NFile::kHostBeOS:
a = Attrib << 16;
a |= 0x8000; // add posix mode marker
break;
// case NHeader::NFile::kHostMacOS:
// kHostMacOS was used only by some very old rare case rar.
// New rar4-rar7 for macos probably uses kHostUnix.
// So we process kHostMacOS without attribute parsing:
default:
a = 0; // must be converted from unix value;
a = 0;
}
if (IsDir())
a |= NHeader::NFile::kWinFileDirectoryAttributeMask;
a |= FILE_ATTRIBUTE_DIRECTORY;
return a;
}
@ -435,13 +451,13 @@ bool CInArchive::ReadHeaderReal(const Byte *p, unsigned size, CItem &item)
size -= sizeof(item.Salt);
p += sizeof(item.Salt);
}
if (item.Name == "ACL" && size == 0)
if (item.Name.IsEqualTo("ACL") && size == 0)
{
item.IsAltStream = true;
item.Name.Empty();
item.UnicodeName.SetFromAscii(".ACL");
}
else if (item.Name == "STM" && size != 0 && (size & 1) == 0)
else if (item.Name.IsEqualTo("STM") && size != 0 && (size & 1) == 0)
{
item.IsAltStream = true;
item.Name.Empty();

6
CPP/7zip/Archive/RpmHandler.cpp
View file

@ -330,11 +330,11 @@ void CHandler::AddSubFileExtension(AString &res) const
if (!_compressor.IsEmpty())
{
s = _compressor;
if (_compressor == "bzip2")
if (_compressor.IsEqualTo("bzip2"))
s = "bz2";
else if (_compressor == "gzip")
else if (_compressor.IsEqualTo("gzip"))
s = "gz";
else if (_compressor == "zstd")
else if (_compressor.IsEqualTo("zstd"))
s = "zst";
}
else

35
CPP/7zip/Archive/Tar/TarHandler.cpp
View file

@ -65,7 +65,7 @@ static const Byte kArcProps[] =
kpidComment
};
static const char *k_Characts_Prefix = "PREFIX";
static const char * const k_Characts_Prefix = "PREFIX";
IMP_IInArchive_Props
IMP_IInArchive_ArcProps
@ -684,10 +684,14 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val
s.Add_OptSpaced("SCHILY.fflags=");
s += item->SCHILY_fflags;
}
if (item->Is_Sparse())
s.Add_OptSpaced("SPARSE");
if (item->IsThereWarning())
s.Add_OptSpaced("WARNING");
if (item->HeaderError)
s.Add_OptSpaced("ERROR");
if (item->Method_Error)
s.Add_OptSpaced("METHOD_ERROR");
if (item->Pax_Error)
s.Add_OptSpaced("PAX_error");
if (!item->PaxExtra.RawLines.IsEmpty())
@ -812,11 +816,16 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems,
inStream2 = inStream;
else
{
GetStream(index, &inStream2);
if (!inStream2)
return E_FAIL;
const HRESULT hres = GetStream(index, &inStream2);
if (hres == E_NOTIMPL)
opRes = NExtract::NOperationResult::kHeadersError; // kUnsupportedMethod
else if (!inStream2)
{
opRes = NExtract::NOperationResult::kDataError;
// return E_FAIL;
}
}
if (opRes == NExtract::NOperationResult::kOK)
{
if (item->Is_SymLink())
{
@ -855,9 +864,9 @@ Z7_CLASS_IMP_IInStream(
bool _needStartSeek;
public:
unsigned ItemIndex;
CHandler *Handler;
CMyComPtr<IUnknown> HandlerRef;
unsigned ItemIndex;
CRecordVector<UInt64> PhyOffsets;
void Init()
@ -879,7 +888,7 @@ Z7_COM7F_IMF(CSparseStream::Read(void *data, UInt32 size, UInt32 *processedSize)
if (_virtPos >= item.Size)
return S_OK;
{
UInt64 rem = item.Size - _virtPos;
const UInt64 rem = item.Size - _virtPos;
if (size > rem)
size = (UInt32)rem;
}
@ -903,17 +912,17 @@ Z7_COM7F_IMF(CSparseStream::Read(void *data, UInt32 size, UInt32 *processedSize)
}
const CSparseBlock &sb = item.SparseBlocks[left];
UInt64 relat = _virtPos - sb.Offset;
const UInt64 relat = _virtPos - sb.Offset;
if (_virtPos >= sb.Offset && relat < sb.Size)
{
UInt64 rem = sb.Size - relat;
const UInt64 rem = sb.Size - relat;
if (size > rem)
size = (UInt32)rem;
UInt64 phyPos = PhyOffsets[left] + relat;
const UInt64 phyPos = PhyOffsets[left] + relat;
if (_needStartSeek || _phyPos != phyPos)
{
RINOK(InStream_SeekSet(Handler->_stream, (item.Get_DataPos() + phyPos)))
RINOK(InStream_SeekSet(Handler->_stream, item.Get_DataPos() + phyPos))
_needStartSeek = false;
_phyPos = phyPos;
}
@ -927,7 +936,7 @@ Z7_COM7F_IMF(CSparseStream::Read(void *data, UInt32 size, UInt32 *processedSize)
next = sb.Offset;
else if (left + 1 < item.SparseBlocks.Size())
next = item.SparseBlocks[left + 1].Offset;
UInt64 rem = next - _virtPos;
const UInt64 rem = next - _virtPos;
if (size > rem)
size = (UInt32)rem;
memset(data, 0, size);
@ -965,6 +974,8 @@ Z7_COM7F_IMF(CHandler::GetStream(UInt32 index, ISequentialInStream **stream))
if (item.Is_Sparse())
{
if (item.Method_Error)
return E_NOTIMPL; // S_FALSE
CSparseStream *streamSpec = new CSparseStream;
CMyComPtr<IInStream> streamTemp = streamSpec;
streamSpec->Init();

155
CPP/7zip/Archive/Tar/TarIn.cpp
View file

@ -181,6 +181,7 @@ HRESULT CArchive::GetNextItemReal(CItemEx &item)
{
char buf[NFileHeader::kRecordSize];
item.Method_Error = false;
error = k_ErrorType_OK;
filled = false;
@ -218,10 +219,7 @@ HRESULT CArchive::GetNextItemReal(CItemEx &item)
break;
item.HeaderSize += NFileHeader::kRecordSize;
thereAreEmptyRecords = true;
if (OpenCallback)
{
RINOK(Progress(item, 0))
}
RINOK(Progress(item, 0))
}
if (thereAreEmptyRecords)
{
@ -335,37 +333,60 @@ HRESULT CArchive::GetNextItemReal(CItemEx &item)
if (item.LinkFlag == NFileHeader::NLinkFlag::kSparse)
{
Byte isExtended = (Byte)buf[482];
if (isExtended != 0 && isExtended != 1)
return S_OK;
// OLD GNU format: parse sparse file information:
// PackSize = cumulative size of all non-empty blocks of the file.
// We read actual file size from 'realsize' member of oldgnu_header:
RIF(ParseSize(buf + 483, item.Size, item.Size_IsBin))
UInt64 min = 0;
for (unsigned i = 0; i < 4; i++)
{
p = buf + 386 + 24 * i;
if (GetBe32(p) == 0)
{
if (isExtended != 0)
return S_OK;
break;
}
CSparseBlock sb;
RIF(ParseSize(p, sb.Offset))
RIF(ParseSize(p + 12, sb.Size))
item.SparseBlocks.Add(sb);
if (sb.Offset < min || sb.Offset > item.Size)
return S_OK;
if ((sb.Offset & 0x1FF) != 0 || (sb.Size & 0x1FF) != 0)
return S_OK;
min = sb.Offset + sb.Size;
if (min < sb.Offset)
return S_OK;
}
if (min > item.Size)
if (item.Size < item.PackSize) // additional check
return S_OK;
while (isExtended != 0)
p = buf + 386;
UInt64 end = 0, packSum = 0;
unsigned numRecords = 4;
unsigned isExtended = (Byte)p[4 * 24]; // (Byte)p[numRecords * 24];
// the list of blocks contains non-empty blocks. All another data is empty.
for (;;)
{
// const unsigned isExtended = (Byte)p[numRecords * 24];
if (isExtended > 1)
return S_OK;
do
{
if (GetBe32(p) == 0)
{
if (isExtended)
return S_OK;
break;
}
CSparseBlock sb;
RIF(ParseSize(p, sb.Offset))
RIF(ParseSize(p + 12, sb.Size))
p += 24;
/* for all non-last blocks we expect :
((sb.Size & 0x1ff) == 0) && ((sb.Offset & 0x1ff) == 0)
for last block : (sb.Size == 0) is possible.
*/
if (sb.Offset < end
|| item.Size < sb.Offset
|| item.Size - sb.Offset < sb.Size)
return S_OK;
// optional check:
if (sb.Size && ((end & 0x1ff) || (sb.Offset & 0x1ff)))
{
item.Method_Error = true; // relaxed check
// return S_OK;
}
end = sb.Offset + sb.Size;
packSum += sb.Size;
item.SparseBlocks.Add(sb);
}
while (--numRecords);
if (!isExtended)
break;
size_t processedSize = NFileHeader::kRecordSize;
RINOK(ReadStream(SeqStream, buf, &processedSize))
if (processedSize != NFileHeader::kRecordSize)
@ -373,46 +394,22 @@ HRESULT CArchive::GetNextItemReal(CItemEx &item)
error = k_ErrorType_UnexpectedEnd;
return S_OK;
}
item.HeaderSize += NFileHeader::kRecordSize;
if (OpenCallback)
{
RINOK(Progress(item, 0))
}
isExtended = (Byte)buf[21 * 24];
if (isExtended != 0 && isExtended != 1)
return S_OK;
for (unsigned i = 0; i < 21; i++)
{
p = buf + 24 * i;
if (GetBe32(p) == 0)
{
if (isExtended != 0)
return S_OK;
break;
}
CSparseBlock sb;
RIF(ParseSize(p, sb.Offset))
RIF(ParseSize(p + 12, sb.Size))
item.SparseBlocks.Add(sb);
if (sb.Offset < min || sb.Offset > item.Size)
return S_OK;
if ((sb.Offset & 0x1FF) != 0 || (sb.Size & 0x1FF) != 0)
return S_OK;
min = sb.Offset + sb.Size;
if (min < sb.Offset)
return S_OK;
}
RINOK(Progress(item, 0))
p = buf;
numRecords = 21;
isExtended = (Byte)p[21 * 24]; // (Byte)p[numRecords * 24];
}
// optional checks for strict size consistency:
if (end != item.Size || packSum != item.PackSize)
{
item.Method_Error = true; // relaxed check
// return S_OK;
}
if (min > item.Size)
return S_OK;
}
if (item.PackSize >= (UInt64)1 << 63)
if (item.PackSize >= (UInt64)1 << 63) // optional check. It was checked in ParseSize() already
return S_OK;
filled = true;
error = k_ErrorType_OK;
return S_OK;
@ -421,6 +418,8 @@ HRESULT CArchive::GetNextItemReal(CItemEx &item)
HRESULT CArchive::Progress(const CItemEx &item, UInt64 posOffset)
{
if (!OpenCallback)
return S_OK;
const UInt64 pos = item.Get_DataPos() + posOffset;
if (NumFiles - NumFiles_Prev < (1 << 16)
// && NumRecords - NumRecords_Prev < (1 << 16)
@ -500,10 +499,7 @@ HRESULT CArchive::ReadDataToBuffer(const CItemEx &item,
do
{
if (OpenCallback)
{
RINOK(Progress(item, pos))
}
RINOK(Progress(item, pos))
unsigned size = kBufSize;
if (size > packSize)
@ -813,6 +809,7 @@ HRESULT CArchive::ReadItem2(CItemEx &item)
item.LongLink_WasUsed_2 = false;
item.HeaderError = false;
item.Method_Error = false;
item.IsSignedChecksum = false;
item.Prefix_WasUsed = false;
@ -838,13 +835,8 @@ HRESULT CArchive::ReadItem2(CItemEx &item)
for (;;)
{
if (OpenCallback)
{
RINOK(Progress(item, 0))
}
RINOK(Progress(item, 0))
RINOK(GetNextItemReal(item))
// NumRecords++;
if (!filled)
@ -1064,9 +1056,14 @@ HRESULT CArchive::ReadItem2(CItemEx &item)
// GNU TAR ignores (item.Size) in that case
if (item.Size != 0 && item.Size != piSize)
item.Pax_Error = true;
item.Size = piSize;
item.PackSize = piSize;
item.pax_size_WasUsed = true;
if (piSize >= ((UInt64)1 << 63))
item.Pax_Error = true;
else
{
item.Size = piSize;
item.PackSize = piSize;
item.pax_size_WasUsed = true;
}
}
item.PaxTimes = paxInfo;

1
CPP/7zip/Archive/Tar/TarItem.h
View file

@ -322,6 +322,7 @@ struct CPaxExtra
struct CItemEx: public CItem
{
bool HeaderError;
bool Method_Error;
bool IsSignedChecksum;
bool Prefix_WasUsed;

18
CPP/7zip/Archive/Udf/UdfIn.cpp
View file

@ -500,10 +500,15 @@ size_t CFileId::Parse(const Byte *p, size_t size)
processed += impLen;
Id.Parse(p + processed, idLen);
processed += idLen;
// const size_t processed2 = processed;
for (;(processed & 3) != 0; processed++)
if (p[processed] != 0)
return 0;
if ((size_t)tag.CrcLen + 16 != processed) return 0;
// some program can create non-standard UDF file where CrcLen doesn't include Padding data
if ((size_t)tag.CrcLen + 16 != processed
// && (size_t)tag.CrcLen + 16 != processed2 // we can enable this check to support non-standard UDF
)
return 0;
return (processed <= size) ? processed : 0;
}
@ -577,15 +582,20 @@ HRESULT CInArchive::ReadItem(unsigned volIndex, int fsIndex, const CLongAllocDes
item.IcbTag.Parse(p + 16);
// maybe another FileType values are possible in rare cases.
// Shoud we ignore FileType here?
if (fsIndex < 0)
{
// if (item.IcbTag.FileType == ICB_FILE_TYPE_DIR) return S_FALSE;
if (item.IcbTag.FileType != ICB_FILE_TYPE_METADATA &&
item.IcbTag.FileType != ICB_FILE_TYPE_METADATA_MIRROR)
item.IcbTag.FileType != ICB_FILE_TYPE_METADATA_MIRROR &&
item.IcbTag.FileType != ICB_FILE_TYPE_METADATA_BITMAP)
return S_FALSE;
}
else if (
item.IcbTag.FileType != ICB_FILE_TYPE_DIR &&
item.IcbTag.FileType != ICB_FILE_TYPE_FILE)
item.IcbTag.FileType != ICB_FILE_TYPE_FILE &&
item.IcbTag.FileType != ICB_FILE_TYPE_REAL_TIME_FILE) // M2TS files in /BDMV/STREAM/ in Blu-ray movie
return S_FALSE;
item.Parse(p);
@ -1210,7 +1220,7 @@ HRESULT CInArchive::Open2()
if (tag.Id != DESC_TYPE_FileSet)
return S_FALSE;
PRF(printf("\n FileSet", volIndex));
PRF(printf("\n FileSet"));
CFileSet fs;
fs.RecordingTime.Parse(p + 16);
// fs.InterchangeLevel = Get16(p + 18);

7
CPP/7zip/Archive/Udf/UdfIn.h
View file

@ -250,9 +250,10 @@ enum EIcbFileType
{
ICB_FILE_TYPE_DIR = 4,
ICB_FILE_TYPE_FILE = 5,
ICB_FILE_TYPE_METADATA = 250, // 2.2.13.1 Metadata File
ICB_FILE_TYPE_METADATA_MIRROR = 251
ICB_FILE_TYPE_REAL_TIME_FILE = 249, // 2.3.5.2.1
ICB_FILE_TYPE_METADATA = 250, // 2.2.13.1
ICB_FILE_TYPE_METADATA_MIRROR = 251, // 2.2.13.1
ICB_FILE_TYPE_METADATA_BITMAP = 252 // 2.2.13.2
};
enum EIcbDescriptorType

9
CPP/7zip/Archive/VmdkHandler.cpp
View file

@ -202,9 +202,12 @@ struct CExtentInfo
// PartitionUUID
// DeviceIdentifier
bool IsType_ZERO() const { return Type == "ZERO"; }
// bool IsType_FLAT() const { return Type == "FLAT"; }
bool IsType_Flat() const { return Type == "FLAT" || Type == "VMFS" || Type == "VMFSRAW"; }
bool IsType_ZERO() const { return Type.IsEqualTo("ZERO"); }
// bool IsType_FLAT() const { return Type.IsEqualTo("FLAT"); }
bool IsType_Flat() const
{ return Type.IsEqualTo("FLAT")
|| Type.IsEqualTo("VMFS")
|| Type.IsEqualTo("VMFSRAW"); }
bool Parse(const char *s);
};

2
CPP/7zip/Archive/Wim/WimIn.cpp
View file

@ -1814,7 +1814,7 @@ bool CWimXml::Parse()
if (!Xml.Parse(utf))
return false;
if (Xml.Root.Name != "WIM")
if (!Xml.Root.Name.IsEqualTo("WIM"))
return false;
FOR_VECTOR (i, Xml.Root.SubItems)

23
CPP/7zip/Archive/XarHandler.cpp
View file

@ -266,7 +266,7 @@ struct CFile
bool IsCopyMethod() const
{
return Method.IsEmpty() || Method == "octet-stream";
return Method.IsEmpty() || Method.IsEqualTo("octet-stream");
}
void UpdateTotalPackSize(UInt64 &totalSize) const
@ -416,7 +416,7 @@ static bool AddItem(const CXmlItem &item, CObjectVector<CFile> &files, int paren
return true;
if (level >= 1024)
return false;
if (item.Name == "file")
if (item.Name.IsEqualTo("file"))
{
CFile file(parent);
parent = (int)files.Size();
@ -435,19 +435,19 @@ static bool AddItem(const CXmlItem &item, CObjectVector<CFile> &files, int paren
{
file.Type = typeItem->GetSubString();
// file.LinkFrom = typeItem->GetPropVal("link");
if (file.Type == "directory")
if (file.Type.IsEqualTo("directory"))
file.IsDir = true;
else
{
// file.IsDir = false;
/*
else if (file.Type == "file")
else if (file.Type.IsEqualTo("file"))
{}
else if (file.Type == "hardlink")
else if (file.Type.IsEqualTo("hardlink"))
{}
else
*/
if (file.Type == "symlink")
if (file.Type.IsEqualTo("symlink"))
file.Is_SymLink = true;
// file.IsDir = false;
}
@ -489,7 +489,7 @@ static bool AddItem(const CXmlItem &item, CObjectVector<CFile> &files, int paren
if (s.IsPrefixedBy(xx))
{
s.DeleteFrontal(xx.Len());
if (s == "gzip")
if (s.IsEqualTo("gzip"))
s = METHOD_NAME_ZLIB;
}
}
@ -692,12 +692,13 @@ HRESULT CHandler::Open2(IInStream *stream)
file.UpdateTotalPackSize(totalPackSize);
if (file.Parent == -1)
{
if (file.Name == "Payload" || file.Name == "Content")
if (file.Name.IsEqualTo("Payload") ||
file.Name.IsEqualTo("Content"))
{
_mainSubfile = (Int32)(int)i;
numMainFiles++;
}
else if (file.Name == "PackageInfo")
else if (file.Name.IsEqualTo("PackageInfo"))
_is_pkg = true;
}
}
@ -1210,9 +1211,9 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems,
else
opRes = NExtract::NOperationResult::kUnsupportedMethod;
}
else if (item.Method == METHOD_NAME_ZLIB)
else if (item.Method.IsEqualTo(METHOD_NAME_ZLIB))
coder = zlibCoder;
else if (item.Method == "bzip2")
else if (item.Method.IsEqualTo("bzip2"))
coder = bzip2Coder;
else
opRes = NExtract::NOperationResult::kUnsupportedMethod;

14
CPP/7zip/Archive/XzHandler.cpp
View file

@ -446,7 +446,7 @@ void COpenCallbackWrap::Init(IArchiveOpenCallback *callback)
struct CXzsCPP
{
CXzs p;
CXzsCPP() { Xzs_Construct(&p); }
CXzsCPP() { Xzs_CONSTRUCT(&p) }
~CXzsCPP() { Xzs_Free(&p, &g_Alloc); }
};
@ -536,6 +536,9 @@ HRESULT CHandler::Open2(IInStream *inStream, /* UInt32 flags, */ IArchiveOpenCal
if (res2 == SZ_ERROR_ARCHIVE)
return S_FALSE;
// what codes are possible here ?
// ?? res2 == SZ_ERROR_MEM : is possible here
// ?? res2 == SZ_ERROR_UNSUPPORTED : is possible here
}
else if (!isIndex)
{
@ -1159,6 +1162,13 @@ Z7_COM7F_IMF(CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt
*/
#ifndef Z7_ST
#ifdef _WIN32
// we don't use chunk multithreading inside lzma2 stream.
// so we don't set xzProps.lzma2Props.numThreadGroups.
if (_numThreadGroups > 1)
xzProps.numThreadGroups = _numThreadGroups;
#endif
UInt32 numThreads = _numThreads;
@ -1183,6 +1193,8 @@ Z7_COM7F_IMF(CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt
CMultiMethodProps::SetMethodThreadsTo_IfNotFinded(oneMethodInfo, numThreads);
}
// printf("\n====== GetProcessGroupAffinity : \n");
UInt64 cs = _numSolidBytes;
if (cs != XZ_PROPS_BLOCK_SIZE_AUTO)
oneMethodInfo.AddProp_BlockSize2(cs);

90
CPP/7zip/Archive/Zip/ZipIn.cpp
View file

@ -1718,61 +1718,49 @@ HRESULT CInArchive::TryEcd64(UInt64 offset, CCdInfo &cdInfo)
HRESULT CInArchive::FindCd(bool checkOffsetMode)
{
CCdInfo &cdInfo = Vols.ecd;
UInt64 endPos;
// There are no useful data in cache in most cases here.
// So here we don't use cache data from previous operations .
// So here we don't use cache data from previous operations.
InitBuf();
UInt64 endPos;
RINOK(InStream_GetSize_SeekToEnd(Stream, endPos))
_streamPos = endPos;
// const UInt32 kBufSizeMax2 = ((UInt32)1 << 16) + kEcdSize + kEcd64Locator_Size + kEcd64_FullSize;
const size_t kBufSizeMax = ((size_t)1 << 17); // must be larger than kBufSizeMax2
const size_t kBufSizeMax = (size_t)1 << 17; // must be larger than
// (1 << 16) + kEcdSize + kEcd64Locator_Size + kEcd64_FullSize
const size_t bufSize = (endPos < kBufSizeMax) ? (size_t)endPos : kBufSizeMax;
if (bufSize < kEcdSize)
return S_FALSE;
// CByteArr byteBuffer(bufSize);
RINOK(AllocateBuffer(kBufSizeMax))
{
RINOK(Seek_SavePos(endPos - bufSize))
size_t processed = bufSize;
const HRESULT res = ReadStream(Stream, Buffer, &processed);
_streamPos += processed;
_bufCached = processed;
_bufPos = 0;
_cnt += processed;
if (res != S_OK)
return res;
if (processed != bufSize)
return S_FALSE;
}
RINOK(Seek_SavePos(endPos - bufSize))
size_t processed = bufSize;
HRESULT res = ReadStream(Stream, Buffer, &processed);
_streamPos += processed;
_bufCached = processed;
_bufPos = 0;
_cnt += processed;
if (res != S_OK)
return res;
if (processed != bufSize)
return S_FALSE;
CCdInfo &cdInfo = Vols.ecd;
for (size_t i = bufSize - kEcdSize + 1;;)
{
if (i == 0)
return S_FALSE;
const Byte *buf = Buffer;
for (;;)
{
i--;
if (buf[i] == 0x50)
break;
if (i == 0)
return S_FALSE;
}
if (Get32(buf + i) != NSignature::kEcd)
continue;
const Byte *p = buf + i;
do
if (p == buf)
return S_FALSE;
while (*(--p) != 0x50);
cdInfo.ParseEcd32(buf + i);
i = (size_t)(p - buf);
if (Get32(p) != NSignature::kEcd)
continue;
cdInfo.ParseEcd32(p);
}
if (i >= kEcd64Locator_Size)
{
@ -1793,29 +1781,24 @@ HRESULT CInArchive::FindCd(bool checkOffsetMode)
// Most of the zip64 use fixed size Zip64 ECD
// we try relative backward reading.
UInt64 absEcd64 = endPos - bufSize + i - (kEcd64Locator_Size + kEcd64_FullSize);
const UInt64 absEcd64 = endPos - bufSize + i - (kEcd64Locator_Size + kEcd64_FullSize);
if (locatorIndex >= kEcd64_FullSize)
if (checkOffsetMode || absEcd64 == locator.Ecd64Offset)
{
const Byte *ecd64 = buf + locatorIndex - kEcd64_FullSize;
if (Get32(ecd64) == NSignature::kEcd64)
if (Get32(ecd64) == NSignature::kEcd64 &&
Get64(ecd64 + 4) == kEcd64_MainSize)
{
UInt64 mainEcd64Size = Get64(ecd64 + 4);
if (mainEcd64Size == kEcd64_MainSize)
{
cdInfo.ParseEcd64e(ecd64 + 12);
ArcInfo.Base = (Int64)(absEcd64 - locator.Ecd64Offset);
// ArcInfo.BaseVolIndex = cdInfo.ThisDisk;
return S_OK;
}
cdInfo.ParseEcd64e(ecd64 + 12);
ArcInfo.Base = (Int64)(absEcd64 - locator.Ecd64Offset);
// ArcInfo.BaseVolIndex = cdInfo.ThisDisk;
return S_OK;
}
}
// some zip64 use variable size Zip64 ECD.
// we try to use absolute offset from locator.
if (absEcd64 != locator.Ecd64Offset)
{
if (TryEcd64(locator.Ecd64Offset, cdInfo) == S_OK)
@ -1881,6 +1864,9 @@ HRESULT CInArchive::TryReadCd(CObjectVector<CItemEx> &items, const CCdInfo &cdIn
items.Clear();
IsCdUnsorted = false;
if ((Int64)cdOffset < 0)
return S_FALSE;
// _startLocalFromCd_Disk = (UInt32)(Int32)-1;
// _startLocalFromCd_Offset = (UInt64)(Int64)-1;

44
CPP/7zip/Archive/Zip/ZipOut.cpp
View file

@ -49,42 +49,54 @@ void COutArchive::SeekToCurPos()
// #define DOES_NEED_ZIP64(v) (v >= 0)
Z7_NO_INLINE
void COutArchive::WriteBytes(const void *data, size_t size)
{
m_OutBuffer.WriteBytes(data, size);
m_CurPos += size;
}
Z7_NO_INLINE
void COutArchive::Write8(Byte b)
{
m_OutBuffer.WriteByte(b);
m_CurPos++;
}
Z7_NO_INLINE
void COutArchive::Write16(UInt16 val)
{
Write8((Byte)val);
Write8((Byte)(val >> 8));
}
Z7_NO_INLINE
void COutArchive::Write32(UInt32 val)
{
for (int i = 0; i < 4; i++)
{
Write8((Byte)val);
// Write8((Byte)val);
m_OutBuffer.WriteByte((Byte)val);
val >>= 8;
}
m_CurPos += 4;
}
#define WRITE_CONST_PAIR_16_16(a, b) { Write32((a) | ((UInt32)(b) << 16)); }
Z7_NO_INLINE
void COutArchive::Write64(UInt64 val)
{
for (int i = 0; i < 8; i++)
{
Write8((Byte)val);
// Write8((Byte)val);
m_OutBuffer.WriteByte((Byte)val);
val >>= 8;
}
m_CurPos += 8;
}
Z7_NO_INLINE
void COutArchive::WriteExtra(const CExtraBlock &extra)
{
FOR_VECTOR (i, extra.SubBlocks)
@ -134,11 +146,9 @@ void COutArchive::WriteTimeExtra(const CItemOut &item, bool writeNtfs)
if (writeNtfs)
{
// windows explorer ignores that extra
Write16(NFileHeader::NExtraID::kNTFS);
Write16(k_Ntfs_ExtraSize);
WRITE_CONST_PAIR_16_16(NFileHeader::NExtraID::kNTFS, k_Ntfs_ExtraSize)
Write32(0); // reserved
Write16(NFileHeader::NNtfsExtra::kTagTime);
Write16(8 * 3);
WRITE_CONST_PAIR_16_16(NFileHeader::NNtfsExtra::kTagTime, 8 * 3)
WriteNtfsTime(item.Ntfs_MTime);
WriteNtfsTime(item.Ntfs_ATime);
WriteNtfsTime(item.Ntfs_CTime);
@ -148,8 +158,7 @@ void COutArchive::WriteTimeExtra(const CItemOut &item, bool writeNtfs)
{
// windows explorer ignores that extra
// by specification : should we write to local header also?
Write16(NFileHeader::NExtraID::kUnixTime);
Write16(k_UnixTime_ExtraSize);
WRITE_CONST_PAIR_16_16(NFileHeader::NExtraID::kUnixTime, k_UnixTime_ExtraSize)
const Byte flags = (Byte)((unsigned)1 << NFileHeader::NUnixTime::kMTime);
Write8(flags);
UInt32 unixTime;
@ -217,8 +226,7 @@ void COutArchive::WriteLocalHeader(CItemOut &item, bool needCheck)
if (isZip64)
{
Write16(NFileHeader::NExtraID::kZip64);
Write16(8 + 8);
WRITE_CONST_PAIR_16_16(NFileHeader::NExtraID::kZip64, 8 + 8)
Write64(size);
Write64(packSize);
}
@ -357,8 +365,9 @@ HRESULT COutArchive::WriteCentralDir(const CObjectVector<CItemOut> &items, const
const UInt64 cdSize = cd64EndOffset - cdOffset;
const bool cdOffset64 = DOES_NEED_ZIP64(cdOffset);
const bool cdSize64 = DOES_NEED_ZIP64(cdSize);
const bool items64 = items.Size() >= 0xFFFF;
const bool isZip64 = (cdOffset64 || cdSize64 || items64);
const bool need_Items_64 = items.Size() >= 0xFFFF;
const unsigned items16 = (UInt16)(need_Items_64 ? 0xFFFF: items.Size());
const bool isZip64 = (cdOffset64 || cdSize64 || need_Items_64);
// isZip64 = true; // to test Zip64
@ -371,8 +380,8 @@ HRESULT COutArchive::WriteCentralDir(const CObjectVector<CItemOut> &items, const
// const UInt32 extraSize = 1 << 26;
// Write64(kEcd64_MainSize + extraSize);
Write16(45); // made by version
Write16(45); // extract version
WRITE_CONST_PAIR_16_16(45, // made by version
45) // extract version
Write32(0); // ThisDiskNumber
Write32(0); // StartCentralDirectoryDiskNumber
Write64((UInt64)items.Size());
@ -389,10 +398,9 @@ HRESULT COutArchive::WriteCentralDir(const CObjectVector<CItemOut> &items, const
}
Write32(NSignature::kEcd);
Write16(0); // ThisDiskNumber
Write16(0); // StartCentralDirectoryDiskNumber
Write16((UInt16)(items64 ? 0xFFFF: items.Size()));
Write16((UInt16)(items64 ? 0xFFFF: items.Size()));
WRITE_CONST_PAIR_16_16(0, 0) // ThisDiskNumber, StartCentralDirectoryDiskNumber
Write16((UInt16)items16);
Write16((UInt16)items16);
WRITE_32_VAL_SPEC(cdSize, cdSize64)
WRITE_32_VAL_SPEC(cdOffset, cdOffset64)

47
CPP/7zip/Archive/Zip/ZipUpdate.cpp
View file

@ -250,13 +250,26 @@ struct CThreadInfo
HRESULT CreateEvents()
{
WRes wres = CompressEvent.CreateIfNotCreated_Reset();
const WRes wres = CompressEvent.CreateIfNotCreated_Reset();
return HRESULT_FROM_WIN32(wres);
}
HRESULT CreateThread()
// (group < 0) means no_group.
HRESULT CreateThread_with_group(
#ifdef _WIN32
int group
#endif
)
{
WRes wres = Thread.Create(CoderThread, this);
// tested in win10: If thread is created by another thread,
// child thread probably uses same group as parent thread.
// So we don't need to send (group) to encoder in created thread.
const WRes wres =
#ifdef _WIN32
group >= 0 ?
Thread.Create_With_Group(CoderThread, this, (unsigned)group) :
#endif
Thread.Create(CoderThread, this);
return HRESULT_FROM_WIN32(wres);
}
@ -450,8 +463,12 @@ static HRESULT UpdateItemOldData(
if (ui.NewProps)
{
if (item.HasDescriptor())
return E_NOTIMPL;
{
// we know compressed / uncompressed sizes and crc.
// so we remove descriptor here
item.Flags = (UInt16)(item.Flags & ~NFileHeader::NFlags::kDescriptorUsedMask);
// return E_NOTIMPL;
}
// we keep ExternalAttrib and some another properties from old archive
// item.ExternalAttrib = ui.Attrib;
// if we don't change Comment, we keep Comment from OldProperties
@ -1000,6 +1017,9 @@ static HRESULT Update2(
#ifndef Z7_ST
UInt32 numThreads = options._numThreads;
#ifdef _WIN32
const UInt32 numThreadGroups = options._numThreadGroups;
#endif
UInt32 numZipThreads_limit = numThreads;
if (numZipThreads_limit > numFilesToCompress)
@ -1014,12 +1034,10 @@ static HRESULT Update2(
}
{
// we reduce number of threads for 32-bit to reduce memory usege to 256 MB
const UInt32 kNumMaxThreads =
#ifdef _WIN32
64; // _WIN32 supports only 64 threads in one group. So no need for more threads here
#else
128;
#endif
// _WIN32 (64-bit) supports only 64 threads in one group.
8 << (sizeof(size_t) / 2); // 32 threads for 32-bit : 128 threads for 64-bit
if (numThreads > kNumMaxThreads)
numThreads = kNumMaxThreads;
}
@ -1264,7 +1282,14 @@ static HRESULT Update2(
threadInfo.Progress = threadInfo.ProgressSpec;
threadInfo.ProgressSpec->Init(&mtCompressProgressMixer, i);
threadInfo.MtSem = &mtSem;
RINOK(threadInfo.CreateThread())
const HRESULT hres =
threadInfo.CreateThread_with_group(
#ifdef _WIN32
(numThreadGroups > 1 && numThreads > 1) ?
(int)(i % numThreadGroups) : -1
#endif
);
RINOK(hres)
}
}

20
CPP/7zip/Bundles/Alone/Alone.dsp
View file

@ -1148,26 +1148,6 @@ SOURCE=..\..\Compress\PpmdZip.cpp
SOURCE=..\..\Compress\PpmdZip.h
# End Source File
# End Group
# Begin Group "RangeCoder"
# PROP Default_Filter ""
# Begin Source File
SOURCE=..\..\Compress\RangeCoder.h
# End Source File
# Begin Source File
SOURCE=..\..\Compress\RangeCoderBit.h
# End Source File
# Begin Source File
SOURCE=..\..\Compress\RangeCoderBitTree.h
# End Source File
# Begin Source File
SOURCE=..\..\Compress\RangeCoderOpt.h
# End Source File
# End Group
# Begin Group "Shrink"
# PROP Default_Filter ""

3
CPP/7zip/Bundles/Alone/makefile
View file

@ -5,6 +5,7 @@ CFLAGS = $(CFLAGS) -DZ7_ZIP_LZFSE_DISABLE
# CONSOLE_VARIANT_FLAGS=-DZ7_PROG_VARIANT_A
# ZIP_FLAGS=-DZ7_ZIP_LZFSE_DISABLE
# USE_C_SORT=1
# USE_C_AES = 1
# USE_C_SHA = 1
# USE_C_LZFINDOPT = 1
@ -221,7 +222,6 @@ C_OBJS = \
$O\Ppmd8.obj \
$O\Ppmd8Dec.obj \
$O\Ppmd8Enc.obj \
$O\Sort.obj \
$O\SwapBytes.obj \
$O\Threads.obj \
$O\Xxh64.obj \
@ -240,5 +240,6 @@ C_OBJS = \
!include "../../LzmaDec.mak"
!include "../../Sha1.mak"
!include "../../Sha256.mak"
!include "../../Sort.mak"
!include "../../7zip.mak"

2
CPP/7zip/Bundles/Alone7z/makefile
View file

@ -148,7 +148,6 @@ C_OBJS = \
$O\LzmaEnc.obj \
$O\MtCoder.obj \
$O\MtDec.obj \
$O\Sort.obj \
$O\SwapBytes.obj \
$O\Threads.obj \
$O\Xz.obj \
@ -164,5 +163,6 @@ C_OBJS = \
!include "../../LzFindOpt.mak"
!include "../../LzmaDec.mak"
!include "../../Sha256.mak"
!include "../../Sort.mak"
!include "../../7zip.mak"

2
CPP/7zip/Bundles/Format7z/makefile
View file

@ -135,7 +135,6 @@ C_OBJS = \
$O\Ppmd7.obj \
$O\Ppmd7Dec.obj \
$O\Ppmd7Enc.obj \
$O\Sort.obj \
$O\SwapBytes.obj \
$O\Threads.obj \
@ -144,5 +143,6 @@ C_OBJS = \
!include "../../LzFindOpt.mak"
!include "../../LzmaDec.mak"
!include "../../Sha256.mak"
!include "../../Sort.mak"
!include "../../7zip.mak"

2
CPP/7zip/Bundles/Format7zF/Arc.mak
View file

@ -291,7 +291,6 @@ C_OBJS = \
$O\Sha3.obj \
$O\Sha512.obj \
$O\Sha512Opt.obj \
$O\Sort.obj \
$O\SwapBytes.obj \
$O\Threads.obj \
$O\Xxh64.obj \
@ -308,3 +307,4 @@ C_OBJS = \
!include "../../LzmaDec.mak"
!include "../../Sha1.mak"
!include "../../Sha256.mak"
!include "../../Sort.mak"

2
CPP/7zip/Bundles/SFXCon/SfxCon.cpp
View file

@ -153,7 +153,7 @@ namespace NCommandType
};
}
static const char *g_Commands = "txl";
static const char * const g_Commands = "txl";
struct CArchiveCommand
{

54
CPP/7zip/Bundles/SFXSetup/SfxSetup.cpp
View file

@ -48,72 +48,60 @@ static bool ReadDataString(CFSTR fileName, LPCSTR startID,
NIO::CInFile inFile;
if (!inFile.Open(fileName))
return false;
const size_t kBufferSize = (1 << 12);
const size_t kBufferSize = 1 << 12;
Byte buffer[kBufferSize];
const unsigned signatureStartSize = MyStringLen(startID);
const unsigned signatureEndSize = MyStringLen(endID);
const size_t signatureStartSize = MyStringLen(startID + 1);
const size_t signatureEndSize = MyStringLen(endID + 1);
size_t numBytesPrev = 0;
bool writeMode = false;
UInt64 posTotal = 0;
UInt32 posTotal = 0;
for (;;)
{
if (posTotal > (1 << 20))
return (stringResult.IsEmpty());
const size_t numReadBytes = kBufferSize - numBytesPrev;
size_t processedSize;
if (!inFile.ReadFull(buffer + numBytesPrev, numReadBytes, processedSize))
return false;
if (processedSize == 0)
return true;
const size_t numBytesInBuffer = numBytesPrev + processedSize;
UInt32 pos = 0;
numBytesPrev += processedSize;
size_t pos = 0;
for (;;)
{
if (writeMode)
{
if (pos + signatureEndSize > numBytesInBuffer)
if (pos + signatureEndSize > numBytesPrev)
break;
if (memcmp(buffer + pos, endID, signatureEndSize) == 0)
return true;
const Byte b = buffer[pos];
const Byte b = buffer[pos++];
if (b == 0)
return false;
if (b == ';' && memcmp(buffer + pos, endID + 1, signatureEndSize) == 0)
return true;
stringResult += (char)b;
pos++;
}
else
{
if (pos + signatureStartSize > numBytesInBuffer)
if (pos + signatureStartSize > numBytesPrev)
break;
if (memcmp(buffer + pos, startID, signatureStartSize) == 0)
const Byte b = buffer[pos++];
if (b == ';' && memcmp(buffer + pos, startID + 1, signatureStartSize) == 0)
{
writeMode = true;
pos += signatureStartSize;
}
else
pos++;
}
}
numBytesPrev = numBytesInBuffer - pos;
posTotal += pos;
posTotal += (UInt32)pos;
if (posTotal > (1 << 21))
return stringResult.IsEmpty();
numBytesPrev -= pos;
memmove(buffer, buffer + pos, numBytesPrev);
}
}
static char kStartID[] = { ',','!','@','I','n','s','t','a','l','l','@','!','U','T','F','-','8','!', 0 };
static char kEndID[] = { ',','!','@','I','n','s','t','a','l','l','E','n','d','@','!', 0 };
static struct CInstallIDInit
{
CInstallIDInit()
{
kStartID[0] = ';';
kEndID[0] = ';';
}
} g_CInstallIDInit;
static const char * const kStartID = ",!@Install@!UTF-8!";
static const char * const kEndID = ",!@InstallEnd@!";
#if defined(_WIN32) && defined(_UNICODE) && !defined(_WIN64) && !defined(UNDER_CE)
#define NT_CHECK_FAIL_ACTION ShowErrorMessage(L"Unsupported Windows version"); return 1;
@ -229,7 +217,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE /* hPrevInstance */,
}
const FString tempDirPath = tempDir.GetPath();
// tempDirPath = L"M:\\1\\"; // to test low disk space
// tempDirPath = "M:\\1\\"; // to test low disk space
{
bool isCorrupt = false;
UString errorMessage;
@ -308,7 +296,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE /* hPrevInstance */,
{
if (appLaunched.IsEmpty())
{
appLaunched = L"setup.exe";
appLaunched = "setup.exe";
if (!NFind::DoesFileExist_FollowLink(us2fs(appLaunched)))
{
if (!assumeYes)

2
CPP/7zip/Common/FileStreams.cpp
View file

@ -753,7 +753,7 @@ Z7_COM7F_IMF(CInFileStream::GetProperty(PROPID propID, PROPVARIANT *value))
{
if (StoreOwnerName)
{
const uid_t gid = st.st_gid;
const gid_t gid = st.st_gid;
{
if (!OwnerGroup.IsEmpty() && _gid == gid)
prop = OwnerGroup;

4
CPP/7zip/Common/FileStreams.h
View file

@ -84,8 +84,8 @@ public:
BY_HANDLE_FILE_INFORMATION _info;
#else
struct stat _info;
UInt32 _uid;
UInt32 _gid;
uid_t _uid; // uid_t can be unsigned or signed int
gid_t _gid;
UString OwnerName;
UString OwnerGroup;
bool StoreOwnerId;

10
CPP/7zip/Common/InBuffer.h
View file

@ -97,6 +97,16 @@ public:
size_t ReadBytesPart(Byte *buf, size_t size);
size_t ReadBytes(Byte *buf, size_t size);
const Byte *Lookahead(size_t &rem)
{
rem = (size_t)(_bufLim - _buf);
if (!rem)
{
ReadBlock();
rem = (size_t)(_bufLim - _buf);
}
return _buf;
}
size_t Skip(size_t size);
};

32
CPP/7zip/Common/MethodProps.cpp
View file

@ -324,15 +324,22 @@ void CCoderProps::AddProp(const CProp &prop)
HRESULT CProps::SetCoderProps(ICompressSetCoderProperties *scp, const UInt64 *dataSizeReduce) const
{
return SetCoderProps_DSReduce_Aff(scp, dataSizeReduce, NULL);
return SetCoderProps_DSReduce_Aff(scp, dataSizeReduce, NULL, NULL, NULL);
}
HRESULT CProps::SetCoderProps_DSReduce_Aff(
ICompressSetCoderProperties *scp,
const UInt64 *dataSizeReduce,
const UInt64 *affinity) const
const UInt64 *affinity,
const UInt32 *affinityGroup,
const UInt64 *affinityInGroup) const
{
CCoderProps coderProps(Props.Size() + (dataSizeReduce ? 1 : 0) + (affinity ? 1 : 0) );
CCoderProps coderProps(Props.Size()
+ (dataSizeReduce ? 1 : 0)
+ (affinity ? 1 : 0)
+ (affinityGroup ? 1 : 0)
+ (affinityInGroup ? 1 : 0)
);
FOR_VECTOR (i, Props)
coderProps.AddProp(Props[i]);
if (dataSizeReduce)
@ -349,6 +356,20 @@ HRESULT CProps::SetCoderProps_DSReduce_Aff(
prop.Value = *affinity;
coderProps.AddProp(prop);
}
if (affinityGroup)
{
CProp prop;
prop.Id = NCoderPropID::kThreadGroup;
prop.Value = *affinityGroup;
coderProps.AddProp(prop);
}
if (affinityInGroup)
{
CProp prop;
prop.Id = NCoderPropID::kAffinityInGroup;
prop.Value = *affinityInGroup;
coderProps.AddProp(prop);
}
return coderProps.SetProps(scp);
}
@ -409,6 +430,11 @@ static const CNameToPropID g_NameToPropID[] =
{ VT_UI4, "offset" },
{ VT_UI4, "zhb" }
/*
, { VT_UI4, "tgn" }, // kNumThreadGroups
, { VT_UI4, "tgi" }, // kThreadGroup
, { VT_UI8, "tga" }, // kAffinityInGroup
*/
/*
,
// { VT_UI4, "zhc" },
// { VT_UI4, "zhd" },

6
CPP/7zip/Common/MethodProps.h
View file

@ -80,7 +80,11 @@ struct CProps
}
HRESULT SetCoderProps(ICompressSetCoderProperties *scp, const UInt64 *dataSizeReduce = NULL) const;
HRESULT SetCoderProps_DSReduce_Aff(ICompressSetCoderProperties *scp, const UInt64 *dataSizeReduce, const UInt64 *affinity) const;
HRESULT SetCoderProps_DSReduce_Aff(ICompressSetCoderProperties *scp,
const UInt64 *dataSizeReduce,
const UInt64 *affinity,
const UInt32 *affinityGroup,
const UInt64 *affinityInGroup) const;
};
class CMethodProps: public CProps

29
CPP/7zip/Common/OutBuffer.h
View file

@ -45,6 +45,7 @@ public:
HRESULT Flush() throw();
void FlushWithCheck();
Z7_FORCE_INLINE
void WriteByte(Byte b)
{
UInt32 pos = _pos;
@ -54,10 +55,34 @@ public:
if (pos == _limitPos)
FlushWithCheck();
}
void WriteBytes(const void *data, size_t size)
{
for (size_t i = 0; i < size; i++)
WriteByte(((const Byte *)data)[i]);
while (size)
{
UInt32 pos = _pos;
size_t cur = (size_t)(_limitPos - pos);
if (cur >= size)
cur = size;
size -= cur;
Byte *dest = _buf + pos;
pos += (UInt32)cur;
_pos = pos;
#if 0
memcpy(dest, data, cur);
data = (const void *)((const Byte *)data + cur);
#else
const Byte * const lim = (const Byte *)data + cur;
do
{
*dest++ = *(const Byte *)data;
data = (const void *)((const Byte *)data + 1);
}
while (data != lim);
#endif
if (pos == _limitPos)
FlushWithCheck();
}
}
Byte *GetOutBuffer(size_t &avail)

2
CPP/7zip/Compress/BZip2Const.h
View file

@ -46,7 +46,7 @@ const UInt32 kBlockSizeStep = 100000;
const UInt32 kBlockSizeMax = kBlockSizeMultMax * kBlockSizeStep;
const unsigned kNumSelectorsBits = 15;
const UInt32 kNumSelectorsMax = (2 + (kBlockSizeMax / kGroupSize));
const unsigned kNumSelectorsMax = 2 + kBlockSizeMax / kGroupSize;
const unsigned kRleModeRepSize = 4;

707
CPP/7zip/Compress/BZip2Encoder.cpp
View file

File diff suppressed because it is too large Load diff

136
CPP/7zip/Compress/BZip2Encoder.h
View file

@ -3,7 +3,6 @@
#ifndef ZIP7_INC_COMPRESS_BZIP2_ENCODER_H
#define ZIP7_INC_COMPRESS_BZIP2_ENCODER_H
#include "../../Common/Defs.h"
#include "../../Common/MyCom.h"
#ifndef Z7_ST
@ -23,80 +22,114 @@
namespace NCompress {
namespace NBZip2 {
class CMsbfEncoderTemp
const unsigned kNumPassesMax = 10;
struct CMsbfEncoderTemp
{
UInt32 _pos;
unsigned _bitPos;
Byte _curByte;
unsigned _bitPos; // 0 < _bitPos <= 8 : number of non-filled low bits in _curByte
unsigned _curByte; // low (_bitPos) bits are zeros
// high (8 - _bitPos) bits are filled
Byte *_buf;
public:
void SetStream(Byte *buf) { _buf = buf; }
Byte *GetStream() const { return _buf; }
Byte *_buf_base;
void SetStream(Byte *buf) { _buf_base = _buf = buf; }
Byte *GetStream() const { return _buf_base; }
void Init()
{
_pos = 0;
_bitPos = 8;
_curByte = 0;
_buf = _buf_base;
}
void Flush()
{
if (_bitPos < 8)
WriteBits(0, _bitPos);
}
// required condition: (value >> numBits) == 0
// numBits == 0 is allowed
void WriteBits(UInt32 value, unsigned numBits)
{
while (numBits > 0)
do
{
unsigned numNewBits = MyMin(numBits, _bitPos);
numBits -= numNewBits;
_curByte = (Byte)(_curByte << numNewBits);
UInt32 newBits = value >> numBits;
_curByte |= Byte(newBits);
value -= (newBits << numBits);
_bitPos -= numNewBits;
if (_bitPos == 0)
unsigned bp = _bitPos;
unsigned curByte = _curByte;
if (numBits < bp)
{
_buf[_pos++] = _curByte;
_bitPos = 8;
bp -= numBits;
_curByte = curByte | (value << bp);
_bitPos = bp;
return;
}
numBits -= bp;
const UInt32 hi = value >> numBits;
value -= (hi << numBits);
Byte *buf = _buf;
_bitPos = 8;
_curByte = 0;
*buf++ = (Byte)(curByte | hi);
_buf = buf;
}
while (numBits);
}
void WriteBit(unsigned value)
{
const unsigned bp = _bitPos - 1;
const unsigned curByte = _curByte | (value << bp);
_curByte = curByte;
_bitPos = bp;
if (bp == 0)
{
*_buf++ = (Byte)curByte;
_curByte = 0;
_bitPos = 8;
}
}
UInt32 GetBytePos() const { return _pos ; }
UInt32 GetPos() const { return _pos * 8 + (8 - _bitPos); }
Byte GetCurByte() const { return _curByte; }
void WriteByte(unsigned b)
{
const unsigned bp = _bitPos;
const unsigned a = _curByte | (b >> (8 - bp));
_curByte = b << bp;
Byte *buf = _buf;
*buf++ = (Byte)a;
_buf = buf;
}
UInt32 GetBytePos() const { return (UInt32)(size_t)(_buf - _buf_base); }
UInt32 GetPos() const { return GetBytePos() * 8 + 8 - _bitPos; }
unsigned GetCurByte() const { return _curByte; }
unsigned GetNonFlushedByteBits() const { return _curByte >> _bitPos; }
void SetPos(UInt32 bitPos)
{
_pos = bitPos >> 3;
_buf = _buf_base + (bitPos >> 3);
_bitPos = 8 - ((unsigned)bitPos & 7);
}
void SetCurState(unsigned bitPos, Byte curByte)
void SetCurState(unsigned bitPos, unsigned curByte)
{
_bitPos = 8 - bitPos;
_curByte = curByte;
}
};
class CEncoder;
const unsigned kNumPassesMax = 10;
class CEncoder;
class CThreadInfo
{
private:
CMsbfEncoderTemp m_OutStreamCurrent;
public:
CEncoder *Encoder;
Byte *m_Block;
private:
Byte *m_MtfArray;
Byte *m_TempArray;
UInt32 *m_BlockSorterIndex;
CMsbfEncoderTemp *m_OutStreamCurrent;
public:
bool m_OptimizeNumTables;
UInt32 m_NumCrcs;
UInt32 m_BlockIndex;
UInt64 m_UnpackSize;
Byte *m_Block_Base;
Byte Lens[kNumTablesMax][kMaxAlphaSize];
UInt32 Freqs[kNumTablesMax][kMaxAlphaSize];
@ -105,20 +138,16 @@ private:
Byte m_Selectors[kNumSelectorsMax];
UInt32 m_CRCs[1 << kNumPassesMax];
UInt32 m_NumCrcs;
void WriteBits2(UInt32 value, unsigned numBits);
void WriteByte2(Byte b);
void WriteBit2(Byte v);
void WriteCrc2(UInt32 v);
void WriteByte2(unsigned b) { WriteBits2(b, 8); }
void WriteBit2(unsigned v) { m_OutStreamCurrent.WriteBit(v); }
void EncodeBlock(const Byte *block, UInt32 blockSize);
UInt32 EncodeBlockWithHeaders(const Byte *block, UInt32 blockSize);
void EncodeBlock2(const Byte *block, UInt32 blockSize, UInt32 numPasses);
public:
bool m_OptimizeNumTables;
CEncoder *Encoder;
#ifndef Z7_ST
#ifndef Z7_ST
NWindows::CThread Thread;
NWindows::NSynchronization::CAutoResetEvent StreamWasFinishedEvent;
@ -127,17 +156,14 @@ public:
// it's not member of this thread. We just need one event per thread
NWindows::NSynchronization::CAutoResetEvent CanWriteEvent;
private:
UInt32 m_BlockIndex;
UInt64 m_UnpackSize;
public:
Byte MtPad[1 << 8]; // It's pad for Multi-Threading. Must be >= Cache_Line_Size.
HRESULT Create();
void FinishStream(bool needLeave);
THREAD_FUNC_RET_TYPE ThreadFunc();
#endif
#endif
CThreadInfo(): m_Block(NULL), m_BlockSorterIndex(NULL) {}
CThreadInfo(): m_BlockSorterIndex(NULL), m_Block_Base(NULL) {}
~CThreadInfo() { Free(); }
bool Alloc();
void Free();
@ -145,16 +171,19 @@ public:
HRESULT EncodeBlock3(UInt32 blockSize);
};
struct CEncProps
{
UInt32 BlockSizeMult;
UInt32 NumPasses;
UInt32 NumThreadGroups;
UInt64 Affinity;
CEncProps()
{
BlockSizeMult = (UInt32)(Int32)-1;
NumPasses = (UInt32)(Int32)-1;
NumThreadGroups = 0;
Affinity = 0;
}
void Normalize(int level);
@ -206,6 +235,7 @@ public:
bool CloseThreads;
bool StreamWasFinished;
NWindows::NSynchronization::CManualResetEvent CanStartWaitingEvent;
CThreadNextGroup ThreadNextGroup;
HRESULT Result;
ICompressProgressInfo *Progress;
@ -218,12 +248,8 @@ public:
UInt64 GetInProcessedSize() const { return m_InStream.GetProcessedSize(); }
UInt32 ReadRleBlock(Byte *buf);
void WriteBytes(const Byte *data, UInt32 sizeInBits, Byte lastByte);
void WriteBits(UInt32 value, unsigned numBits);
void WriteBytes(const Byte *data, UInt32 sizeInBits, unsigned lastByteBits);
void WriteByte(Byte b);
// void WriteBit(Byte v);
void WriteCrc(UInt32 v);
#ifndef Z7_ST
HRESULT Create();

1
CPP/7zip/Compress/BitlEncoder.h
View file

@ -33,6 +33,7 @@ public:
_bitPos = 8;
_curByte = 0;
}
Z7_FORCE_INLINE
void WriteBits(UInt32 value, unsigned numBits)
{
while (numBits > 0)

75
CPP/7zip/Compress/BitmEncoder.h
View file

@ -8,8 +8,9 @@
template<class TOutByte>
class CBitmEncoder
{
unsigned _bitPos;
Byte _curByte;
unsigned _bitPos; // 0 < _bitPos <= 8 : number of non-filled low bits in _curByte
unsigned _curByte; // low (_bitPos) bits are zeros
// high (8 - _bitPos) bits are filled
TOutByte _stream;
public:
bool Create(UInt32 bufferSize) { return _stream.Create(bufferSize); }
@ -24,25 +25,65 @@ public:
HRESULT Flush()
{
if (_bitPos < 8)
WriteBits(0, _bitPos);
return _stream.Flush();
}
void WriteBits(UInt32 value, unsigned numBits)
{
while (numBits > 0)
{
if (numBits < _bitPos)
{
_curByte = (Byte)(_curByte | (value << (_bitPos -= numBits)));
return;
}
numBits -= _bitPos;
UInt32 newBits = (value >> numBits);
value -= (newBits << numBits);
_stream.WriteByte((Byte)(_curByte | newBits));
_stream.WriteByte((Byte)_curByte);
_bitPos = 8;
_curByte = 0;
}
return _stream.Flush();
}
// required condition: (value >> numBits) == 0
// numBits == 0 is allowed
void WriteBits(UInt32 value, unsigned numBits)
{
do
{
unsigned bp = _bitPos;
unsigned curByte = _curByte;
if (numBits < bp)
{
bp -= numBits;
_curByte = curByte | (value << bp);
_bitPos = bp;
return;
}
numBits -= bp;
const UInt32 hi = (value >> numBits);
value -= (hi << numBits);
_stream.WriteByte((Byte)(curByte | hi));
_bitPos = 8;
_curByte = 0;
}
while (numBits);
}
void WriteByte(unsigned b)
{
const unsigned bp = _bitPos;
const unsigned a = _curByte | (b >> (8 - bp));
_curByte = b << bp;
_stream.WriteByte((Byte)a);
}
void WriteBytes(const Byte *data, size_t num)
{
const unsigned bp = _bitPos;
#if 1 // 1 for optional speed-optimized code branch
if (bp == 8)
{
_stream.WriteBytes(data, num);
return;
}
#endif
unsigned c = _curByte;
const unsigned bp_rev = 8 - bp;
for (size_t i = 0; i < num; i++)
{
const unsigned b = data[i];
_stream.WriteByte((Byte)(c | (b >> bp_rev)));
c = b << bp;
}
_curByte = c;
}
};

16
CPP/7zip/Compress/DeflateDecoder.cpp
View file

@ -117,15 +117,13 @@ bool CCoder::ReadTables(void)
if (_numDistLevels > kDistTableSize32)
return false;
Byte levelLevels[kLevelTableSize];
for (unsigned i = 0; i < kLevelTableSize; i++)
{
const unsigned position = kCodeLengthAlphabetOrder[i];
if (i < numLevelCodes)
levelLevels[position] = (Byte)ReadBits(kLevelFieldSize);
else
levelLevels[position] = 0;
}
const unsigned kLevelTableSize_aligned4 = kLevelTableSize + 1;
Byte levelLevels[kLevelTableSize_aligned4];
memset (levelLevels, 0, sizeof(levelLevels));
unsigned i = 0;
do
levelLevels[kCodeLengthAlphabetOrder[i++]] = (Byte)ReadBits(kLevelFieldSize);
while (i != numLevelCodes);
if (m_InBitStream.ExtraBitsWereRead())
return false;

100
CPP/7zip/Compress/DeflateEncoder.cpp
View file

@ -19,12 +19,16 @@
#define NO_INLINE
#endif
#define MAX_HUF_LEN_12 12
namespace NCompress {
namespace NDeflate {
namespace NEncoder {
static const unsigned k_CodeValue_Len_Is_Literal_Flag = 1u << 15;
static const unsigned kNumDivPassesMax = 10; // [0, 16); ratio/speed/ram tradeoff; use big value for better compression ratio.
static const UInt32 kNumTables = (1 << kNumDivPassesMax);
static const unsigned kNumTables = 1u << kNumDivPassesMax;
static const UInt32 kFixedHuffmanCodeBlockSizeMax = (1 << 8); // [0, (1 << 32)); ratio/speed tradeoff; use big value for better compression ratio.
static const UInt32 kDivideCodeBlockSizeMin = (1 << 7); // [1, (1 << 32)); ratio/speed tradeoff; use small value for better compression ratio.
@ -77,7 +81,7 @@ public:
static CFastPosInit g_FastPosInit;
inline UInt32 GetPosSlot(UInt32 pos)
inline unsigned GetPosSlot(UInt32 pos)
{
/*
if (pos < 0x200)
@ -162,13 +166,13 @@ HRESULT CCoder::Create()
// COM_TRY_BEGIN
if (!m_Values)
{
m_Values = (CCodeValue *)MyAlloc((kMaxUncompressedBlockSize) * sizeof(CCodeValue));
m_Values = (CCodeValue *)MyAlloc(kMaxUncompressedBlockSize * sizeof(CCodeValue));
if (!m_Values)
return E_OUTOFMEMORY;
}
if (!m_Tables)
{
m_Tables = (CTables *)MyAlloc((kNumTables) * sizeof(CTables));
m_Tables = (CTables *)MyAlloc(kNumTables * sizeof(CTables));
if (!m_Tables)
return E_OUTOFMEMORY;
}
@ -268,19 +272,21 @@ NO_INLINE void CCoder::GetMatches()
UInt32 distanceTmp[kMatchMaxLen * 2 + 3];
const UInt32 numPairs = (UInt32)((_btMode ?
const size_t numPairs = (size_t)((_btMode ?
Bt3Zip_MatchFinder_GetMatches(&_lzInWindow, distanceTmp):
Hc3Zip_MatchFinder_GetMatches(&_lzInWindow, distanceTmp)) - distanceTmp);
*m_MatchDistances = (UInt16)numPairs;
UInt16 *matchDistances = m_MatchDistances;
*matchDistances++ = (UInt16)numPairs;
if (numPairs != 0)
{
UInt32 i;
size_t i;
for (i = 0; i < numPairs; i += 2)
{
m_MatchDistances[(size_t)i + 1] = (UInt16)distanceTmp[i];
m_MatchDistances[(size_t)i + 2] = (UInt16)distanceTmp[(size_t)i + 1];
matchDistances[0] = (UInt16)distanceTmp[i];
matchDistances[1] = (UInt16)distanceTmp[(size_t)i + 1];
matchDistances += 2;
}
UInt32 len = distanceTmp[(size_t)numPairs - 2];
if (len == m_NumFastBytes && m_NumFastBytes != m_MatchMaxLen)
@ -291,11 +297,11 @@ NO_INLINE void CCoder::GetMatches()
if (numAvail > m_MatchMaxLen)
numAvail = m_MatchMaxLen;
for (; len < numAvail && pby[len] == pby2[len]; len++);
m_MatchDistances[(size_t)i - 1] = (UInt16)len;
matchDistances[-2] = (UInt16)len;
}
}
if (m_IsMultiPass)
m_Pos += numPairs + 1;
m_Pos += (UInt32)numPairs + 1;
if (!m_SecondPass)
m_AdditionalOffset++;
}
@ -535,6 +541,7 @@ NO_INLINE void CCoder::WriteBits(UInt32 value, unsigned numBits)
}
#define WRITE_HF2(codes, lens, i) m_OutStream.WriteBits(codes[i], lens[i])
#define WRITE_HF2_NO_INLINE(codes, lens, i) WriteBits(codes[i], lens[i])
#define WRITE_HF(i) WriteBits(codes[i], lens[i])
NO_INLINE void CCoder::LevelTableCode(const Byte *levels, unsigned numLevels, const Byte *lens, const UInt32 *codes)
@ -619,17 +626,22 @@ static NO_INLINE UInt32 Huffman_GetPrice(const UInt32 *freqs, const Byte *lens,
return price;
}
static NO_INLINE UInt32 Huffman_GetPrice_Spec(const UInt32 *freqs, const Byte *lens, UInt32 num, const Byte *extraBits, UInt32 extraBase)
static NO_INLINE UInt32 Huffman_GetPrice_Spec(
const UInt32 *freqs, const Byte *lens, UInt32 num,
const Byte *extraBits, UInt32 extraBase)
{
return Huffman_GetPrice(freqs, lens, num) +
return
Huffman_GetPrice(freqs, lens, num) +
Huffman_GetPrice(freqs + extraBase, extraBits, num - extraBase);
}
NO_INLINE UInt32 CCoder::GetLzBlockPrice() const
{
return
Huffman_GetPrice_Spec(mainFreqs, m_NewLevels.litLenLevels, kFixedMainTableSize, m_LenDirectBits, kSymbolMatch) +
Huffman_GetPrice_Spec(distFreqs, m_NewLevels.distLevels, kDistTableSize64, kDistDirectBits, 0);
Huffman_GetPrice_Spec(mainFreqs, m_NewLevels.litLenLevels,
kFixedMainTableSize, m_LenDirectBits, kSymbolMatch) +
Huffman_GetPrice_Spec(distFreqs, m_NewLevels.distLevels,
kDistTableSize64, kDistDirectBits, 0);
}
NO_INLINE void CCoder::TryBlock()
@ -658,7 +670,7 @@ NO_INLINE void CCoder::TryBlock()
CCodeValue &codeValue = m_Values[m_ValueIndex++];
if (len >= kMatchMinLen)
{
UInt32 newLen = len - kMatchMinLen;
const UInt32 newLen = len - kMatchMinLen;
codeValue.Len = (UInt16)newLen;
mainFreqs[kSymbolMatch + (size_t)g_LenSlots[newLen]]++;
codeValue.Pos = (UInt16)pos;
@ -666,10 +678,10 @@ NO_INLINE void CCoder::TryBlock()
}
else
{
Byte b = *(Inline_MatchFinder_GetPointerToCurrentPos(&_lzInWindow) - m_AdditionalOffset);
const unsigned b = *(Inline_MatchFinder_GetPointerToCurrentPos(&_lzInWindow) - m_AdditionalOffset);
mainFreqs[b]++;
codeValue.SetAsLiteral();
codeValue.Pos = b;
codeValue.Len = k_CodeValue_Len_Is_Literal_Flag;
codeValue.Pos = (UInt16)b;
}
m_AdditionalOffset -= len;
BlockSizeRes += len;
@ -704,16 +716,24 @@ NO_INLINE void CCoder::SetPrices(const CLevels &levels)
}
}
#if MAX_HUF_LEN_12 > 12
// Huffman_ReverseBits() now supports 12-bits values only.
#error Stop_Compiling_Bad_MAX_HUF_LEN_12
#endif
static NO_INLINE void Huffman_ReverseBits(UInt32 *codes, const Byte *lens, UInt32 num)
{
for (UInt32 i = 0; i < num; i++)
const Byte * const lens_lim = lens + num;
do
{
UInt32 x = codes[i];
x = ((x & 0x5555) << 1) | ((x & 0xAAAA) >> 1);
x = ((x & 0x3333) << 2) | ((x & 0xCCCC) >> 2);
x = ((x & 0x0F0F) << 4) | ((x & 0xF0F0) >> 4);
codes[i] = (((x & 0x00FF) << 8) | ((x & 0xFF00) >> 8)) >> (16 - lens[i]);
// we should change constants, if lens[*] can be larger than 12.
UInt32 x = *codes;
x = ((x & (0x555 )) << 2) + (x & (0xAAA ));
x = ((x & (0x333 << 1)) << 4) | (x & (0xCCC << 1));
x = ((x & (0xF0F << 3)) << 8) | (x & (0x0F0 << 3));
// we can use (x) instead of (x & (0xFF << 7)), if we support garabage data after (*lens) bits.
*codes++ = (((x & (0xFF << 7)) << 16) | x) >> (*lens ^ 31);
}
while (++lens != lens_lim);
}
NO_INLINE void CCoder::WriteBlock()
@ -721,24 +741,28 @@ NO_INLINE void CCoder::WriteBlock()
Huffman_ReverseBits(mainCodes, m_NewLevels.litLenLevels, kFixedMainTableSize);
Huffman_ReverseBits(distCodes, m_NewLevels.distLevels, kDistTableSize64);
for (UInt32 i = 0; i < m_ValueIndex; i++)
CCodeValue *values = m_Values;
const CCodeValue * const values_lim = values + m_ValueIndex;
if (values != values_lim)
do
{
const CCodeValue &codeValue = m_Values[i];
if (codeValue.IsLiteral())
WRITE_HF2(mainCodes, m_NewLevels.litLenLevels, codeValue.Pos);
const UInt32 len = values->Len;
const UInt32 dist = values->Pos;
if (len == k_CodeValue_Len_Is_Literal_Flag)
WRITE_HF2(mainCodes, m_NewLevels.litLenLevels, dist);
else
{
UInt32 len = codeValue.Len;
UInt32 lenSlot = g_LenSlots[len];
const unsigned lenSlot = g_LenSlots[len];
WRITE_HF2(mainCodes, m_NewLevels.litLenLevels, kSymbolMatch + lenSlot);
m_OutStream.WriteBits(len - m_LenStart[lenSlot], m_LenDirectBits[lenSlot]);
UInt32 dist = codeValue.Pos;
UInt32 posSlot = GetPosSlot(dist);
const unsigned posSlot = GetPosSlot(dist);
WRITE_HF2(distCodes, m_NewLevels.distLevels, posSlot);
m_OutStream.WriteBits(dist - kDistStart[posSlot], kDistDirectBits[posSlot]);
}
}
WRITE_HF2(mainCodes, m_NewLevels.litLenLevels, kSymbolEndOfBlock);
while (++values != values_lim);
WRITE_HF2_NO_INLINE(mainCodes, m_NewLevels.litLenLevels, kSymbolEndOfBlock);
}
static UInt32 GetStorePrice(UInt32 blockSize, unsigned bitPosition)
@ -787,10 +811,10 @@ NO_INLINE UInt32 CCoder::TryDynBlock(unsigned tableIndex, UInt32 numPasses)
{
m_Pos = posTemp;
TryBlock();
unsigned numHuffBits =
(m_ValueIndex > 18000 ? 12 :
(m_ValueIndex > 7000 ? 11 :
(m_ValueIndex > 2000 ? 10 : 9)));
const unsigned numHuffBits =
m_ValueIndex > 18000 ? MAX_HUF_LEN_12 :
m_ValueIndex > 7000 ? 11 :
m_ValueIndex > 2000 ? 10 : 9;
MakeTables(numHuffBits);
SetPrices(m_NewLevels);
}

10
CPP/7zip/Compress/Lzma2Encoder.cpp
View file

@ -52,7 +52,15 @@ HRESULT SetLzma2Prop(PROPID propID, const PROPVARIANT &prop, CLzma2EncProps &lzm
case NCoderPropID::kNumThreads:
if (prop.vt != VT_UI4)
return E_INVALIDARG;
lzma2Props.numTotalThreads = (int)(prop.ulVal);
lzma2Props.numTotalThreads = (int)prop.ulVal;
break;
case NCoderPropID::kNumThreadGroups:
if (prop.vt != VT_UI4)
return E_INVALIDARG;
// 16-bit value supported by Windows
if (prop.ulVal >= (1u << 16))
return E_INVALIDARG;
lzma2Props.numThreadGroups = (unsigned)prop.ulVal;
break;
default:
RINOK(NLzma::SetLzmaProp(propID, prop, lzma2Props.lzmaProps))

18
CPP/7zip/Compress/LzmaEncoder.cpp
View file

@ -101,6 +101,24 @@ HRESULT SetLzmaProp(PROPID propID, const PROPVARIANT &prop, CLzmaEncProps &ep)
return S_OK;
}
if (propID == NCoderPropID::kAffinityInGroup)
{
if (prop.vt == VT_UI8)
ep.affinityInGroup = prop.uhVal.QuadPart;
else
return E_INVALIDARG;
return S_OK;
}
if (propID == NCoderPropID::kThreadGroup)
{
if (prop.vt == VT_UI4)
ep.affinityGroup = (Int32)(UInt32)prop.ulVal;
else
return E_INVALIDARG;
return S_OK;
}
if (propID == NCoderPropID::kHashBits)
{
if (prop.vt == VT_UI4)

13
CPP/7zip/Compress/Mtf8.h
View file

@ -13,6 +13,18 @@ struct CMtf8Encoder
unsigned FindAndMove(Byte v) throw()
{
#if 1
Byte b = Buf[0];
if (v == b)
return 0;
Buf[0] = v;
for (unsigned pos = 0;;)
{
Byte a;
a = Buf[++pos]; Buf[pos] = b; if (v == a) return pos;
b = Buf[++pos]; Buf[pos] = a; if (v == b) return pos;
}
#else
size_t pos;
for (pos = 0; Buf[pos] != v; pos++);
const unsigned resPos = (unsigned)pos;
@ -31,6 +43,7 @@ struct CMtf8Encoder
Buf[pos] = Buf[pos - 1];
Buf[0] = v;
return resPos;
#endif
}
};

127
CPP/7zip/Compress/Rar5Decoder.cpp
View file

@ -936,31 +936,30 @@ HRESULT CDecoder::ExecuteFilter(const CFilter &f)
HRESULT CDecoder::WriteBuf()
{
DeleteUnusedFilters();
const UInt64 lzSize = _lzSize + _winPos;
for (unsigned i = 0; i < _numFilters;)
{
const CFilter &f = _filters[i];
const UInt64 blockStart = f.Start;
const size_t lzAvail = (size_t)(lzSize - _lzWritten);
if (lzAvail == 0)
break;
// (lzAvail != 0)
const CFilter &f = _filters[i];
const UInt64 blockStart = f.Start;
if (blockStart > _lzWritten)
{
const UInt64 rem = blockStart - _lzWritten;
// (rem != 0)
size_t size = lzAvail;
if (size > rem)
size = (size_t)rem;
if (size != 0) // is it true always ?
{
RINOK(WriteData(_window + _winPos - lzAvail, size))
_lzWritten += size;
}
// (size != 0)
RINOK(WriteData(_window + _winPos - lzAvail, size))
_lzWritten += size;
continue;
}
// (blockStart <= _lzWritten)
const UInt32 blockSize = f.Size;
size_t offset = (size_t)(_lzWritten - blockStart);
if (offset == 0)
@ -987,10 +986,8 @@ HRESULT CDecoder::WriteBuf()
}
DeleteUnusedFilters();
if (_numFilters)
return S_OK;
const size_t lzAvail = (size_t)(lzSize - _lzWritten);
RINOK(WriteData(_window + _winPos - lzAvail, lzAvail))
_lzWritten += lzAvail;
@ -1367,6 +1364,12 @@ enum enum_exit_type
Z7_HUFF_DECODE_CHECK(sym, huf, kNumHufBits, kNumTableBits, bitStream, { LZ_LOOP_BREAK_ERROR })
/*
DecodeLZ2() will stop decoding if it reaches limit when (_winPos >= _limit)
at return:
(_winPos < _limit + kMaxMatchLen)
also it can write up to (COPY_CHUNK_SIZE - 1) additional junk bytes after (_winPos).
*/
HRESULT CDecoder::DecodeLZ2(const CBitDecoder &bitStream) throw()
{
#if 0
@ -1656,6 +1659,13 @@ decode_error:
/*
input conditions:
_winPos < _winSize
return:
_winPos < _winSize is expected, if (return_res == S_OK)
_winPos >= _winSize is possible in (return_res != S_OK)
*/
HRESULT CDecoder::DecodeLZ()
{
CBitDecoder _bitStream;
@ -1679,6 +1689,8 @@ HRESULT CDecoder::DecodeLZ()
if (winPos >= limit)
{
_winPos = winPos < _winSize ? winPos : _winSize;
// _winPos == min(winPos, _winSize)
// we will not write data after _winSize
RINOK(WriteBuf())
if (_unpackSize_Defined && _writtenFileSize > _unpackSize)
break; // return S_FALSE;
@ -1854,7 +1866,15 @@ Z7_COM7F_IMF(CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream
{
// if (_winPos > 100) _winPos -= 100; // for debug: corruption
const UInt64 lzSize = _lzSize + _winPos;
if (!_isSolid || !_wasInit
/*
if previous file was decoded with error or for some another cases, then
(lzSize > _lzEnd) is possible
(_winPos > _winSize) is possible
(_winPos < _winSize + kMaxMatchLen)
*/
if (!_window
|| !_isSolid
|| !_wasInit
|| (lzSize < _lzEnd
#if Z7_RAR_RECOVER_SOLID_LIMIT != 0
&& lzSize + Z7_RAR_RECOVER_SOLID_LIMIT < _lzEnd
@ -1863,9 +1883,9 @@ Z7_COM7F_IMF(CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream
{
if (_isSolid)
_lzError = LZ_ERROR_TYPE_HEADER;
_lzEnd = 0;
_lzSize = 0;
_lzWritten = 0;
// _lzEnd = 0; // it will be set later
// _lzWritten = 0; // it will be set later
_winPos = 0;
for (unsigned i = 0; i < kNumReps; i++)
_reps[i] = (size_t)0 - 1;
@ -1873,51 +1893,67 @@ Z7_COM7F_IMF(CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream
_tableWasFilled = false;
_wasInit = true;
}
#if Z7_RAR_RECOVER_SOLID_LIMIT != 0
else if (lzSize < _lzEnd)
else
{
#if 0
return S_FALSE;
#else
// we can report that recovering was made:
// _lzError = LZ_ERROR_TYPE_HEADER;
// We write zeros to area after corruption:
if (_window)
const size_t ws = _winSize;
if (_winPos >= ws)
{
UInt64 rem = _lzEnd - lzSize;
const size_t ws = _winSize;
if (rem >= ws)
// we must normalize (_winPos) and data in _window,
_winPos -= ws;
_lzSize += ws;
// (_winPos < kMaxMatchLen < _winSize)
// if (_window)
memcpy(_window, _window + ws, _winPos); // memmove is not required here
}
#if Z7_RAR_RECOVER_SOLID_LIMIT != 0
if (lzSize < _lzEnd)
{
#if 0
return S_FALSE;
#else
// we can report that recovering was made:
// _lzError = LZ_ERROR_TYPE_HEADER;
// We write zeros to area after corruption:
// if (_window)
{
My_ZeroMemory(_window, ws);
_lzSize = ws;
_winPos = 0;
}
else
{
const size_t cur = ws - _winPos;
if (cur <= rem)
UInt64 rem = _lzEnd - lzSize;
if (rem >= ws)
{
rem -= cur;
My_ZeroMemory(_window + _winPos, cur);
_lzSize += _winPos;
My_ZeroMemory(_window, ws);
_lzSize = ws;
_winPos = 0;
}
My_ZeroMemory(_window + _winPos, (size_t)rem);
_winPos += (size_t)rem;
else
{
// rem < _winSize
// _winPos <= ws
const size_t cur = ws - _winPos;
if (cur <= rem)
{
rem -= cur;
My_ZeroMemory(_window + _winPos, cur);
_lzSize = ws;
_winPos = 0;
}
My_ZeroMemory(_window + _winPos, (size_t)rem);
_winPos += (size_t)rem;
}
}
}
// else return S_FALSE;
// else return S_FALSE;
#endif
}
}
#endif
}
// _winPos < _winSize
// we don't want _lzSize overflow
if (_lzSize >= DICT_SIZE_MAX)
_lzSize = DICT_SIZE_MAX;
_lzEnd = _lzSize + _winPos;
// _lzSize <= DICT_SIZE_MAX
// _lzEnd <= DICT_SIZE_MAX * 2
// _lzEnd < DICT_SIZE_MAX + _winSize
size_t newSize = _dictSize;
if (newSize < kWinSize_Min)
@ -1941,10 +1977,11 @@ Z7_COM7F_IMF(CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream
// If dictionary was increased in solid, we don't want grow.
return S_FALSE; // E_OUTOFMEMORY
}
// (newSize <= _winSize)
// (newSize <= _dictSize_forCheck)
}
else
{
// !_isSolid || !_window
_dictSize_forCheck = newSize;
{
size_t newSize_small = newSize;
@ -1964,7 +2001,7 @@ Z7_COM7F_IMF(CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream
if (!_window || allocSize > _winSize_Allocated)
{
Z7_RAR_FREE_WINDOW
_window = NULL;
_window = NULL;
_winSize_Allocated = 0;
Byte *win = (Byte *)::BigAlloc(allocSize);
if (!win)

27
CPP/7zip/Crypto/MyAes.cpp
View file

@ -153,7 +153,26 @@ Z7_COM7F_IMF2(UInt32, CAesCtrCoder::Filter(Byte *data, UInt32 size))
#ifndef Z7_EXTRACT_ONLY
#ifdef MY_CPU_X86_OR_AMD64
#define USE_HW_AES
#if defined(__INTEL_COMPILER)
#if (__INTEL_COMPILER >= 1110)
#define USE_HW_AES
#if (__INTEL_COMPILER >= 1900)
#define USE_HW_VAES
#endif
#endif
#elif defined(Z7_CLANG_VERSION) && (Z7_CLANG_VERSION >= 30800) \
|| defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40400)
#define USE_HW_AES
#if defined(__clang__) && (__clang_major__ >= 8) \
|| defined(__GNUC__) && (__GNUC__ >= 8)
#define USE_HW_VAES
#endif
#elif defined(_MSC_VER)
#define USE_HW_AES
#define USE_HW_VAES
#endif
#elif defined(MY_CPU_ARM_OR_ARM64) && defined(MY_CPU_LE)
#if defined(__ARM_FEATURE_AES) \
@ -186,15 +205,15 @@ Z7_COM7F_IMF2(UInt32, CAesCtrCoder::Filter(Byte *data, UInt32 size))
#define SET_AES_FUNC_2(f2) \
if (algo == 2) if (g_Aes_SupportedFunctions_Flags & k_Aes_SupportedFunctions_HW) \
{ f = f2; }
#ifdef MY_CPU_X86_OR_AMD64
#ifdef USE_HW_VAES
#define SET_AES_FUNC_23(f2, f3) \
SET_AES_FUNC_2(f2) \
if (algo == 3) if (g_Aes_SupportedFunctions_Flags & k_Aes_SupportedFunctions_HW_256) \
{ f = f3; }
#else // MY_CPU_X86_OR_AMD64
#else // USE_HW_VAES
#define SET_AES_FUNC_23(f2, f3) \
SET_AES_FUNC_2(f2)
#endif // MY_CPU_X86_OR_AMD64
#endif // USE_HW_VAES
#else // USE_HW_AES
#define SET_AES_FUNC_23(f2, f3)
#endif // USE_HW_AES

3
CPP/7zip/ICoder.h
View file

@ -136,6 +136,9 @@ namespace NCoderPropID
kAffinity, // VT_UI8
kBranchOffset, // VT_UI4
kHashBits, // VT_UI4
kNumThreadGroups, // VT_UI4
kThreadGroup, // VT_UI4
kAffinityInGroup, // VT_UI8
/*
// kHash3Bits, // VT_UI4
// kHash2Bits, // VT_UI4

6
CPP/7zip/Sort.mak Normal file
View file

@ -0,0 +1,6 @@
!IF defined(USE_NO_ASM) || defined(USE_C_SORT) || "$(PLATFORM)" == "ia64" || "$(PLATFORM)" == "mips" || "$(PLATFORM)" == "arm" || "$(PLATFORM)" == "arm64"
C_OBJS = $(C_OBJS) \
!ELSE
ASM_OBJS = $(ASM_OBJS) \
!ENDIF
$O\Sort.obj

2
CPP/7zip/UI/Agent/AgentProxy.cpp
View file

@ -636,7 +636,7 @@ HRESULT CProxyArc2::Load(const CArc &arc, IProgress *progress)
file.Name = (const wchar_t *)p;
file.NameLen = 0;
if (size >= sizeof(wchar_t))
file.NameLen = size / sizeof(wchar_t) - 1;
file.NameLen = size / (unsigned)sizeof(wchar_t) - 1;
}
else
#endif

2
CPP/7zip/UI/Client7z/makefile.gcc
View file

@ -24,7 +24,6 @@ else
SYS_OBJS = \
$O/MyWindows.o \
$O/TimeUtils.o \
endif
@ -53,6 +52,7 @@ WIN_OBJS = \
$O/FileName.o \
$O/PropVariant.o \
$O/PropVariantConv.o \
$O/TimeUtils.o \
7ZIP_COMMON_OBJS = \
$O/FileStreams.o \

82
CPP/7zip/UI/Common/ArchiveCommandLine.cpp
View file

@ -63,17 +63,46 @@ EXTERN_C_END
#else
// #define MY_isatty_fileno(x) (isatty(fileno(x)))
// #define MY_IS_TERMINAL(x) (MY_isatty_fileno(x) != 0);
static inline bool MY_IS_TERMINAL(FILE *x)
static bool MY_IS_TERMINAL(FILE *x)
{
return (
#if defined(_MSC_VER) && (_MSC_VER >= 1400)
_isatty(_fileno(x))
#else
isatty(fileno(x))
#endif
!= 0);
#ifdef _WIN32
/*
crt/stdio.h:
typedef struct _iobuf FILE;
#define stdin (&_iob[0])
#define stdout (&_iob[1])
#define stderr (&_iob[2])
*/
// fprintf(stderr, "\nMY_IS_TERMINAL = %p", x);
const int fd = _fileno(x);
/* (fd) is 0, 1 or 2 in console program.
docs: If stdout or stderr is not associated with
an output stream (for example, in a Windows application
without a console window), the file descriptor returned is -2.
In previous versions, the file descriptor returned was -1.
*/
if (fd < 0) // is not associated with an output stream application (without a console window)
return false;
// fprintf(stderr, "\n\nstderr _fileno(%p) = %d", x, fd);
if (!_isatty(fd))
return false;
// fprintf(stderr, "\nisatty_val = true");
const HANDLE h = (HANDLE)_get_osfhandle(fd);
/* _get_osfhandle() returns intptr_t in new SDK, or long in MSVC6.
Also it can return (INVALID_HANDLE_VALUE).
docs: _get_osfhandle also returns the special value -2 when
the file descriptor is not associated with a stream
in old msvcrt.dll: it returns (-1) for incorrect value
*/
// fprintf(stderr, "\n_get_osfhandle() = %p", (void *)h);
if (h == NULL || h == INVALID_HANDLE_VALUE)
return false;
DWORD st;
// fprintf(stderr, "\nGetConsoleMode() = %u", (unsigned)GetConsoleMode(h, &st));
return GetConsoleMode(h, &st) != 0;
#else
return isatty(fileno(x)) != 0;
#endif
}
#endif
@ -312,7 +341,7 @@ static const CSwitchForm kSwitchForms[] =
{ "spf", SWFRM_STRING_SINGL(0) },
{ "snh", SWFRM_MINUS },
{ "snld", SWFRM_MINUS },
{ "snld", SWFRM_STRING },
{ "snl", SWFRM_MINUS },
{ "sni", SWFRM_SIMPLE },
@ -397,7 +426,7 @@ static NRecursedType::EEnum GetRecursedTypeFromIndex(int index)
}
}
static const char *g_Commands = "audtexlbih";
static const char * const g_Commands = "audtexlbih";
static bool ParseArchiveCommand(const UString &commandString, CArcCommand &command)
{
@ -1088,7 +1117,7 @@ void CArcCmdLineParser::Parse1(const UStringVector &commandStrings,
const UString &s = parser[NKey::kLargePages].PostStrings[0];
if (s.IsEmpty())
slp = 1;
else if (s != L"-")
else if (!s.IsEqualTo("-"))
{
if (!StringToUInt32(s, slp))
throw CArcCmdLineException("Unsupported switch postfix for -slp", s);
@ -1338,7 +1367,7 @@ void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options)
const UString &s = parser[NKey::kFullPathMode].PostStrings[0];
if (!s.IsEmpty())
{
if (s == L"2")
if (s.IsEqualTo("2"))
censorPathMode = NWildcard::k_FullPath;
else
throw CArcCmdLineException("Unsupported -spf:", s);
@ -1400,6 +1429,7 @@ void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options)
const bool isExtractGroupCommand = options.Command.IsFromExtractGroup();
const bool isExtractOrList = isExtractGroupCommand || options.Command.CommandType == NCommandType::kList;
const bool isRename = options.Command.CommandType == NCommandType::kRename;
options.UpdateOptions.RenameMode = isRename;
if ((isExtractOrList || isRename) && options.StdInMode)
thereIsArchiveName = false;
@ -1449,14 +1479,8 @@ void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options)
SetBoolPair(parser, NKey::kStoreOwnerId, options.StoreOwnerId);
SetBoolPair(parser, NKey::kStoreOwnerName, options.StoreOwnerName);
CBoolPair symLinks_AllowDangerous;
SetBoolPair(parser, NKey::kSymLinks_AllowDangerous, symLinks_AllowDangerous);
/*
bool supportSymLink = options.SymLinks.Val;
if (!options.SymLinks.Def)
{
if (isExtractOrList)
@ -1464,7 +1488,6 @@ void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options)
else
supportSymLink = false;
}
#ifdef ENV_HAVE_LSTAT
if (supportSymLink)
global_use_lstat = 1;
@ -1473,7 +1496,6 @@ void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options)
#endif
*/
if (isExtractOrList)
{
CExtractOptionsBase &eo = options.ExtractOptions;
@ -1497,7 +1519,15 @@ void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options)
if (!options.SymLinks.Def)
nt.SymLinks.Val = true;
nt.SymLinks_AllowDangerous = symLinks_AllowDangerous;
if (parser[NKey::kSymLinks_AllowDangerous].ThereIs)
{
const UString &s = parser[NKey::kSymLinks_AllowDangerous].PostStrings[0];
UInt32 v = 9; // default value for "-snld" instead of default = 5 without "-snld".
if (!s.IsEmpty())
if (!StringToUInt32(s, v))
throw CArcCmdLineException("Unsupported switch postfix -snld", s);
nt.SymLinks_DangerousLevel = (unsigned)v;
}
nt.ReplaceColonForAltStream = parser[NKey::kReplaceColonForAltStream].ThereIs;
nt.WriteToAltStreamIfColon = parser[NKey::kWriteToAltStreamIfColon].ThereIs;
@ -1516,9 +1546,9 @@ void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options)
const UString &s = parser[NKey::kZoneFile].PostStrings[0];
if (!s.IsEmpty())
{
if (s == L"0") eo.ZoneMode = NExtract::NZoneIdMode::kNone;
else if (s == L"1") eo.ZoneMode = NExtract::NZoneIdMode::kAll;
else if (s == L"2") eo.ZoneMode = NExtract::NZoneIdMode::kOffice;
if (s.IsEqualTo("0")) eo.ZoneMode = NExtract::NZoneIdMode::kNone;
else if (s.IsEqualTo("1")) eo.ZoneMode = NExtract::NZoneIdMode::kAll;
else if (s.IsEqualTo("2")) eo.ZoneMode = NExtract::NZoneIdMode::kOffice;
else
throw CArcCmdLineException("Unsupported -snz:", s);
}

1144
CPP/7zip/UI/Common/ArchiveExtractCallback.cpp
View file

File diff suppressed because it is too large Load diff

233
CPP/7zip/UI/Common/ArchiveExtractCallback.h
View file

@ -52,7 +52,6 @@ struct CExtractNtOptions
{
CBoolPair NtSecurity;
CBoolPair SymLinks;
CBoolPair SymLinks_AllowDangerous;
CBoolPair HardLinks;
CBoolPair AltStreams;
bool ReplaceColonForAltStream;
@ -66,6 +65,8 @@ struct CExtractNtOptions
bool PreserveATime;
bool OpenShareForWrite;
unsigned SymLinks_DangerousLevel;
UInt64 MemLimit;
CExtractNtOptions():
@ -74,10 +75,10 @@ struct CExtractNtOptions
ExtractOwner(false),
PreserveATime(false),
OpenShareForWrite(false),
SymLinks_DangerousLevel(5),
MemLimit((UInt64)(Int64)-1)
{
SymLinks.Val = true;
SymLinks_AllowDangerous.Val = false;
HardLinks.Val = true;
AltStreams.Val = true;
@ -166,53 +167,79 @@ struct CFiTimesCAM
ATime_Defined |
MTime_Defined;
}
bool SetDirTime_to_FS(CFSTR path) const;
#ifdef SUPPORT_LINKS
bool SetLinkFileTime_to_FS(CFSTR path) const;
#endif
};
struct CDirPathTime: public CFiTimesCAM
{
FString Path;
bool SetDirTime() const;
bool SetDirTime_to_FS_2() const { return SetDirTime_to_FS(Path); }
};
#ifdef SUPPORT_LINKS
enum ELinkType
{
k_LinkType_HardLink,
k_LinkType_PureSymLink,
k_LinkType_Junction,
k_LinkType_WSL
// , k_LinkType_CopyLink;
};
struct CLinkInfo
{
// bool isCopyLink;
bool isHardLink;
bool isJunction;
ELinkType LinkType;
bool isRelative;
bool isWSL;
UString linkPath;
// if (isRelative == false), then (LinkPath) is relative to root folder of archive
// if (isRelative == true ), then (LinkPath) is relative to current item
bool isWindowsPath;
UString LinkPath;
bool IsSymLink() const { return !isHardLink; }
bool Is_HardLink() const { return LinkType == k_LinkType_HardLink; }
bool Is_AnySymLink() const { return LinkType != k_LinkType_HardLink; }
bool Is_WSL() const { return LinkType == k_LinkType_WSL; }
CLinkInfo():
// IsCopyLink(false),
isHardLink(false),
isJunction(false),
LinkType(k_LinkType_PureSymLink),
isRelative(false),
isWSL(false)
isWindowsPath(false)
{}
void Clear()
{
// IsCopyLink = false;
isHardLink = false;
isJunction = false;
LinkType = k_LinkType_PureSymLink;
isRelative = false;
isWSL = false;
linkPath.Empty();
isWindowsPath = false;
LinkPath.Empty();
}
bool Parse(const Byte *data, size_t dataSize, bool isLinuxData);
bool Parse_from_WindowsReparseData(const Byte *data, size_t dataSize);
bool Parse_from_LinuxData(const Byte *data, size_t dataSize);
void Normalize_to_RelativeSafe(UStringVector &removePathParts);
private:
void Remove_AbsPathPrefixes();
};
#endif // SUPPORT_LINKS
struct CProcessedFileInfo
{
CArcTime CTime;
CArcTime ATime;
CArcTime MTime;
UInt32 Attrib;
bool Attrib_Defined;
#ifndef _WIN32
struct COwnerInfo
@ -229,8 +256,76 @@ struct COwnerInfo
}
};
COwnerInfo Owner;
COwnerInfo Group;
#endif
void Clear()
{
#ifndef _WIN32
Attrib_Defined = false;
Owner.Clear();
#endif
}
bool IsReparse() const
{
return (Attrib_Defined && (Attrib & FILE_ATTRIBUTE_REPARSE_POINT) != 0);
}
bool IsLinuxSymLink() const
{
return (Attrib_Defined && MY_LIN_S_ISLNK(Attrib >> 16));
}
void SetFromPosixAttrib(UInt32 a)
{
// here we set only part of combined attribute required by SetFileAttrib() call
#ifdef _WIN32
// Windows sets FILE_ATTRIBUTE_NORMAL, if we try to set 0 as attribute.
Attrib = MY_LIN_S_ISDIR(a) ?
FILE_ATTRIBUTE_DIRECTORY :
FILE_ATTRIBUTE_ARCHIVE;
if ((a & 0222) == 0) // (& S_IWUSR) in p7zip
Attrib |= FILE_ATTRIBUTE_READONLY;
// 22.00 : we need type bits for (MY_LIN_S_IFLNK) for IsLinuxSymLink()
a &= MY_LIN_S_IFMT;
if (a == MY_LIN_S_IFLNK)
Attrib |= (a << 16);
#else
Attrib = (a << 16) | FILE_ATTRIBUTE_UNIX_EXTENSION;
#endif
Attrib_Defined = true;
}
};
#ifdef SUPPORT_LINKS
struct CPostLink
{
UInt32 Index_in_Arc;
bool item_IsDir; // _item.IsDir
UString item_Path; // _item.Path;
UStringVector item_PathParts; // _item.PathParts;
CProcessedFileInfo item_FileInfo; // _fi
FString fullProcessedPath_from; // full file path in FS
CLinkInfo LinkInfo;
};
/*
struct CPostLinks
{
void Clear()
{
Links.Clear();
}
};
*/
#endif // SUPPORT_LINKS
class CArchiveExtractCallback Z7_final:
public IArchiveExtractCallback,
@ -278,8 +373,9 @@ public:
private:
const CArc *_arc;
public:
CExtractNtOptions _ntOptions;
private:
bool _encrypted;
bool _isSplit;
bool _curSize_Defined;
@ -287,8 +383,8 @@ private:
bool _isRenamed;
bool _extractMode;
// bool _is_SymLink_in_Data;
bool _is_SymLink_in_Data_Linux; // false = WIN32, true = LINUX
bool _is_SymLink_in_Data_Linux; // false = WIN32, true = LINUX.
// _is_SymLink_in_Data_Linux is detected from Windows/Linux part of attributes of file.
bool _needSetAttrib;
bool _isSymLinkCreated;
bool _itemFailure;
@ -311,7 +407,9 @@ private:
CMyComPtr<ICryptoGetTextPassword> _cryptoGetTextPassword;
FString _dirPathPrefix;
public:
FString _dirPathPrefix_Full;
private:
#ifndef Z7_SFX
@ -323,49 +421,7 @@ private:
CReadArcItem _item;
FString _diskFilePath;
struct CProcessedFileInfo
{
CArcTime CTime;
CArcTime ATime;
CArcTime MTime;
UInt32 Attrib;
bool Attrib_Defined;
#ifndef _WIN32
COwnerInfo Owner;
COwnerInfo Group;
#endif
bool IsReparse() const
{
return (Attrib_Defined && (Attrib & FILE_ATTRIBUTE_REPARSE_POINT) != 0);
}
bool IsLinuxSymLink() const
{
return (Attrib_Defined && MY_LIN_S_ISLNK(Attrib >> 16));
}
void SetFromPosixAttrib(UInt32 a)
{
// here we set only part of combined attribute required by SetFileAttrib() call
#ifdef _WIN32
// Windows sets FILE_ATTRIBUTE_NORMAL, if we try to set 0 as attribute.
Attrib = MY_LIN_S_ISDIR(a) ?
FILE_ATTRIBUTE_DIRECTORY :
FILE_ATTRIBUTE_ARCHIVE;
if ((a & 0222) == 0) // (& S_IWUSR) in p7zip
Attrib |= FILE_ATTRIBUTE_READONLY;
// 22.00 : we need type bits for (MY_LIN_S_IFLNK) for IsLinuxSymLink()
a &= MY_LIN_S_IFMT;
if (a == MY_LIN_S_IFLNK)
Attrib |= (a << 16);
#else
Attrib = (a << 16) | FILE_ATTRIBUTE_UNIX_EXTENSION;
#endif
Attrib_Defined = true;
}
} _fi;
CProcessedFileInfo _fi;
UInt64 _position;
UInt64 _curSize;
@ -407,19 +463,21 @@ private:
// CObjectVector<NWindows::NFile::NDir::CDelayedSymLink> _delayedSymLinks;
#endif
void CreateComplexDirectory(const UStringVector &dirPathParts, FString &fullPath);
void CreateComplexDirectory(
const UStringVector &dirPathParts, bool isFinal, FString &fullPath);
HRESULT GetTime(UInt32 index, PROPID propID, CArcTime &ft);
HRESULT GetUnpackSize();
FString Hash_GetFullFilePath();
void SetAttrib();
void SetAttrib() const;
public:
HRESULT SendMessageError(const char *message, const FString &path);
HRESULT SendMessageError_with_Error(HRESULT errorCode, const char *message, const FString &path);
HRESULT SendMessageError_with_LastError(const char *message, const FString &path);
HRESULT SendMessageError2(HRESULT errorCode, const char *message, const FString &path1, const FString &path2);
HRESULT SendMessageError(const char *message, const FString &path) const;
HRESULT SendMessageError_with_Error(HRESULT errorCode, const char *message, const FString &path) const;
HRESULT SendMessageError_with_LastError(const char *message, const FString &path) const;
HRESULT SendMessageError2(HRESULT errorCode, const char *message, const FString &path1, const FString &path2) const;
HRESULT SendMessageError2_with_LastError(const char *message, const FString &path1, const FString &path2) const;
#if defined(_WIN32) && !defined(UNDER_CE) && !defined(Z7_SFX)
NExtract::NZoneIdMode::EEnum ZoneMode;
@ -482,23 +540,32 @@ public:
UInt64 packSize);
#ifdef SUPPORT_LINKS
#ifdef SUPPORT_LINKS
private:
CHardLinks _hardLinks;
CObjectVector<CPostLink> _postLinks;
CLinkInfo _link;
// const void *NtReparse_Data;
// UInt32 NtReparse_Size;
// FString _copyFile_Path;
// HRESULT MyCopyFile(ISequentialOutStream *outStream);
HRESULT Link(const FString &fullProcessedPath);
HRESULT ReadLink();
HRESULT SetLink(
const FString &fullProcessedPath_from,
const CLinkInfo &linkInfo,
bool &linkWasSet);
HRESULT SetPostLinks() const;
public:
// call PrepareHardLinks() after Init()
HRESULT CreateHardLink2(const FString &newFilePath,
const FString &existFilePath, bool &link_was_Created) const;
HRESULT DeleteLinkFileAlways_or_RemoveEmptyDir(const FString &path, bool checkThatFileIsEmpty) const;
HRESULT PrepareHardLinks(const CRecordVector<UInt32> *realIndices); // NULL means all items
#endif
#endif
private:
#ifdef SUPPORT_ALT_STREAMS
CObjectVector<CIndexToPathPair> _renamedFiles;
@ -506,6 +573,7 @@ public:
// call it after Init()
public:
#ifndef Z7_SFX
void SetBaseParentFolderIndex(UInt32 indexInArc)
{
@ -527,7 +595,6 @@ private:
HRESULT Read_fi_Props();
void CorrectPathParts();
void GetFiTimesCAM(CFiTimesCAM &pt);
void CreateFolders();
HRESULT CheckExistFile(FString &fullProcessedPath, bool &needExit);
@ -536,18 +603,8 @@ private:
HRESULT CloseFile();
HRESULT CloseReparseAndFile();
HRESULT CloseReparseAndFile2();
HRESULT SetDirsTimes();
const void *NtReparse_Data;
UInt32 NtReparse_Size;
#ifdef SUPPORT_LINKS
HRESULT SetFromLinkPath(
const FString &fullProcessedPath,
const CLinkInfo &linkInfo,
bool &linkWasSet);
#endif
HRESULT SetSecurityInfo(UInt32 indexInArc, const FString &path) const;
};

Some files were not shown because too many files have changed in this diff Show more