From a51c4d87d72504721045628b3ad0b0bb94af9e53 Mon Sep 17 00:00:00 2001 From: cadmic Date: Thu, 25 Jul 2024 15:15:15 -0700 Subject: [PATCH] Fix bug where wrong data is emitted for gzip fixed blocks (#24) * Fix bug where wrong data is emitted for gzip fixed blocks * Update test data * Add CHANGELOG entry --- CHANGELOG.md | 5 +++++ lib/src/gzip.rs | 29 +++++++++++++++++--------- test_data/dirt.png.gzip-6-small-mem | Bin 270362 -> 270362 bytes test_data/dirt.png.gzip-9 | Bin 270242 -> 270242 bytes test_data/dirt.png.gzip-9-small-mem | Bin 270362 -> 270362 bytes test_data/ground.png.gzip-6-small-mem | Bin 237537 -> 237536 bytes test_data/ground.png.gzip-9 | Bin 237421 -> 237420 bytes test_data/ground.png.gzip-9-small-mem | Bin 237537 -> 237536 bytes test_data/stones.png.gzip-6-small-mem | Bin 270847 -> 270848 bytes test_data/stones.png.gzip-9 | Bin 270721 -> 270722 bytes test_data/stones.png.gzip-9-small-mem | Bin 270847 -> 270848 bytes test_data/tile.png.gzip-6-small-mem | Bin 226852 -> 226852 bytes test_data/tile.png.gzip-9 | Bin 226658 -> 226658 bytes test_data/tile.png.gzip-9-small-mem | Bin 226852 -> 226852 bytes 14 files changed, 24 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8019efa..edc22fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- Fix a bug where the gzip compressor may output incorrect data when emitting + "fixed blocks" (which are emitted when compressing high-entropy data). + ## [0.5.0] - 2024-06-04 ### Added diff --git a/lib/src/gzip.rs b/lib/src/gzip.rs index 628f1ef..ac34a09 100644 --- a/lib/src/gzip.rs +++ b/lib/src/gzip.rs @@ -1022,16 +1022,34 @@ pub fn compress(bytes: &[u8], level: usize, small_mem: bool) -> Result pos += prev_match_len - 1; lookahead -= prev_match_len - 1; + if should_flush { + if pos >= block_length { + writer.flush_block(&mut output, Some(&window[pos - block_length..pos]), false); + } else { + writer.flush_block(&mut output, None, false); + } + block_length = 0; + } + has_prev_char = false; prev_match_len = MIN_MATCH - 1; prev_match_dist = 0; } else { - // Remember current match and emit previous character as literal if it exists + // Emit previous character as literal (if it exists) and remember current match if has_prev_char { writer.add_literal(window[pos - 1]); should_flush = writer.should_flush_block(block_length); } + if should_flush { + if pos >= block_length { + writer.flush_block(&mut output, Some(&window[pos - block_length..pos]), false); + } else { + writer.flush_block(&mut output, None, false); + } + block_length = 0; + } + block_length += 1; pos += 1; lookahead -= 1; @@ -1041,15 +1059,6 @@ pub fn compress(bytes: &[u8], level: usize, small_mem: bool) -> Result prev_match_dist = pos - 1 - best_pos; } - if should_flush { - if pos >= block_length { - writer.flush_block(&mut output, Some(&window[pos - block_length..pos]), false); - } else { - writer.flush_block(&mut output, None, false); - } - block_length = 0; - } - // Refill window if lookahead < MIN_LOOKAHEAD && !eof && pos >= WINDOW_SIZE + MAX_DIST { window.copy_within(WINDOW_SIZE..2 * WINDOW_SIZE, 0); diff --git a/test_data/dirt.png.gzip-6-small-mem b/test_data/dirt.png.gzip-6-small-mem index 96e67dce0041f4d8c2114d34dd28e278b4b318dc..9c548aae213e5b03e785178926b8e7945691b1d9 100644 GIT binary patch delta 424 zcmXYsyDx)b6o>o1Z&@tWHmbgs(~H)IV8K9;;158;pkZ)X1dGHVSQrh4Z)%X3Cftc@*?k8BORK9 zkfL8}ZZyE$4+qp|nM-}lldwsK^%0+7u7o!VTZ1Pa<^ViV+?vk-a}~T$0!<;jEFO5L zQalWZ3YHp3QyLSUqMD@|GIWbSG2v&4!aWV3Z&^fGLhwj6*t;cyEPi;VDV#YKK9**< zqG{ZC7ELTVU;?kC)B*aLMf?s*{f!$wX&B48q>IG`U$l?mG3jJ6AV;Uzz9Oqw>fny@ zII=CHEM9n}Yg{{%mBk%}yV=4{Ej-Wy9w`;(Nd%H~i60?V%i@D9&7!+o87#WOI<|}} vT|plm!q}?PM|D677j_B(jc`NtxP7h~c%qy7Fqu=_R5*=%&X1zR?lt}a;=q!n delta 424 zcmXAkzfV(P6o$|Def2CZ{0eQkxA3-cw9pECt;8@1zy%D@Mc z15Z*0WtCSO#s^PP54Kg_m(2rkF$Qc~8v$l-W#lqvBb(>cgg+{G`)wom6E)(mN`AsN zfH%lPRi$scEl8nsa&Z78P))Qfqe| z;Me4%qSExldEj?(!owFXN*IX&;|neV9LIpt$_?Ni-N&xV@}}zs`*G=E)kUQlQ}10? J#oS4xy=(G)lBECu diff --git a/test_data/dirt.png.gzip-9 b/test_data/dirt.png.gzip-9 index 7ac995c087e8983250d4703dd53fe33eebca2f9e..2283bd412c9f86028a62121f16bfeb2d0afa707b 100644 GIT binary patch delta 117 zcmZ3qUtrOG0TzblhMJ8mEe&i8?;8Z_Pi^jNcn4$`HuTl|Y;R&@EC6!mH6+&0-2S7H zaW0UP&@dm!na#u$59Aaz^wxjg*3480WKV3!tDn2QoSAt7kkj8#P_M9^w}rW{fsOH( P6zkJ{+ZVF1FiHae2I?*t delta 117 zcmZ3qUtrOG0TzaqhU$$hEe&j^7~VGs)Nk%bLW@ VF!wdE?PL5Y#rAajLKYTAX#fs~E*Jm+ diff --git a/test_data/dirt.png.gzip-9-small-mem b/test_data/dirt.png.gzip-9-small-mem index 96e67dce0041f4d8c2114d34dd28e278b4b318dc..9c548aae213e5b03e785178926b8e7945691b1d9 100644 GIT binary patch delta 424 zcmXYsyDx)b6o>o1Z&@tWHmbgs(~H)IV8K9;;158;pkZ)X1dGHVSQrh4Z)%X3Cftc@*?k8BORK9 zkfL8}ZZyE$4+qp|nM-}lldwsK^%0+7u7o!VTZ1Pa<^ViV+?vk-a}~T$0!<;jEFO5L zQalWZ3YHp3QyLSUqMD@|GIWbSG2v&4!aWV3Z&^fGLhwj6*t;cyEPi;VDV#YKK9**< zqG{ZC7ELTVU;?kC)B*aLMf?s*{f!$wX&B48q>IG`U$l?mG3jJ6AV;Uzz9Oqw>fny@ zII=CHEM9n}Yg{{%mBk%}yV=4{Ej-Wy9w`;(Nd%H~i60?V%i@D9&7!+o87#WOI<|}} vT|plm!q}?PM|D677j_B(jc`NtxP7h~c%qy7Fqu=_R5*=%&X1zR?lt}a;=q!n delta 424 zcmXAkzfV(P6o$|Def2CZ{0eQkxA3-cw9pECt;8@1zy%D@Mc z15Z*0WtCSO#s^PP54Kg_m(2rkF$Qc~8v$l-W#lqvBb(>cgg+{G`)wom6E)(mN`AsN zfH%lPRi$scEl8nsa&Z78P))Qfqe| z;Me4%qSExldEj?(!owFXN*IX&;|neV9LIpt$_?Ni-N&xV@}}zs`*G=E)kUQlQ}10? J#oS4xy=(G)lBECu diff --git a/test_data/ground.png.gzip-6-small-mem b/test_data/ground.png.gzip-6-small-mem index 30b61cb55aa9cd8370327cd7332d4686581c7a7b..8c947295c89b24d45f62ee26ae61f000e071a78e 100644 GIT binary patch delta 364 zcmXZWKP*F06vy%2`>L-AeT`DuD$Xm4gvlflG!m&qj7HH~ZC!NoZ?~8X618+PGD>?B+eDv_mi8G`zsf$Qo(XM#-W&VPJqkdhB~Fx44B*|eA0@V zpLB5tphC+kx!BGfgeS_Y)wKZkFr3qv+T9!CZi8pKRA+~N?k*@(Tvbb*-2HGx2j~$Z z%FrLwanu(22Ys=H(=nk>THu8iFp&^Gh99aF#PqsoWq6@RA>7Ug>mR3>E&ibiKIjYc zWzonl8aX@>QbUi-;;k+9N*}yZ1;1mmnGuEuUf@g04u)Rvh9etNF90BtIJYmm8G~>^ zvzR)P1B_m{Avb33We+qsY`7ylmWIs_A$X)8tVN6Tp GcEZ0H2z{ym delta 366 zcmXBOJ1m1?6vpxMzF(b6P*K-b@w6xsCX*NtiAy3zlZeY|voN{s7L!4u7L!4~;S2_s zO+q9Fmk37*jfSWg2$A@-<9~Sa{OjKYEK0ehQgdaRhetfAr(#_y$ zszOTL$?;0?E`{)dmFz2~OH-@LNvMzGOXtB|ue%H*Fnw zh+1&2GIMB~!THpHYn8;ktw)|x6t{9@ohbO1!gy54#GDWq6u`Smaj#PZ_EQ{JDzPc2 M5oT2Smz}l9A0zjE1^@s6 diff --git a/test_data/ground.png.gzip-9 b/test_data/ground.png.gzip-9 index bea320fd774d3d8c298264d7dc470abe8789da05..9a97b2c6c9ff5ce32351db2cd66f42656edadc40 100644 GIT binary patch delta 73 zcmaF6kMGSsJ{E?IhUptwG8;DMH8eIbUf=qjF{*)$!Kq&cfR?cYX@CKIvu%KZfVJ-dT7U)(078Jhf45J80YQKUQUF$f hseiYY0RmNk0VlWXfdY$ww|4>qaex5;x9EZcgdB-29Yz2E diff --git a/test_data/ground.png.gzip-9-small-mem b/test_data/ground.png.gzip-9-small-mem index 30b61cb55aa9cd8370327cd7332d4686581c7a7b..8c947295c89b24d45f62ee26ae61f000e071a78e 100644 GIT binary patch delta 364 zcmXZWKP*F06vy%2`>L-AeT`DuD$Xm4gvlflG!m&qj7HH~ZC!NoZ?~8X618+PGD>?B+eDv_mi8G`zsf$Qo(XM#-W&VPJqkdhB~Fx44B*|eA0@V zpLB5tphC+kx!BGfgeS_Y)wKZkFr3qv+T9!CZi8pKRA+~N?k*@(Tvbb*-2HGx2j~$Z z%FrLwanu(22Ys=H(=nk>THu8iFp&^Gh99aF#PqsoWq6@RA>7Ug>mR3>E&ibiKIjYc zWzonl8aX@>QbUi-;;k+9N*}yZ1;1mmnGuEuUf@g04u)Rvh9etNF90BtIJYmm8G~>^ zvzR)P1B_m{Avb33We+qsY`7ylmWIs_A$X)8tVN6Tp GcEZ0H2z{ym delta 366 zcmXBOJ1m1?6vpxMzF(b6P*K-b@w6xsCX*NtiAy3zlZeY|voN{s7L!4u7L!4~;S2_s zO+q9Fmk37*jfSWg2$A@-<9~Sa{OjKYEK0ehQgdaRhetfAr(#_y$ zszOTL$?;0?E`{)dmFz2~OH-@LNvMzGOXtB|ue%H*Fnw zh+1&2GIMB~!THpHYn8;ktw)|x6t{9@ohbO1!gy54#GDWq6u`Smaj#PZ_EQ{JDzPc2 M5oT2Smz}l9A0zjE1^@s6 diff --git a/test_data/stones.png.gzip-6-small-mem b/test_data/stones.png.gzip-6-small-mem index d5f4d4369fde6f6a1fec9e57c44ee66f777f4e59..513851b454e2db3f5888b4ba3a61397c2762ed5b 100644 GIT binary patch delta 411 zcmXYtJ1oOd6o=h=s}Ftnw-S_UPQ4bji%E?JVJA_-X0r&95{bcXFo)I7+G!w4NcAsB;_J_vPl?^ZT9Rz7abxw)#LEntfn^wX6!B$Zw8y`IrU#(#rbG z5ThI3C}+;cT+9aK>C{|Wtz-_s9krQ>-FoIa$WYomzVS2HLY9)-kD`Xz0zZ^Mdq^mj zW=K&7dLyEV#Rt!{h@Da4Wzj1Jv42+R9WHn$14rV*$>M=m8o-G?;r^eQ#D#QeMyDOc z6(M!p2DqhV+zH75iygkG2QMSi#!?Oiy2eaYT3DR$LFbsCl@%=dgAFXkOCQ+an~tz1 zDXsj{3RjfHrd!#eR#~YP{r5(=b9;drin8gb@a^mK&(wEc~rtn}+ n=?Z#J7%$e84yq66z~`jW7t}#hDDM;p6%WDcbO9@s0o2Dm5PP7*PB1!ob&sg`9t^Ldv|{X-ZQx11hwEoy{W-C zc#RDH>elNqaD^Iit={5%1NalUxKwXtqZ!;s9XL~O_gy=(Ccr=)h=NaQ~xYU53Dla!p6#RmkaHTSH zU>d=*wailCJ~J86%DQoU+*RsCQAN6(4)7DI!*7-RkP`-fqX)QCDUCZJ@E3~Wmr6C` z)PuK!g16cG{09o-R;4ZHFt|*;$g8A^P7nBFU?xu;>#H?UJ#JK{AK3`FME&@zGB<3~ z;3&mVQQ4cY{*kY021g6lH%QkwzhJ7J^WefqbZ5j^OKijU}$Y>4ZI5h09x7nV- z$Yc-X_%&>-U$A{`Ba<(XliDz=zIz)Ja|)1c)o`?a-}bB~X3GXP#*1=#58rKH$;5I) F4gioZDFFZg delta 104 zcmV-u0GI!Q!w`YP5U_H9vvYt|fCe!DRe-90wax)lfClCOJb=T0w7h zf46A?0zZHTpa4#Qv46L&fdWl{28#e?fSP}|0RjVLfCjz*I)KH0w`+m}Ie-T40ni>V K;kT*+1lS&SuPAo_ diff --git a/test_data/stones.png.gzip-9-small-mem b/test_data/stones.png.gzip-9-small-mem index d5f4d4369fde6f6a1fec9e57c44ee66f777f4e59..513851b454e2db3f5888b4ba3a61397c2762ed5b 100644 GIT binary patch delta 411 zcmXYtJ1oOd6o=h=s}Ftnw-S_UPQ4bji%E?JVJA_-X0r&95{bcXFo)I7+G!w4NcAsB;_J_vPl?^ZT9Rz7abxw)#LEntfn^wX6!B$Zw8y`IrU#(#rbG z5ThI3C}+;cT+9aK>C{|Wtz-_s9krQ>-FoIa$WYomzVS2HLY9)-kD`Xz0zZ^Mdq^mj zW=K&7dLyEV#Rt!{h@Da4Wzj1Jv42+R9WHn$14rV*$>M=m8o-G?;r^eQ#D#QeMyDOc z6(M!p2DqhV+zH75iygkG2QMSi#!?Oiy2eaYT3DR$LFbsCl@%=dgAFXkOCQ+an~tz1 zDXsj{3RjfHrd!#eR#~YP{r5(=b9;drin8gb@a^mK&(wEc~rtn}+ n=?Z#J7%$e84yq66z~`jW7t}#hDDM;p6%WDcbO9@s0o2Dm5PP7*PB1!ob&sg`9t^Ldv|{X-ZQx11hwEoy{W-C zc#RDH>elNqaD^Iit={5%1NalUxKwXtqZ!;s9XL~O_gy=(Ccr=)h=NaQ~xYU53Dla!p6#RmkaHTSH zU>d=*wailCJ~J86%DQoU+*RsCQAN6(4)7DI!*7-RkP`-fqX)QCDUCZJ@E3~Wmr6C` z)PuK!g16cG{09o-R;4ZHFt|*;$g8A^P7nBFU?xu;>#H?UJ#JK{AK3`FME&@zGB<3~ z;3&mVQQ4cY{*kY021g6lH%QkwzhIDx$2w_4{R9J&c9Hb}*g9x{l&I@hN!#SLfU#aaawY{b*4tM8f0^F`CsE|;TkYbL& zGj*$cD$KkId74q>>;`iH-f2rEcKysTcqCiRpG29Lphyi>xOSMM@I<%jr#Zqr0XI}f zONt~*2wrIcb0N{s;=nPr@E|P~S?1t^zOdtnS(Y%o&^BiF#4rm0Q2^IZMUbTrnq=cq zRrIoq!UILH@+M*|de0|z45{}FL4%4IAC%UATNSg@Qn%@8Yk0UUgMj6Ii6A_9m~+4>Xg#h$`~AzFnzz@m)Nsnczl0j CBYM98 delta 347 zcmXAly)T1d6vglJymAN9bTZK3D5(TdUtMZYjWh;EgQ-=*WUxpWe1t9%t3?f>{s7)^ zBatS8L`0A>bdeh&7)V5n2CsHMIlps$xes6N!&gBOTcp+OqdQ@6ktUE;cl|UCzN9ej)ct^fucN_^cdD4O%ZYTW;H&v<)!Ta=2y%GP7rs z;C1T3wMyZ_c7spK6DnV=H{qh>MWXC@Vwy@vWjW+{Pva#^@}6)!O~=OZ|9(l$v{U8( E18<#r$N&HU diff --git a/test_data/tile.png.gzip-9 b/test_data/tile.png.gzip-9 index 3b7c4837e6321f766707bbe7897c5485e5c5403b..1dc1d69a12f838f31e155764e676322a69a9a1cd 100644 GIT binary patch delta 49 zcmV-10M7s7>J8%R4X_u10b{clf!~0)_W=P7fdO;31cL#bfCd1zfJuKOx26LEwSc!= HgaQWf9aI!$ delta 49 zcmV-10M7s7>J8%R4X_u1vlxNjfC0w0_W=P7fwu*N0i1vaBmlO6Nq@Je0|K>x0VTIw HgaQWfFjN$M diff --git a/test_data/tile.png.gzip-9-small-mem b/test_data/tile.png.gzip-9-small-mem index dc03913ff94d3fa64dd6fbfab89c430d2160fe7b..e48aac519de4640a7e9bdb5888b5ee57a1bcf522 100644 GIT binary patch delta 347 zcmXYty-Px26o-A^dvQp_sUf92S==Bct#q*=laPiQ8=9`&!ls6npoTsu7ooK!8zTDy z>IDx$2w_4{R9J&c9Hb}*g9x{l&I@hN!#SLfU#aaawY{b*4tM8f0^F`CsE|;TkYbL& zGj*$cD$KkId74q>>;`iH-f2rEcKysTcqCiRpG29Lphyi>xOSMM@I<%jr#Zqr0XI}f zONt~*2wrIcb0N{s;=nPr@E|P~S?1t^zOdtnS(Y%o&^BiF#4rm0Q2^IZMUbTrnq=cq zRrIoq!UILH@+M*|de0|z45{}FL4%4IAC%UATNSg@Qn%@8Yk0UUgMj6Ii6A_9m~+4>Xg#h$`~AzFnzz@m)Nsnczl0j CBYM98 delta 347 zcmXAly)T1d6vglJymAN9bTZK3D5(TdUtMZYjWh;EgQ-=*WUxpWe1t9%t3?f>{s7)^ zBatS8L`0A>bdeh&7)V5n2CsHMIlps$xes6N!&gBOTcp+OqdQ@6ktUE;cl|UCzN9ej)ct^fucN_^cdD4O%ZYTW;H&v<)!Ta=2y%GP7rs z;C1T3wMyZ_c7spK6DnV=H{qh>MWXC@Vwy@vWjW+{Pva#^@}6)!O~=OZ|9(l$v{U8( E18<#r$N&HU