From faf013ec84051b92ae0f420a658b8d35bb7bb000 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Thu, 18 Nov 2021 12:15:22 -0800 Subject: [PATCH] Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s). --- Makefile | 2 +- cmd/bucket-replication.go | 2 +- cmd/erasure-metadata.go | 7 +- cmd/metacache-entries.go | 8 +- cmd/testdata/xl.meta-v1.2.zst | Bin 0 -> 58055 bytes cmd/xl-storage-format-utils.go | 87 +- cmd/xl-storage-format-utils_test.go | 93 + cmd/xl-storage-format-v1.go | 23 + cmd/xl-storage-format-v2-legacy.go | 89 + cmd/xl-storage-format-v2.go | 2404 ++++++++++++++------------ cmd/xl-storage-format-v2_gen.go | 930 +++++++--- cmd/xl-storage-format-v2_gen_test.go | 145 +- cmd/xl-storage-format-v2_string.go | 27 + cmd/xl-storage-format-v2_test.go | 118 +- cmd/xl-storage-format_test.go | 222 +++ cmd/xl-storage-free-version.go | 29 +- cmd/xl-storage-free-version_test.go | 47 +- cmd/xl-storage-meta-inline.go | 408 +++++ cmd/xl-storage.go | 8 +- docs/debugging/xl-meta/main.go | 93 + go.mod | 1 + go.sum | 2 + 22 files changed, 3282 insertions(+), 1463 deletions(-) create mode 100644 cmd/testdata/xl.meta-v1.2.zst create mode 100644 cmd/xl-storage-format-utils_test.go create mode 100644 cmd/xl-storage-format-v2-legacy.go create mode 100644 cmd/xl-storage-format-v2_string.go create mode 100644 cmd/xl-storage-meta-inline.go diff --git a/Makefile b/Makefile index 859e04093..4cdf1c074 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ help: ## print this help getdeps: ## fetch necessary dependencies @mkdir -p ${GOPATH}/bin @echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.43.0 - @echo "Installing msgp" && go install -v github.com/tinylib/msgp@latest + @echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.1.7-0.20211026165309-e818a1881b0e @echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer@latest crosscompile: ## cross compile minio diff --git a/cmd/bucket-replication.go b/cmd/bucket-replication.go index f49271a96..57bb334ed 100644 --- a/cmd/bucket-replication.go +++ b/cmd/bucket-replication.go @@ -724,7 +724,7 @@ const ( // matches k1 with all keys, returns 'true' if one of them matches func equals(k1 string, keys ...string) bool { for _, k2 := range keys { - if strings.EqualFold(strings.ToLower(k1), strings.ToLower(k2)) { + if strings.EqualFold(k1, k2) { return true } } diff --git a/cmd/erasure-metadata.go b/cmd/erasure-metadata.go index a1a1da380..0a381aecc 100644 --- a/cmd/erasure-metadata.go +++ b/cmd/erasure-metadata.go @@ -476,9 +476,7 @@ func GetInternalReplicationState(m map[string][]byte) ReplicationState { // getInternalReplicationState fetches internal replication state from the map m func getInternalReplicationState(m map[string]string) ReplicationState { - d := ReplicationState{ - ResetStatusesMap: make(map[string]string), - } + d := ReplicationState{} for k, v := range m { switch { case equals(k, ReservedMetadataPrefixLower+ReplicationTimestamp): @@ -497,6 +495,9 @@ func getInternalReplicationState(m map[string]string) ReplicationState { d.PurgeTargets = versionPurgeStatusesMap(v) case strings.HasPrefix(k, ReservedMetadataPrefixLower+ReplicationReset): arn := strings.TrimPrefix(k, fmt.Sprintf("%s-", ReservedMetadataPrefixLower+ReplicationReset)) + if d.ResetStatusesMap == nil { + d.ResetStatusesMap = make(map[string]string, 1) + } d.ResetStatusesMap[arn] = v } } diff --git a/cmd/metacache-entries.go b/cmd/metacache-entries.go index 3df3a0ac6..7d2b353af 100644 --- a/cmd/metacache-entries.go +++ b/cmd/metacache-entries.go @@ -148,11 +148,15 @@ func (e *metaCacheEntry) isLatestDeletemarker() bool { if !isXL2V1Format(e.metadata) { return false } + if meta, _ := isIndexedMetaV2(e.metadata); meta != nil { + return meta.IsLatestDeleteMarker() + } + // Fall back... var xlMeta xlMetaV2 - if err := xlMeta.Load(e.metadata); err != nil || len(xlMeta.Versions) == 0 { + if err := xlMeta.Load(e.metadata); err != nil || len(xlMeta.versions) == 0 { return true } - return xlMeta.Versions[len(xlMeta.Versions)-1].Type == DeleteType + return xlMeta.versions[0].header.Type == DeleteType } // fileInfo returns the decoded metadata. diff --git a/cmd/testdata/xl.meta-v1.2.zst b/cmd/testdata/xl.meta-v1.2.zst new file mode 100644 index 0000000000000000000000000000000000000000..5eb4c5da9a44fb90b2504bbbf644df4a57d27537 GIT binary patch literal 58055 zcmV(*K;FM7wJ-euXe>qp&Tz(f7AO!Hx;5y^79)eJpaVmk`vU_xGxGs0+3u3Ebs!a&Qf+8Yvvha3hUII?)|gSR$*5Q5PTZAh9rR242k)pCZv<18f%;kbuwRvAivz@h1U+3<3dfTIo2XOGf=ZsJWAP_j+L0Y9&qwdWVyDC-^psK~`M4?4h z;x0fr;1>lRvVUp4&y`j$slOF-F1ghC-mfXnoP=@Oz_&puLiF~eSRpTRR|lISw?@%5 zdZq+Btv!}#t#T7hD|8C^7z(Fkn#`g2a7+|+hZ@I_w-X zAf~~x!Im&Gu;vG(+|P&jnq#t}*uez5X26xj*mv)*t_k zm||ab68LU7(@_W2l}d|l(tqe@%1K1xNHE@tB(MS`;80GrX3FW38$~t$K z-}dUgzf$*EDAohM0e+n-L8h6XOTG>U3q`u@vmjIkz?Gy@FFgssrMcxpgOUKQ#hTkr ztL3|LE4}62bKkGuc>6*u#x&sb6lYYE2zL5eZEYV_EOW7Y>4k7z-7re4ks@9)y$9X*{BQu_VBy+Tx7k{~?e9O{>8rnf|NOlKTM3kf zVj^ph85^z)VVLVQ=m?xIYY>!z1*=Yi3XVp>@l%<$q;Y%K5<}bPoRQ)!_x4iPz9ann z!%Io}yv&K90SS%;i2yn_ScPsZ`-JJU*n_K4i4N{ngjZ~`5XdXNuDZ)=^RFD`pc#A?LlEI?@mWRsEH zW3&_QiGSVlTO6gYvRk>Mwb%yx1k6>4V0}ErodAMM}7z@5QYwIAW?x% zCABCi=HP;Rg|(OT=SVk&H%dNp?4R=d^PS$yH^g+QNrHNH4_Xa^1_vlqoZbd6L?W3Y z4H7}A(?&OF*#=EE5_M`TroBf=eg5-fs3r8B-pVnw#$2&W8%o*Ad+&twLrSCNSZ^vJtucQZCGIlG)+;TGDy6x>MGgUK zz9v<;gd_n;JvL|vb%(9YiZ&!j7`S?)z)yHT=DEweVYgmiDeKl1@7p)7z9KCqaI)B5q9|qYqCo%#dV2gfvHTn6BLY@ms;+jFNcy=9Smpap!&Ot^3q=*K8-w z93tau<>J#GDnmrrNkofnSTIHbO!-mk!IfOy>N_v+4B(^N!Q#UnGn_K>+B4)B$4mQ- z6yq+x%%4h#vdY2}XL1lUJw2V^0MM|)Q!!MqRN-hqDFK}uWq1vk@l%1r4)61x9Yad7 zuJ+%GcZC>29=rZs4@rhV6=ECu_^^R$3;}j`Tpr1wL$OKO~8=!n&ZD0m0Z90_Ym(T2t#5qcOUq zz>2G&h>@Z>Iz@o>O#pE}j96pZHKh7Z$mP^tcE9C?pVDO&6q+ulhsy(GWMt408o>$D zixII{P|-v8K$00XZaUcDIrD~7*78RG^`B5$JEx@a?kcbCRO{>Q0vUL9+Eq#cN(Rh| zMF;|&BwWT52N_&8j2gv2UIIjN+(@Ja6ZRm`wwlA-{p{1@3@M-A%k90V+G;x4xXLtU z43G|(kdzv13HJwBnQb@bHHn6PVG6{)`&Yl?5K zJ$89|wO{$4`Bp!AK2?L_nKXYJP;U zv{pAphp9LriZm<`yN+<8v4)QuBy0D|&+psrzm&1x+vkNp$LlNK8T0JtjcqEL?6q1n z=8Lf*c{k>&KvM;$??juD0#NlkK=zceMS50hmzh6VN3Y|C_EN4ty_#0bxvY{ynK|Fm z>`wwD^y*0RqtyqQ8nZh(y(+ai(v$^Df;mv&WN@ovgX1H}0sG55>-G_P-X)H7V+y_A zbWU3-r6a2JS&(7Pr7&G!G^%dM;vfM4f$X!8zzaq~R zZA)9Rhp}@%KZT!vn(?l7^W5*8jqt>(BWY#PWFpH8LU43>lu1goz)eRpw?WYIoI!-* zq5+BaR>u~dc*cLHy!ze?r@l4X?k&_>?hDB{TZ<#79le*6N`^Lx9x!k#V}|YxGCV`8 z+TguW7H7dPytX67^ajnr$P?Dqr~~x%(%UV~8AkhizP@|>d5^cpx@-MWQ)`jN0c?)n zqD5^t`UsH6FV7}YqNFA-U&xqwd+`QLuMSFud(MnKo|@wOyN#Da-Tk&6e@JDeWg^hn z)z#P%1|D)mgzl6cP#xs3*k!590Ss}uELA)cRU)w|kU>jZy@xr&c&GQA@>unk9((?` z)ZEafG9sYBS%Uxq94~MVY?Q)~CYhF9r+*?;Dh!@EVt9TQHPUordNbzodTs62eRdvk zx4FW~Gl#QeOp6*vDer{eB}8chH~_%QDyV>v4GJ*O1Iyk6I8?ysAQe#%>BRI_{qNTo zUr)chHO~$u=XYPpE0GseN)!O3C_6fpjRQzs{R zwJ%hjDR6nOM2x6}>0XIv{}=wKEzVMAj3MkZ=8G}hajaY+RH#b;B8Jg4veNwKqNrk^ zfB+mUT>!c$K&io-AvQsTK!EA?LvAUZyYjzf<}&+FBgeBx-Z6(RC8~r#WM&!LuR>(V zuwj^^%McLYL4pQlrcDABdimSV_7Y z9b|3{YUYN_iG&7)JmC1mMhpWr3P6(4b_CARiXGo#+Nqb=TWev={BFDH%^H3RE$xvw zxp@+VPNf>o)Z`N5>~0YQFro>%<-kV+k1+J8GRcgbo=tk9Tr!2J=Uv(f{gzzc{Ac&Q zOHaS_65_~(Q(Y5El59od$)Y7#NI=MQqa^?xL^I@^@K9weQUo1y{&nwY ztEBzv9p#7HPe?D+I$s}t2acpg>Tn_qmPDrP1Qo);ir~mp8qE>8dga}kdvW)rP!(m+ zTh1wm`$Nia)_ZqfFNb>XjG>2Bk$N*BBO>tflJwr-esEIbl6?`8l;(o1JtQU_OOt1O zNMU*WyfR{Vb-rIi-!-&yXWMzDve#;H)Yx(U8o7}B@d}ZMxCCyTO7qudTDXCSB-N!l zb!|LCJ^Opc%EEhh)^g5S@%PnZ%`e1S{^~2YF;fIgMc%+c;-r>9fg~ApaXmB@AnO2B zZYSU}6)?gOrny0LH>B)hKwrx@?)_)GG5o*YeQEzO`mS&HT0~WJmzWl_Em= zP_?+VNZ!G?MX8E;v}(>RkEo3fG+yR(Ey7}F1<>W3$`~`|ebz{!hH%sQeU$LRYqg3g zQG6H+G2zO^RYp?+bD-vMy)Xs05V2#UN*G;ip4yPo^(xPm_Fw94w()X#f5qAMS#j?C zfBzHs5Gsc-V{xjiX{4;q0d%m^d_a{bkQhfQjwR`PtE50Kj-6j>iKfJuq1O^p9BbcN z(hTAEdUNmj?ASZt;crNYQ~?+S!_=fdwNzcEsBp4k8HYA}j#&Xyr$ieSK0cLprjPgU zJ7#>nj?hZUE3FjHnLV|-qiMr{2Wd^N1r~~IfHXkdePiGiE?An7EHk0x*p9tvcWe`q zJ0^lmUcJ6sOPy=Zm)5^ypHSzi|Mh=^QlqC1#H~Aj95AbN2yQKp52A&M6P^T25qiYz z)See~M+*2G8aSC;ZlAA}-~NlS=5%W8>;6&ql7(R|)r354uUrxG(}oKKB1F&j3j|-@ zV6};6a>W^7j1mF4vgE#Lo|8`+C)Hd=pZ(0ZYTRX|7Ulxx2kt?HI2w!kSRmvEj*l9& zAJO6(HJ~pl*_fLXC+A3lq7-17$+>IBJ4!#frxX6~tL56$OSSIMqDnTy?ZD>q!zjP7 zQ(`U=lmxmKRe5qM2p$3Ag1Js4$V;M&Wh<=Lf3cAtd#+Vv?E2^An_KndmsX9t~^i@{n>;F(-1_#S{)$Z!%YG+mm-FdHWn&u z&7quD>&h*}aaJvLmm5!tVVzjd7+jT55V5!Cz^h4qaJ7`os)Q%d2NQf&Il;4q4}&xg zLW1n#C8BO#J;u1pj^TtmPxxuKbn-3nClzA>dCQcAyI0RJ5sKypViIP>C{meUn0U?k z1>;V$IS>945DK|yO5saK_JkH1S~%sk(wHxqDBi;kpg30siQj}XOJqn1Af|0 zX<4)M&l<0-{>O^_&y?c$zl2xib}_nK%gh02r`R6G^_uO&&2JvAy)w2eat5U_$X#cN5D z!w;M(N3;$%DpIr!LkQqZX;S5miWF_5P?IV-*a`u_V1a@J2plx9LuE>mHtsg40T$n# z$ZT z+*kl5B}rS`o)9Apo)9Ym;s6EQnq4{OA{ZA7pxWdBT{wVC$;OjB1R?NHvgp_o(?fx+ zDNddwnIgsWC=^D`jRglp9Y}Bhfe2aWBXh^=ZI=Gi&tvv;#u;_i`k47sRqw@zM@Qrc z2sDWSx&rF4xdoIa)T!jAjnOX&W0KK=S>tUd#$HdY`|tbapX2^gUfgHY8`An(nbL3v z5+fsR%EZOdL4XZK!}b&?phngMXp|PX`w*i^C<&hMw^VzIeeF8y8E5U(dXM|Pee>99 zZ$_UQ-ky*Py$pe1JGS6cB@J(C#kd93YV`3TVdq9=lLI6hYj`>RzQ*d|j$KAerRK24 z|MNC6V@M#zW(polbo8v{$ck32z;M9C1vevTj1W@-PfpT;TWIy@ZTBNt2`$>=N`NttAqkR$`;_B|n^*eZh-t1cdrx(? ze{(qbmoP%^a|VXN_ND#O+sk&uNhGDs7?3Gnd5LUiEH(Y~XIgXAKI6K%f>bT~2K8VOB~QkuI4tJKz?gBxJHnCa9R@nB z`5}bIYmAgi8SMJa@N1m8kF-bLEzh!N+^y}}%U;8X0ezk_1e-f_2NNo}UNKf78PW<) zj4x1ar4SfoWX(w`QC`_}pPUGJ`v28I)TU@~~&G(!iZ0qVM7 zxxuLcE@63Y(g>65vuKdoA}Jp>k%3fuz5RRYWy~^S-2cu0W3H{%7D90B4g!J;>Dh8P(sgJXiNIXW(E zdTAgro0BE~Aix2r_s`+Ino}OR&o*nluisws*dg8ZU_xJ$B0l){k_QM5o-8Fsf-Tq} zC5R(kK?LvsXHDFXza+U_1lgS0+9Q>n!d5Emsq5>W@Jn4c$UiX8gwSDGTyZn;Ii>ubrPHw-a zncJx54?!P-NjDl&v6cx+8lX&(_WVkgr-R3y5T2xLDnf?=hAtUnh|vFUm2}ShY1h@l zuWSC(4s5=NmdgdJFGl&XZ`tIOm84@(0QUkU(IKa9A` z4`Ge^PM&kWP-gtS&QQh$Y_lUh_?W4S)c0z^1<>Y3g_s;98sMbbGbu=;xU^KLX6y;l zr_;;-)qZl{xwl#W`2FSdPRujjaG@Fypb-uyIj${64(`Yt;5H^ho)fFgh+1O;%F(oT z3Ywc?Y{$5!6Ytz7{{LEgJ(uy{`tzJMQ%rrG_~WZZK6~W#^Gw@*u>@uGFi-+a z1VCBlj_8pjDdSoIFX0hm7UR++x(|jDeZW+s(mr{%lX6b${yutq^Osrvzp>wKcnMom z1Xjjy>5-os^oT;5Bqm7hO;PrR1uHv|>KNFdW@(U_+$WBlV}CRMdPZNh%rw@iGyXAJ zF|z=lB@h98T|3~^rQsU3Niq$g^8)Feex9HjKx_Wm=Kmix=K?DxVs zci!7_-S@okf`Dd}8DbI+XG7*AN0~lIpm=CwCQK1CZjQ+f0z;@(vp;rsjPJ#ra!Eh8 zG{U@N-@ogh@yuzbeB}Z(>X922e{bm42q*?Z&W8>IGF z?%na{ICsCX{C39wwUkg75nOx>mOu%$^e%0;8LHtE)>Gn3>FF%z=p;g=BZr-Q{3 zMQA>#L88O!$TAxB$J?pCf9pzl|5(p>EB#l)jkB#f$X+>W+cS$wp*R4@`I(hTfk4iV zc|!B(AAw3OYK@-!$A z;FC29jE)3&*m1un`%bU5ES%2=k-?{TX zdxck#AX9vrRzdj^Bu^bMZ39d}p@7rfnM1(zzA0Xto~VVw0^-XPXZ6Ay_nvk}+W+SE zPnc)Dv}W%0oEa!7J{w~&Sd1}q~zxwUC0Yf26Ue&|Y*ANgtl=|tV0f<*uT zQgmyNc1zhUt~1-+^~}Cojce3cYKUTrCuPrQ`Dqk+^m3@JS|UmB5H;ZqvP(o7KvsxL z94ACPOhB4?Z@90f7HZwKw13BqyQfpj?$7)k5aWjsGaL|*L>u7dJc+Q7tIZY!L5B!Y zCJZ5h^F_!$ik3-sYOl5#My%`J98=z-tupU@sn;D=6isH_?8s1bOirGT{z!ku%6v|Ebjo&931HKl%1PZ}`2d6MMkfHMoJ3SJ0g z4Y20IB#($WZ7$7AeDFqf$@Rig#8!7~Iqo_}n>&^oYpXw}9eyo&8=E9PV88x25ONGo zrMm^BEHVV5%#A+B;*=4BB51>%h&Z-_Z3uMvr}su^b)7NC3;+EROPnLWvGR$;(<2g- zNN7=l!Ox^8Z#m*J7HK4MD!83P3?a-D!aUSNbFQj+pYOkkT!f3y=`ErkG z#WIRf9S&q7C7IOc&l)&5t;kp)BM3r@CT%p;*sJuG_9aZH69vYE8S41^j5%_ur`H>H zizC&);&=n_idwLO(S=4UHKXKkh`C_E9ALqCEkm_ofjC4D=1RF@1nh;~=k(qwag|tV zsVmJ{YV2#YTjJg~q|6~fJ2X+!vIU(f_V$H>ks_6(z(G(0#j#At3gxm}G3iW~1RBTa zWwhN!Ijx2{Uc2XyK39wBtr2z+8f7B_k_cwju%dgLo7_e;py1C`6?@A@8!B%mB%#=l z1LGQLhqP0CYId^`c9@-r7N*4= z9V*M>SOX|~m$h=5Z?Dt-TfOu+LoR8JlMgIFd~_uw0r2NX88^34mu`qKo z#F`TV&yJwHVi(b*F~{8hrh8Za^Ne|Bi2e1}Lt9C}^`VnWYnq&<-n+YoQFnPS<^9|2@kFSjwoFxq zSP9Jh$z?JG23I6qOJE4AWoknnB^Y-az^KuTjMY!|kF@vCx5ijYekaY?#%yu5|Bq=Q z$A}6>b67gk*vjMwR3#Jnzh{0ntSz^_Nh0Q(MF3s>>6@^Tf`{iVS!l;2*4-}+}flFx4;c%iY|HJp=^W| zOQzm@#3(eznIez%^c+(UFa7fGE&Y#MT4u(`8Z;2i4sfGO z&xbUi4v|@VbU{r?7zc`cS(-6h8GX&(N__2wHPR?GwS8+WFQzhJ_&!;%lmU&OpfIq_ zMI~8o8eo~b%tY=aVU4UeR#A8~k%Qwx+BdvE_m8=a`r=CauRUI9Ih_BNUKhR#N<5gM z;~>Za9woI8JL+EOFrX8TR7JXjRdgQRY186UEuiQQnFS1FLBz+)PaoWhNRJ@FVsTQ;sogwc9c$(he#!5x zHNFv{stTW?H&|9!5SavNZUO*CJP_fL3W_aXL-3r)qlyhu%`?FgDqk_|*TzaE-`3aP ztInGDxT&99Zb_6QZ?4$%3TCF>Jt+gG zDaJPH3S+JR%UZ9;9>>Zx7Gq2Jz+_xCo{&Ibksmfseq}0+S!)3+8jwI8>MoUH^MT-( zhg`h&*3wr?`TTlsucxP9^9}!pmd|TnbxFcEg;FvFcxhl7HA?$~BLvGb3Zc~a9dL?3 z$yDL37XlwvL9wvbKXs*gYOnv4Q}-_SwRGP+u|5&!W198JR|dW)Zp_fZf~!*fG4{3# zfhu^dw(#Mj6Xk+f2re)(q&3ogcZTw3yYrRz>gcJ&5MLc3LFa?Epv{ytS0L?43juvp zMGw9JaNzHh_T}e69hNeJeDFX*JO3Vci|>ZB@1J?ya{9f$r1^4TQ8b?eJp+(Z;8dkQ zIyfCVpt1>2E*zr^9EI6r;x0`BzaUu|)cO7r-ii1A-r^nY^wr`YJI9u83u`JA;j2L5 zf({s?4pBm;R>+UafgEUL)EX27Nv8~GM;#YsbaGNevAoi)>EG4QPdEQn?m54_zi%69 zX97gZTCHJRvLs1!NKQcj}sU^?yeI?^dS ztsG0)@%-K5e}D8f$T1y!SSp?p&(;Pw= zh=4^c7o{k8v}&B1geXFd)N0?e=Dq(stIgSBI3@2r%e-ZT9Um1io*0cP@@KEZ-5iTL zQVb{!;gJ9a5j|%h9bo&!oVj2iO!JRprqla=>Eu)1xHZ*XPFcU@vKYtBK zj-33dGIK16n;vut>Y~_Cqyw8MfWa>UR!#b{mi8c4DK%8{*=f$5&nU0#xB7l9r@hvg ztKX_rwDFD50OQWb0yTuvrWj>%5RvKi<|0B~B@b_Cx!v(~Y?BrjpTpau#eH^t^S#je z&vS(MZqD)VknE`O8T<1~Q3GX{XwYqdfeG=+!lnZ+MRy@mu_4k{iop*WT=p{FRpV;2 zhu>$}f0Y$_%cu8~<`1h74L(CtFf4#X3(l!F8eMAiaiWncNi+!}bdlPVHK*kShb4P* z>HnWV!BgjlAcov)5ft-XZ69e)xNxb1P$T`Q{jaaRKFS$N>PjAz7x0`cQzK8g|+YfrIpfe?ID#h4onNaFG8~9=yURC zQ3f|X7i<&;NYujsMoiS$DW(Y7oSY<0s4n2}b;A1N-cWCjv-Xf@KV_wwda6w%qOgLm zfG;`?nj&eE_UtO`>9s3Yv^h73SV(|m<4lLNIW`45He6Zbj{HxGCFEM`neFt~QoSqW zkk4wPg6|Z~I}|I0WCXW=cS^w}4+8qjSg*8vXCAS))JCCvUfKa6wQ$syGD@=mXwd|nK(hINhr zRQ&1&cp#;juuF1{#a*DhSYj8TMuq~6Y&=R}?u!mS3+&-?@4VGsysf_#V;*n6azpGZ zme5ZrDT*JlM!4)!LAaJC)s48)Nl1uO%lb6wRVBk1HD}~b?cL24U@F&|EzeuVYwy%> z@0$I+F~S|^^pIH-Cw?e8;hI5V#TiL!CfJ;advoQ=UJteoHB1a@K&|ZT zLE^?BiU9+d(tKi@f1ZEHk2F6ETFYyZE!G^U6j8xC=%#0or_;6&VV zk;+A!06JtqmH-0*j3M>*ES8EKSvY8a^~Cc^{Hc!}^64Ya9Y%`(zX7}w;`a*9#3ep$ zZ!)<_c(HesTg(Td52+?3m{e@8)gDA)o~Y^mL&@Xb^Ul8c?Dl`Dv;6qdt7WzM;ite) z9|tvT0Q-{%u0#qFv-ALv#Fwo<67ocyNmTdB8o?#Y!>+Mg&gYGBPADU^SWAd8_nUVp z@0Jxk{D3&Sbm6?{74W9V2Fw~2vEU44H()K$LCMt}k}zEY0kg7l9rOQLQ_iLBHRsB) zoErX$yQHKN=%57&vngUHAq=;B)&BqxmAnr_H7o&S)f&g66cf!C^(2gg+;q zn|kQ6_cQWIYwWYauA#(s{!hs6{9@U8!o=*2omgvcEkd}kxQf-1;)>CXO^X!Sm7z2& zM;RkBe%rZ@R&F0X+#Bxyb?<+|I$^YxKAhj17+rY!1|GXO2su!xA<;AjPadaWs=x$7 z7FP<#-WVO4rqvPO&oA%PYAOBoac2v2jj=)r#|UZjGeQT?Efsly^jeJt%81#6dsbw~ zvORRVE~%PZmI%h3P2kh-^WFYJnf=$l%09c4vFm-kjUJVnpO8IqP^5+f=Hf)iVb8v} zd8D@F2M9kAP^Y>aa-RL*Agfxq;f7NC$*rCke;u>lwR&hQYWQtda!!_f7z2lwH^R*U z5)%?s0ENJ?OfwFti9^POh5~e&NLsV4d}hCYpPByICo9V9eW}WesmikCQ zte+bhcKDFla3Tz=G)rp5>|%8Zbj(;7W?`(n+4YuJ+>V6>z3lvp$U))EU9kfEtRVp@XR_shAm3v*_)~l{72I z)=aqVhF|uI{l$J-93!StZ=OHq*%L4ne$W&U;^dMg9-3IRUC?A>txvFF!ERW|ppW=j z5;qU*QDT#;@-88-`$Ju)jq-MT_q><(%d4OsCj5}hP^Dy(-iAVB++?_o$SXj~mMbsd zK=Yu-hJ~93PqrdmU=?q)lXISX*R=o0ErgrS{cYSx}LLRgKmO{#9q_ANCzciBW zY=Bm2O&&pY#{6VhLIRo{QhOvFs}m^BrnX7QCQOjytH!>k&lX47Vf9j4ZU6Q&UgG3Ek8W>9#*funSn1B!)1mYOmOcKTq zh_qVsn4U9hwB?)I{_T`@{#vKUuuJ&!pS5QgO_T$_EGSAEj9ER{IEte{{nWY%2@Bo= z5cR|vn+9qfQp^}sXa$U#!i+8UT59j{<`cq7z1^B~kEfw10Qk+&3lfMYun7@%Q{rYq zoleCDm>`+OisA=}R9zGdspRSy;5Vcge;aq^8slm4tv}u!tK{5P0U#O*P~4G>23iWi zy9`yB6J?0Flz(xsYXoUo!@#Gp}vt`>Lum8d#vTP=bS0X2JTzZwCWH0P!PmYEofU=jOqhm{yglSsK`X{JCIu_@6`>#u}7f9dUhby<+$1)4~4ohWSy1I-kUxc?}vhTl(G zxv$?t9pksP`^gdHK-PD_Ar+{et*GOpt;gLMLOb+CF`OV?Cy`&6I3{W)ya?=rF5myt2#fi z>P+g<=*!6Uh4RlDz3g%B`@7cn-~TC>IgS8kBsc18Dv-(4mNls;Wu*$SFK}}t3f-Y} zM2%3gH84jk;TcjN6ADCHUI;qj2a*H#Z zmqOHFZ9tWZ$(s&DZs6b+0FdeouTz;m?681m>A@10r#50@0IE!H-g4V5EB!p;{V~=v z&S-DMv$9!*w<;sZ73RSKjp`iy`H)~r0V98e;OsQ>vB;HOjy$_!2uXnVnse>^dJD7v z*u%)@g*irUqnD5l0jc^B0HMQXBgdY#G$GLwkiSJ}7|1fVESiH!YISK|G>{|p#*5r7F^}B1bTu#}xC91<)q#LvXG|+K zFeQK)Yis6Z?D6vYdE_$MYvq*?PPjXkeRpF(MD_hGfa<}qNR14+?IEE6MGczI|(hE$aG6GG_CXYN^(J+Su*C6>|M+t`KjjWro{hGG0FQx#45y%OL?Cb#X%=NiW)} zrhME;AonG&iC|ZWjGU#tXmV4Hsnm31%X9tM>Yp?1l|~I|FIAi()R|coqRksT#|Ks< z5n#eJ2y?5CF+Edxd~9}y7*SRPe1AEOIqNL{jk)v8rRBR%4{83pBfw=-Uz8(x9#~{e zG~Chz`CzfaQ4BM%{5%j3b4;;4igKeEC81_10{#`?=Yo;7u?(e@ZT43s{B;m~j z?=(h^E>o5XcKC7%8i0Ty+Nl;9p^IjK89%uka4#+OI(jd0joxZGKh{uQjw^%^vhh-> zuaF-#wdAgpTJ&p;04YC;@!BK7dmNJ!M-n0)bKVIHWqFv#FFCw)^P4rUTuR72v>V#% z;oYT5lAIWr2cSk10dRSCsPb#FrdKo@!#81csyLa!yS zJ;Ta1&hyjyE1TM=7+$apEf&2A(`;@>6cb`j1V~XPfcjEP!=_z8=* zUvu>`>-_hJAI^#8m4AsKMSU_XnSj#6m4Y>T(#mwO8;B}jg-Fmb&=aO7Cyq!99cEaG z(LO>vebzQwpDq8__wT>f97_y2Svu6~5wAN9RP-U0HtSglzb8PtMA`twQ4kP^_U@Vx z@#9s*#iccVDP!JQW_xj_mTL+z?s`gyvjRAT2ms~F6S6Bb6q#|0L}wWS8jzh3F$KY~ zPHKLl;3W%!W>*3<;OE>@U;C|c>q;rVc1BzAjXdjbK!ZPhcXnC0T*v|?4Zq&qTTeax_Vw%hd%YFQeK{sodHO5_z+jOd z5IsZg`fM;F1XuyfZpmmxXV42pvfP#}vU6Zo2-^FESL+Vx-Fx4Df4q2B+-K+5Zg_F0 z0wsK>Hbma`EL-v5$OxS=?)zg%CGAtsUA4yb&&Vge z*SkVNr_Yw2wGxGf8Nsv&Pf8U!V$%4FyD@XD#9eH9t?javVi%6N>9v{iJ1vfz`p@Nr zJ3=h&+!no1lAMt^z~xewIaO{5@xryoMzAbE9&{OICW)OH7iqdMhFRzJ65qPH&Ux$n zWuDQ}|9OuPegvw#=`)ogS%O7E{4kQ!F!#(y76oeDz_epZ3|*TZy^VPSMven6zUR0r zmpbMQr;K^ef4QZ5ZVdkfp{$K98H6BI27`9oC7L@{Ce4k6QGPhch*ckZ9cpCdoq zV0!CIX2vcErq5^1+;oF z50XxitTaQ!2hYPZL0k;)QBP+uUN_Y0X>g-{eH!*7}xNw3o>E%cpCM>=w>W3W_P26jSvb@(p5?+W&9A*>7|EWa=Ja1I#$}NR}03ak3+0a zGjIeGluFJOtSWc6#+btO$t)N~M$(}-JtldAaDAe=;>fA(z01pawYx*FdFLDN-FqGw zTKcX?NyEk}ni(>2m~a?+Kga7x_l>a@VIrQEzv%WsyO zV~i=S5^7lGmpG;X7DWLgNz4pWi-TgV7>QOP8t^cN!I>&L2EdN+9hM@|09!z$ze4#M zaVM90ep#W+dSj@em0WWO>DK&i+v7j!6Q(4TEk0#)*y1|9al*1;0s}nN1>dPOkcP;0 z8geQ@e#=^R$ou@zN}KVfHS*o3pAc_OJ%~F@`e@ZnIv}HnB*M;zi?~?3cJxhh5ebiy zCT1+WNf<@P6}_Xl=Gtwp+e0{Sg*9`THleAcW#=)I5&qgdN?h$+S7>P_dajTrL1sR9N96!RHw8^Pu?^zN63Yi z#RK8m-~z9k6dRBeUuo>`R%mZ*PjJ=ru!D` z=_=yp7$jtNc{~Ec7!=ZL#Q~A?CtlpLbzwIzH6d7wsjcnNj%d>OOPKl-oX{8@XiEwrBV z=_gQ-K0$k6XqZN4%&_IDOG417#TKOp7*7aH(DQ?B*_JIBo`K47Yg%FSnL-O?v_4ZQ zFPAxcZZ)<7%ST@?3(AO*7!kwJle!%-Mx8t(+#E4d!@?P$JScI;C?v7O?AWh*N{uP3 zkjBsX=a$cE|L!=(pP7Bx(FdVZ1RQh9$o=8;M(R?dFs2GYIV0khCk(>CK%yhzAs4SV zhq#}=`;I%*cgoB^&9GB^Vf6Y&A{25kX#0zE7orKDCUHSHgqmALXiLoi6iTxa$Vb~0 z8jw`z06}N%Rbq%U{1VT-f9F|uDE+07O3mlQM&DnpwI7%w64T|QBuM;mKsUz1jcl-lzWY^~a7m*8t2BCR0qzys_O(lFZ zf)rxdFE)u}XriK4$7l<$TO2&ixl#`?<@|O(dE67is;RCvUL*Cj>+_c6$J3n7B562KM+csk zs%%pn6+;vt@q@sqIB-U>5wX*(4s?{M$Gbz>_stn5tIeFJU^~ z2-+d1!jKYKc{Uc%NReV!pEzlnxYZE0>BXzPeK)ore*CkP^;%pp-`#TyZLM;dF#Uzd z!6wB4gg;E=Iz;N@lE4!JiEsfUb(i8!k~bhx3Kbg`WcH{z`dv4zF#8O(jx^d| z?XNY7C`o@!{CtAcDN~_jcr!L_snbBk7B+A+SXyR@kpgr>UKLS*A#QeyJ?8N41u2F zeSP#XPF|_6zgB#4p1;pgqQ5R*8gNZQB2*%Aw~82>M*uWXc1uqJ1xSO;E4N0^B(brf z0o(5>q}pB!@27cooOh%?MjZLy`couBf0fVJBweI_h`9sTwgM9y5(%UMCIy7Cw>ff8XZ-wnh`HA{?y04;c*BiBxe)Xx#TbJwHGtsJ0Vj)q8=VF~z$GAR z7Um{ER7x$b8YO@N9L9_*rC;W0as06688N@}bBnp3qXGRL3gL+WCcg;v;W;@VhY~P+ zHbEki1WYbsiJ-~Z!MkLqk289G%^A!1b+58pi*J zfmoe@VJO#wM{14S0Llw-_bbzM;$_g@RdYY1^uA;KsqFvi&#T|K*SPtvPkDIYh(qHN z9!!kVo&?aML#(ic5ei>Z5RhlZPJ&((Lc(-queFr9V@zX}nA%vQzT9poz4tWl{6$`f za*)}y$;=$7yCt98B8X}Mtd360Bspl_WT8P|z$Jq1-_qM(|CaRY%>S-1Ul={SUEiv9 zYtEmlF`oj*5FW9ybCzZkGZJTz^ugo1k|7717HxPo=p1Qj`}4xQ=Z5$9%(=Hd`i~)n zm}*_GEfPZS?}i}_#1^G;llL}&paX#nSu#Y5Q-=j`4#9D|Tcc+zs_?~Hf47y|!f3ny z`d7*?=atIMHQoDj^Op?UC0Tah%p1HV{!fop%c6zCWXX;Jl|6oJNbJ0r(I88v-P(v@ zx1QF@rNv&uKPA8Mdk!lNxaJR4i$r`tvGC(v( zo!VUG#{P2*qokGZ`yr$=?k*?)G8#DZ7X*X`L$EaMQBw<%7MBA-9S=H}QqVyGAWu-6 z8mn5+6b+j-=UgwAS7N@sk5~K2p{;oGEsuC2^Eb-E1wS5o@Huri=}w6kpk}m9`P2+d z9it;~MWR`FWlTwpG{cT-&bQhLJ*3d~s;h@~X1L5(m_HM2AhuuiMvfh&Czl4XAea~IvzBAo-N}Z$h6>l$q8ZUpxmV}~H`w#`w0lXA@j<0}T zei@~|TAnOGs&j`b#r!a=O>*oh=l(~jz%7c##*tl*7GD~0v8B@+BF;|bIctOBZjUJ!U2qh^v;WI?rFuikuY{Ub zZ|(J&>m6&}Q}E=^pO|q9gc3$$;4B^Tq$pM&T?P~b#OXn+HcRG;#ToHeCe{6P(z@l% zwMy8z-ZT3uwd6ZyXa}8I@@Gd8tt18ji335+j>6EY%QG2?^e9mj%gLW4X*y(-@=-AQ>#MaZEC+)ZAG0>3wy-Cs+*f2>D+gk%qW#YuRCFykMw~$jMjuN?9j20Nu znZY=3_j&KR`Q=)B9qHCSN6)P!1Q#06BtX2NN6`Xc*zW z`H;te9W4E6_C5B>KbKfyI_&lw+u^84kT0o_&A~qH z0OI8qGXb2Du>%*&!VNA(py43!+JZ?{Q?-SBp^Nrr( zY%T1uXKgiw!TUp?lbXHa^U&0Q92-!k2q}Worxt}Jf_zm`)Jd+;h*Ef381eU02&0Wu z^Vxsg()UUC_7K*{vDCZjEHS%53yBC8io( zlrFUS;pv$WfmPiwdQ7?Wm22yDtU2?Ivy}2y96d6&0P2icrFNG`P~D*(ek@E#QoEu< zbWlU(sE(SMUd*DU(C{zCmwuV4g;RF<^M$?lYyaFF`|v~KPf@Q~4B&_;qzNyJuJ0ui zlfDA*tVlwqPMQ*RaxA1eL}Lb}K1=^$v>5Yzx&B#G`?06eb3N=+jK3awMKm}~b?hL( z=3x_s49=t^6I92|g^de&JmS_Sq_Xpe)U`W)KKbSR^NYK*`%eFVow?Tqya1~YLP9-z zB1c0TrULMs<@un%KmmOI(8ayMYPdW=14N?^tk6REt=y@1dcy4JH$td-QM9qa`_hgo&H*?YEq%jl8R^PYO7}Mowe8RER-dW# zmD~I0)${gxq0HQ7Pb(ZGDgH7a0j%hFi)3l7RS^hiRM;`|eJT2`?dCz;jt5W$8x%fe z?A1!`ZJnLQ>GS-YbBN`SmcpoH;t#hNWx`FbOIPk7q`C9rzyTY%-q6s|YC#?}YZZhw zK>$deq@P|*IsUp!y5;n~L+fGp{?^aGz!>po37~^@XjUc+bI~49K;fmsR*N%?OJ5%y zFK!u3`4PrCWC*W@S#JJ${*hu`bL1QAtoP-;uc-mzkIx|qEKIWll(3{F7&gbsnHkXr zAhG}zvs!W(P*ZdFLBt8KyYzW$Z|$Z%=g%?a{MuUOwYF;&emeZMiTo5Xzdvb2)gj9f z;{ha5B#tzCqa;tqv&Pl{0w*GNCCi)ozW;?@LTn@KyXIVV>~-3_KLxj|-WaQ|-d8N^yjE7t`<+zx4`)mO ze|vnKGhMP=ifo7y<;BINsR=6L@i7pR7`P=;aF>hCPKO&)Py4qKPVS?=9nbBzhnj9Z zmGEKs`!u)Y0v)4ASK=rdWnXp=MkaFHp>!u%BsDxvn^CL=0Bl@s*V0=*Ki4_;+57&w zRy;Yzxm5=Ve{+c1FvYbw1kKJ!0V0^8Kox~dD^*CQ+8q>i&35cnkd5Nxxo;VL)jLlu zY0h!}d1LNV*K8=1gFjQqrqBVjppgW-{l=%OSBDK_{Qg)qs02rknz0~X@-HNIerRL& zG0wUngqHfaQX1M9#D zT2*?_JKQruDD~A+U(Eg9x%TYwDz{<@{tf}cfGJX5qgithDA+P_tN?BJ)TBz0!cZHx zXm@|;oJv&tjrLOMGtYBp{h{Rk+HNI}HqWTxfxif+4q>@crq3HHc(L}r(4tEVM5V7! zPcKz!I2@e>wh(R7K+HAIIm0XepIlB(ch7v6YVfRX4j%{3U{EQzKR3Tf#Pa-#)h7T3 z2n!4ea98S!q+37wY_-4UtfTjK%IWo;S7ME~?s4}9R50@safH})vq}u5yIFFUQeo-z zX@r`iC|3?xBZ01mr(Zm5(8d<4g_i_2Mbc_?(KM^h2?`wWF3i%zNl%^YMR(_lRh;@`%<=CV(l0gF zzV44N|8iSt8{A%gxH7S!d~UeGf~Uw(1mFY$I8CYOV5-xrOsG0wv-$v|eyrScRyr?+ zdrG=}l`z-OKc`ejJPNunxH`39&5|2ia2oz#3%|Y-ywxyIeJb8Spwm%mff;WH*lfMN z9qwp#zj^;2;m5J^Xra9sqFG>zfLaGsk?|ErO5G19Yay&aA@=KovH(o&*p$bvLj@T& zXdrNJkJip^cdoMjKI!JOYU%6kR#FsxmfssAYX-nIzQW1EsF6Wby#qlzSP;;V&}N9(tzTynf+=eOQmB?xfJuaRG@ z1Adyhf+Ho43l{|zVCKXOm_85$(Zz6NC5_mdC8J7hs`%>ZC+=8!|9`DiV!o-hvP*fN z0UPD#Y0H~2JK8+b6qi7gWI1*{l5{DBp8rVzJ9y%{lyb7BNybep^|jy5Y1Ep|3@QAX zL%C_EoVF6x$&W>wKbeqtXjBBRmnjuze$3@qbBv0uDk9EgSPKNIfMkNM$ZG4x`ojKk zydFb5skPE)iao?SncD$=ZO$AiR>Mdk9d&EGAR$fyctGb1NQ5>umde=m7ePQgfHOIT z_FG)>tdUorHLZB>IV<-0`UZ5F{GL(x6v+aTEP4LcW_d8fsmUKr`HY1Mr-_>}Y8v27 zi9(BSa|yloHp0*At@KN(yR?>GQ=!Bz$da_D@r zGxp+Dp)SCUyI)@Q-rQoWvyVMTI6L24Mh=PeNjgr^NVV4_#U^5I&LA}~=L;bKh)of6 z;#Z?KpAJHyy%Kv8(b%QrwbnW-zxVIyFN8D7dtRKE z5gyTg@!CKsMH(g3K!YVSeAQZa_&eWGbB*(tbAONhn=ACaj`R-J1FaN%F+Rtg1-IGW% z$WMnRDS!ZQt5fMtjg_ZmuCgF|aH&;-G$;n`(ZQ)PMTFR%$8t`a^`4j4|LxS*>iMyR z*HfyQqy_TR5)0ZbJA%j<9i1#T_=rn{CdlejD^C?fkMOMA$WYS!h$rOVLc8^hxnc^h z-S&H3|CCW)WGMXjT?#}BfMGTfUz~$Fnoeja^65pgR?D7M(E~Tv<<qmo z6hnV-J!s@DF>^)FzT>6T&o3d(ltWHGjk5OHub)uGo;Q9l5OI0Z$03f|n=(BFb@+IA zVLzO3p<|gL6>5TTkyupMMsF%%&2w(qv8BFGzw7t>dQI{C`@yOw>C|H^a4Gt%2{j=%rfeXf*3TsvS`uU zVxvZvi2)Km&LZ7O3d8^sIBCq3G(0@$1X@5TTdyhRl*1Xh)OubIGnAB0TJhw(8ldqT z;sVK;4RC{e$>kUU4Ytt*49176E5oL8x;=ZMTHq!btnC^vh0tytCW= z|HmH6_yMs4!4{qkN>tV=3@vQGL_nH@ijD&W1TDf+hb7b7GhuJ|dRwE1ddB?mmYUjL zXRUMcj`tiA<7WqisRPD(9~5nOc~lYURX`d_dgvcskWd@EDJo6~5mu*9595`&(yeXu zR9oJ2#s0^9D~t$WJ|sY~OoXJR$iAI;^{h;ep#`po1T7mfnMrks^X;MR z0fwHa;{X6iqznLnKmY(B002}4006WX0Hc@)001CpAPl6TuIVR(KZK!!WxX=KQtomTq2Knpc)%ACh4_u(50J&*o zsD(>q>rqO|mK6*#1O#qo|w2r86Iq2_|5t7t%|L3%(QC~9#U z){teYr+6QwP3RdkP|^xDcdAM!tJ7dzT)D4q3V^%j#0(J$^+p!cEmK#>vYby-x zGs2IgM&hQx06I|?TJcxA4VTRWT!sBoLJc2>4T_J&2~2_x@d{cGo&%#H*bGV+q=a_` z0?UC08lAKlpANCq?yj`dYz!KXL%!t;G{CNM-)0ROJ^M*GDvBPcd+-%@E z49B2wF*86<42GR7RL5c<12twa^WH(+;tD*D@q?ZzOg%acGN>gDB~jt3z<$(7jGhWg zf9eT};tm{m24*~=!cOxWD=-R%AII7}GY6{grRic09bKxBKwt^*+wC*dvDAm=fPYCd zSOK&U6a@_m0U|Nv%3zx?2OtM{-h2fDTh{S`W{mk&(8lPNJj9c7i6jBNxo*~NCvjU=?dkR3qR3J(f0bnBu!U=VV=RniVhFmVN#1O^6-j=*n zJr0>fA7~xU@o+-Jfvz9Ofv6$X0|_$vnL)+O*>K*kSV~YML%F3u4k?R#9K)Hn1$Lew zX}Dof=CPiS6#y+mZBl6awRmR&*is5ydC^u-@KmmUBtV#*LEH(IKqeq8(cCa)lboN^ zg#=L|)`p@t9w%Gq@S(umYS%@y1*>w~5C~^9KYF52N%ss72^KFyCgun22b8HF#}3*Q zK*9iwHk*1lgqt@|z%o?^fDc6^)EYh{B$*U4uM!FWF@rxCG4t4iGQ~8wKwC7X3PyNk zhFS{q)BFqq1J95GQUzOp{%V*Zha_57thKTt5;6vk?s5oJfH&EJ;eiU_SQ*Hw@C==| z<>F8xR)vOn8jcUvY-a<+U+pk-lhu!mftAgeL6Jx^G$K65YD)tR7ITBw_zWnh_$lmk0iMFUI$t%gH^>3|uA@CL_|bQD4{L7{O0B4AhwGpGWgbrb_S z87f$~sku{19uj4S-WlQq3PDNC3YZ#*fdti=!GU^mjIbGYSL- zE`k;)%n(K&Ck#0lOV6OOGwYy3qyol1bEjqHUnT~iVdF$8z(X>Jp-W-%v2%d1gw2r5 zO9~4bq#>Yz)Oxw`0Qa2nD^E)Z4>dUEA8J5lfCJB{XRyRz3DjuFQrU4t;qqy%bs#zt zNn!JPh9xiZuTujPL%CikfU;2mQNc$*vp1;B}B;C=61A*Kc8*m#~V7VG0!z>>6Z33eH#cgGUKy*cqZah%MNi zfDe=iI8D(FQWS9+DYLnru-xPQJ%a{Q2YH)!n9E$s8Rf|M04QJj)b{r@}P$mw;1Jz2$Vqn6D8NA6( zgNpswlGiVMK<0!4jsPLtGeb-Pl!)Z~3dWKvENHjq&ai+Kv+Rd0 zC9x|?{HnXwX>^79J@)Uxh;&B?3UYgN#P#!R>(BL@*%GCxq(4 zDJW>isa7$79Y-#}F}Z=j!~itdCJyU)bYyC^rbFqYgPRPz4(*ZAplX567)Y%Ih$Nmg zLdF8!V;88P_$5QsI%AL>05s=E-}Ve52Jp=C07k(3C1!9oLF#x605>XFX0sG{XJZCZ zD*$YQr3liq(om?a(?}JV#8Lr);u656I0{1ObNEO~<~@a?Nl~rX8cSB;Js%leR(71E z(B>Z@TtqurR*33=CQ!PJ8df$D2!sfD{1g@zoC~HQlL4q7^>&%64@L$u{xKbq^7@nk z>Z7cnx)3u=L&5=JNe5kX`MHBuE%Di(!IMH$(=KylF369FX zsxF)GS`5iNhJkf}X2Y3>OcE7X^IM}G6eAD2=pifcX#j-PH(F4NC21|$pc-XwbRDjfahSGyi1Dtml5V{`nA$6Xs0yESH#}fvwq!(q84RJEzgohw) zk`=JGQU#{Su@N3BEH5M$1a}}qgh3fpkTwkohUN>Q1YRHeV4Do4IOM-oH|ZHRFUP;w zg5G!=wqFlA3r7E@@}W@mmji z3Jgdeb%e~xGH@+0FEuV;c`<-zj+C@%5@ncz_NLVw-Xs`$bK}loaGah;S0dvBQfP1$ zAdI3Ir9uYUxoooZ$<}8GnqVognAHtYHc|S00PokZp_>dqC|xC^6gW_|3hEghL=#mGOqO5z9hRa|ubc!D*m4l0SQh|fg$tc5cQ{8pr=jyl_D#;sg zzw`{!e+kMUaf-w+D7Y##D3K1|7?{&D;YD+8(olv*a>rTp&Ot`6avVKfsU)%tq|q5m za@^c3ZH5XDh!?!H3_{XR@$0Efx1qxU%450`x(8QfF;YZLM9Oe{fSZNcRF)Zxzmp1N zL_B7O#SGN=6N5wWn2uIDyl1%3BUFwjK$n3*A!31lZ9d$PK#%&;aMD}Lz$kk<#8hPC zyvm^dHfrGSlt?iw7o1?Vb7s1!2V^ebIJ30VHULTRq3HwUhatJ>DSjl`!%LFUT4iLO z=at0CK=Mz0t|v&Ifxv(d$qFcg!X;C5c*d!ENneI29g3p^Tpu&z;M8zkc9C}NBV$)h@A&*Dx|{`!(s|G8;|+drUOxH zumPWAP%c0VXQ*O3TEHwlLj%5}X9{Ov7(Iu-k?PnOo53?vLmAv0xE zyF+MTKUC788nhVH5x`n-_+-FIH)bgJphVc;L9vqM^yKyz1b>OEiKEirllMH5>!O&5kCMHiK{j3rLe1?La+Z z25j-joMloBVx2jifj|N~kKuubO=4hwAWbtQPz%Y%v2Hp;8J03Eg_lH5Q^EP+^#hFf z`w#vfsGoGBe))_?1HMH@;}s|n7`in{yW3}>j5>-i3_4bfB_$PL7}{O3-6h-mT~bm~ zQlX|@OhEzP71b5Y6}?N?=kS9Bv+U_TWHLD!ni$@edbTIjC(>mrRQ?)P&g=*Je#7`X%oYNjY^z~35B!`I}GSh z@gN2!3veasJI-;a#ojW#JCZmvIl1N*Kw$yl z$P*nhP%XG+z=JRq+QZKne_{&i7{Oul+y99-*$XM~Qokk>U$i zM(mB~l8PAsbKd;brIq?}$BvVU7H6U@LGXu88?^!o2=u_gBl)Yp(noo#-Pdn^HSE0N z-T96_;{YJU-m5~kQqZv>$g?Gr6U`4zvdXQ;8-^zgG|_m|mB|1>c`S0BQbPS}#IaxL zYrYd|3vr|rM@aAXc-VuYs!9Tt98of)W+{u3Mjt%Y=@F0!4uY&ZMec&tQI$kB`c!Xt zJ=`Am>p7&cOJBRsSAW7}{xiK-ZW<^3Gee1OhEsrpy)6v{T4rRB7zEamar%X&!jv>l zium%Bm|eo71}fARIB^2-Fm}!B=dnik^ZzqSu64ee+uTVh*lR-q7A7e@Zwy-wFQOv2 zs^Ev1tTi_ulI-NQMFHdK3yqC!w-fFjp{=ve8)Jo7&iO?-JFus7DSOeUlb$ge?n*#{ z7O0Q`5O6r!;Ad+t4H32nYNGUELQ3iztDH6N$>D}tf4$?hS^7Tf-X!Ij-lYN@#rR+% zgQhk(JSFJ%G;nf&A~$cq_}D>+5#3)KNNL!3i2dze=iBx6z30z$zgPZCaq;POVzG$* z6qDk}S!(jb#DPX@Ah9rStn4gGU{Fvv8|JT;f3N5E| z|G2gG@pC&Zi6W+F15W@?G|hp;V~)@s1O#*PEbvdIG^_&F##LO~vNeRt$ayF|{Qhe? zubz@lKl{%7-)n!Avq~8iOwTQWLbc2!2osFX1g|HTFx&xK(E$Tg1jb7mwH}m^EhA$= zN?xbL9p`OjmAB^#Yt?*Ky5&pniVq4gv`~2y$xfUtA_er=(clD%rYwwJFhw`u7GTyP z3>{oK+7ETzIm60zq_b%Sjr82Il&Ci znzlek+(3Z#D@zG`#E?@w{gu~F?_f)zmQc+`Y4iTD^sR7Z{KNp08S!ew|c5)&$#QDBc>Wp=%vTl&I|F{Em?X`M3ks?=a-`f-s1GQk`-fu zTF11*$pI|XNP3gBYRwlIw92}_g?CRXF`P2`sVA0K_ZpN~*Ee>pv)5kVUw@pt*B&ZYsWcXY=ma1G zOqaX1A$#ENY^ae$l+92~?M;n01{%f+QWY@Ncf+|euJ~^3d(FGYI&WkZN-q!70$3yJ zm>G7@C^3SPWdRsW1E7&~Di|OobVc$!{V7Ah)jr=j=hPa`PGf}`&*}TzJI-rMG4llN zjRi(CG7xGLH$;&x539gHLO=-xL8mA=ns%TPCBTb}rCu%9U;9b#?mJ6(JIV7X_dU1${(Ik}#Q*0i zdHi})`lI~fcS%pqZALRa4H!~}$M&Votu_&I!7Z@&DU=$cM{z(#h~h(tS^8myFjrY= z|5E>cxwSCP9qG0>FER#6uZkQSI3_ppM5{ZJW(8NV>{J;}mGIu|6%ybs)Y`F42-M;k zO35ju)z>Ux#h+@Ny`Nus2G?C;_Ze zZA`0#7{d#@-Q3rkt>w_eA8pKXuIQ9VuaciMxiED}!=P=hloSVb#Q3mc#u*}BMl$)j z6LhyifhnM(?9%#rrM7tgdbPHbL%g}JG1K1Vl_9-Ca!T=dA`1dU zNUhAt2AiWm5r~n96n4F#-PiL>XNOdKOQrRGLenvbE&-g(iw`C}aGaWfG2_Pv2xq_; zHRJK%5UsgFeKxe%umNJ9^|dg1T({g&ce}mK9(w#|W#;LR9t;lxNYNvLQ!cuR>=Xhj zu*gA01Oy@vwkox;J9R=w2cJOqRmTW%_j6ATC(V%W{4vHB)-Q(vK6*Wp5W~g(`V5<+ zm1+SXpb*SZ_61OgPdyI#sl)Mv6@w&e#~FNHB%+2w}5$FK89 zxY6ryntj;AW6_{waL@`VNK;C+P2n_bS*tOY7O(@+#S95f&?^2FV-0n-utwT1^j~jI zY1Mf{zWcmUqvtD=E<6t%krOo@y^Jx4qosj5r+9t(p~w;_wkHhkI3vUYTdt?~8BXu} zt{+<{;pS1tS>wmH1URGj7KI;?0o71}8&p}@I>sST8VxX3uq4a0Wlqo`6mGhl>7~y+ z_6Q^FHGAv3j4}Ut<>VJ@x@?Tzpf)X_tcf`n=&TAJa|meB$rGSVu(wi$bR05e#c=%< zi?QA*L z^rbG>js+Zeva>(@16v(Nw9$KOhJYRf-t_U?Q=DV0HG6wGjy`@&Gn{)z>@Ob#n&{D@ zrA7m>6JA)c8N$`+(Y#WK;V9N<29QRV+@}2PA$5xrm%2w>d6hNp4K?Ll!d{`}-Oo?k zAfVT0&YNecl*H*l2jYvWXk69|!SjJbuViKrp15Qqf~4s>pyuB7){_2tW7j^~|10kI z`uV5ldQA=zy<;CLp9UX#dYlkAb2!;i_9ua;9WufMJA2X^DAT7)%w>3el{Io~JJr_W zo;~l{{=B)Be9{=L_v7xkaMxp2qCj*tykJn%=Aw=$Lmd?OfNieK)i6s`{2w%aX{pY0 z^LcyByY4+})HG`yDdaSk4AE0XW`(XvVF*pIV=-hN5b2j1I<#)l>7BWRWDTmYAg7Cz zw^etDDa?6)PjQrRX8N(ey)4uH*5H zEtHoKy589J%~3{sbG}$cD0$Z1Q(EcvF{6i`tOD*3(sN{K4c@LJNf=DvkA&Sa6Agiq zG!3Y!IaxrBN(^fy&3;SSd7QK3tY_aH+8rmvkY5FO=+KkHfT`NT%A`baz_tY=1h2#0 zk{DTRY(To?Ec6nDfnPX3?Y4g$>yG~JsCT`6`n&ax66XU;$$qZ(;X%_V)?PKXSr)V$$BzQNK=iRw!7bUJQ+kF8OF%~}mdb+vV}}=nPN^PRs?_fxj#_@6 zYp(ZxD1G$1X3Bl^vr)Wyas&zeVO|8ra8g%$d4<;78A7x~i*%{cmceFHnfDKI&K|=UIi!E<{<*{$_9_ij=-IVesX)2;nS$5Ah@?R~5Xe)M z2cnJ~BTja3r@S<&5kpd!eeeD4zW3hi_5E1GcqfH-67Xk2?~xrDJ31$M>b#Zm)uF>7 zx;HM?G{8z0gOIJbU51E(cqAz6uVuWx_6~XNJ;&-PmsQ4`ZPXxxgdPCNH0&Xua7fMr zx?T_HD>$NJ5rE?kkpVK+xXA&lv!~3(GYf66@z!j;-&)Gs{hyZCUb*eilK~?1Q02OT zB*w5!ecJvyCuOgoy5q*@TnD)xS7<>m%*5Z&^9inM>`zoD=h#?QwD94mQ$A&~4D>YMiXek!^t#j`g(%-3`6z@4V z-;qw*GxvfBgC3T{F^e%-V0EOTgN9PoAjUFd z#CC4aqux@+h$RyRJz=L-IO7${k`*^vph{Q+egw&)VGB(Z1`!9hJ;fLxGC)YRo$4Mn z^|SwK_l!GJJ!8eu4Os-eETrU$(89#WQrcdwwOVx_P#8ir!j`83D%~_RbFP&ACz}2Z zImf;GY%P|)eky;ZnbUZG_Q3ES=xu>BgeVgrE(r>5;~f+Pe%R0QgTfo|A>RtWZRx}yMQ zPah!{u8(qiOfAJR`raq+diu^`u9ZR~Vi4$&p;UJj17+qEUW7hWe`MtsAj2^Kg6z0^|X3C+and9H-hZNGd|Av3cC}_@fqzdGrjx0(%%|ahHkJNA{zIZJg z5mzK?wsyk~fwV&NUdZYF^z+H-uCU{ev6R=sx^ee4a@lj{WKYroDOkk#$=ThyRL4yN zVGQ!zGDZqR8y9$b8JRq-XJ#oStlM@k^PiE^F8$7U=2+&YaO`W#md921QKn zR-{_gt4NJMY~pOl8Bs-$DZ6BA;3Z2LI+omLYHgOV^9v({RObk-=ageJXplL35(5CX zOD!15#iAE-1o?vdtkq0NLn_upMgsWaVO_Z(-8 z`){jn*0XB)GVBEuIKDGDw=90IaDEa(n!bzl$_IB3i+F=5Lg^Q<=B9zXZvEO6=1tNQGz~HU*^uQ2qd`d$*wke1%qvy0+Ki~aM?IzV z8TyU8yztoiERF}Bfb9dCsZww!fRk;ZA0phb$(!q7os1y?Ivg|48munPp74RRcS zB{Q_CPoH^qO1IB&M%;hLA4_Q^*YSRuH3XC8>=aa@$di>#Fy=k6ed zgs}W6;RlbHDs;h$lmSy?oElO->Fjt@n6I^V`#PcQnrl4C6)R_bn)1j`h04f~8|G_E zq9HW&qA}2CO2}%HHs|6K2`N5iV6WrczuOw;=h}9d@Ams%iTl)il3UTs-E2*EI`50(5@pFg(odTl+{8(+BlEkUK6ftlQo zi=IDzo)SJbDKYX6> zENj)on6pU(tX&a7r~zkc9wnNj1B!>eMM>oBj8gIlh$~ya?fm|GHPx~A=zGNbMi{yN zKk5%<_H>FgIP7RZsm`tdJia1hFkY}?b|P2;0_e`08Zwds@p)w_rSwi7Z_Z!$K6Bl3 z-mEo`I`SB5@SJfQQ_GjF>)pqJPV|mRxiD zq38PdENjJe;_PLd_thFYA?SF@vw#gIHh7ZUK-`HS;|7vAM8+6`glXBFpI*yMp?C(j z#4*ZBOSMPiK!O{-<{Pi?_Qu?MV zlt68{vaB7DgiSW8G%0I4;AGaOL5g6lap#Nm-rH--?XHt+DY?zJQer#6@P)9kAx@q^ zit@~jX@ugpN--geqJ#jeJB4)E#aQ)+5tCq1N8RiFUs_o&{I%O2E8LhuO@FjHUHD9B z@Y5#Af+UXm=DKv^@u<~-Kwd})3l+;#9KSeSJz9DC#f0`hvyM`C+c(u*^Sd$qvubNQ zrjv!wgDU|(-C8zCm>n@5ou1tpdBKX&uQzx01RCNNg6&BHHfm`2sg-ztANBluQwk@I zlFse_*Y#G(WQFg>3|=*JcFN`Kl+LWC(-Xc@F?bMJpiF|DH*#JsSTRxKjT(~mEtIsadC;S0 z5*suTSQe1#yyO1iwejN2b^UqoEdAxVOUb*oJ;oT}QxTzx4rD$}w7EFy297sjgNBt( zb8vcpRK#g$!wHAvzG}Gd)EYt=ch9xnoip{?`q{lCS)H#LjwW^nMX6*2uj&eQAW5A+ zvRu7#HHp)l6m)r7TuGGNc)yq4yl37$Zz;Xh7}JXNsx3wMt_3Tf2m9fLdjSWee12%|3fx+($PB1oTb*x(?M`S!cL=4{ z7+T85*&N>j4MsKyYT!_aH7sCyVKGI(BtfJ2TGZl*_`o4z>sO|1m=oTNwT!h#DWmnZ z_u6yizTcj!90LW8uSpPET2Rn%Btp&0nH*_o`Oy;Rg9Sv16qs=&M+gWS1_*X>Tyfld zexK+4JMK6o^!5M!yWKxW6K#CT28cqyFgar6NU+0I@c5|o0M&scHReFLK}JbiATdpd zAR0hpI{oFC?m4-I+2fmag)w3-GrUmx()b{O_^`Z@LrU;^4M1T8iy(OcMFx~n5$%hg z-H->IT-B1Zvs3xCm9_TCGqiv2&%1z$k#r6O5GV9k0CJ*RDIpSmG(8rTbqCCm5Q+7fjJF6dTwoahdy~;!Y7V z!R#n#fCs}bE?#KZWiTT~?Cr{0&zrA?u*ThQhTh6N=Y`wO+kjraE(s_)@+ik9Fm(rb z2qbrTK(Yd4=2X%%i4I-Qfh7zM9%Y}K(#mbsc76zB@0j`>PjeYQ@ z!CQz%u>e^qOoebp0EIP@7I}ipo#OloQ}?UHRC`aeyzqBQqmR=1i=*3!7JQ-9z@k+^ zDK8N0W&ofTt*$KyL|_mZShXhUVLP^JgXO}o)%{Q3d-gi&+9R~x!uYx8f6_gNs^HTj zLxUMnkj@xMdsgd$AVc5*1qhkaZZ%0XD=irlSgYzpXtlRf-y12M{Lk6H+_htCdG=aH znc!QnClD)4xE8sQLtqP;4nV1Pv`I1`IGtjHHF!m00Vu~vV}(~zPou`tJkP2`E^a2vy2VVW|cqN`4ORcTf zwDw3fwioOFB`#vD17D>%qxyhIH0T$e+?Pf*uS6XqQDCspv4{F7zq|K#r}TIIpncl>f^1P=Fh9GH{&5O9`G?TMPWx526$w_-HJmj3@{jP zq%>-PO&F6paBy)nm;xa|?+GQhRDX=4^tjT?t;h4$U2&GUDo{)TUxx`kRH&+y2gqBl zIa^rW(sarIO*CyyZ%hk15~il4AW;F@@2AWdLk@YQ-Dk-=|MOR!vDUYg9pKZG!vqsX zvgky)Hd&GPp+OUaLdH_`IY3W}H6wKh*e|H^^|Im}tISnrJ!Ov8 zT7$C62myvny&<`ywt(NMPMJ6%Vitw=QMJ~Hfsb8#uJFyZwjXA1`K+_+C^6o%cDjGX zmjOK)10ZYUO72a!Ya#1Ulr@@YnGhB%!rlOK*4|Wf&J2T(_w!n})^yelWBmVL+u@~C z(QJ=S?pdrgJ>IZzOLG9FM_3pevKPbXZ1Eby2S*2L6fj?@D3x_;%^~zx_bh3J_-75F zD!*iV5}*XHO%yREs?A=mJV_xG(?tZ(DPoyyN=s z&Jp`v{e~ni96LbCl}19QpJQ|fC@e_RrN%E&3jovGP-@Q`+lLFjmNa4^Z^e>tX=UZp z#z}RgcuTt@omF=1Y}nqRNx1;{B*aLiH**ROxoC5TgTw%eV&QnUMvbip1|1agXyS5O ziszpGdhMa_eOq0py%K}4UwdkS_19`|O`?&%9NuWZLWZjA_Txs0MA7j zE?#QXnML=<#iS%k3&!6|Y`g7~LXQ3Kb8oD1{_xWZC)*8L#O#&=#Jrm03T}Wad7vcu z!SiTHuPl9-Bx=)^WcP>mH>C}DiY^{bnL)*QK^k*5jyccsq1Z%HX2}xj7G2t}A zZicKAjmn2og)-UVwZMT#&FDgKX-gyJ6~3No%I(~r(n+PQkVm??oE2Lo!&vRT@xbFk zV$G5pI36}`DZe{K7Zjjm zAZE?2jusSx*5K)Kum~938!kivDT5Ti$piy0N*O?^1YzPB(qHw@ zly3QNtaaiYG43{6Ipr@JYEM{SA}YaLyiovmg9aWYv;!nO9c-b<8N@_TV?v16k0Waw zQwe3i{$n}irjg5;q0HUV?%@z+%H-X?lu4y#LYt2b3?#7ulEG;r^kvNkJWSR+?O}r? z%__@xhOkfC>4o+8459bkU+T%YS=vh@!xbBjn#c~WLWvMh@FvLkxGLosA}1)W4iQ`! zBByL!>_5GlQVy@SoyPy`?05H%YxjIoB<=0dRJTS{mp4h~G~8P%3c;{-kw8;mFZ+^+ z4N;gF5tz_udDk)L+Bt@hRvxvr7G4?W=QU8BM|+^09552+03RG~{R17OIctPuGKGK` zuMLWv$hk=1P%AdHD*hkBnf0yvRR(`uV?eOO9^{qeZKE1v&2+`h`#l;q^P)hQ6 zv=Oq!3PoU~rfOELUVwOLigknV!WbbEX;auQy?oj%Iq$mS$m!)dW}da~+mV7odubXR zh!v+K&XYGAH?~ZfE46166$F28F%CJxGj!oq969FFYN53hRxbag*k(%U^?Pa=H5w~u zj{`C|HS(7-x7@0zNzovvEl&15T^Twt66Gc(iUu)0Tv#rBrM%)=b^O_T-!I=f%5L-j zy)2)-A|!4w(3?_+Nt>oc>xdFVB{IoT0;NqXI}+sdz*s-3;(g?LpD57AX^e0KKC5RXm;bm!2ohC^~Isu7*@Ag%9J!SN> z`fcg18G4B^o!C#y$3~sK5{@Q7QRAwTg*H5>G?8dBs?N;(;;IX)KPA$LT%k0h3Ya?~ zgjrubp@-Uc+r5u4+I{KP^Sd}9XU_>Gx%>?2VRZ-1$qPyfsPf`w6^I_C2AIUEZjrDm zU-Rys_>WU$1BcnmrS+R~{3e-Z)T8vp@)v4od1^ zak!F((hoVL*gR;uk%d(vjiJ8$c8mFkz1qt8-m^m(WBxz(G9<|C&1F&0C*})UpvEOs zvqVG~Ia+h!*{?AD4H>&w6&TTI5h2>j`<44%S~=FS%J1X8lXm<)zq}G$m^}u-9N049 zMjR$_J~*^EKo#7cOg#LMDbsAwCplim22%+}y-|T;;lzKh=j2-4YEME4O z>1i`EC*c6~c(Zk+$BYAG%*?8>sSlzmrVt!NBnAnUYdP5QV(4@|LLw&;ry4E((RaFQ+|z$A;&4-_b^S0`xuL8TMjorg z^X?6KfTE(uj+`I`Y@|s*Ee%v-pi-~61EM7`m1t6!wKXwa;)twepi<6hZ>E>~dvnE+ z(~bGHUw17jCMbJshA3IIz^pG7)S6`2Lj+fmDLhWS0N_s)93glVAVo+PjE(eFTT5}i zl2+_D^xt-=>-R8kz2!PJ+2aPtqBKcRp*VF=TCMpVFe9N>hh09DtO=TXE8@!pCKzZ^ z=`FW$Yk4j7v+|!|YAQAL9(UZimVBs)?8PDpz?v}^0eib-O2^F*MaJz| zAO<`z5VWXG)f6wnJjvmCRu1{DF@8_6=O24(z0{oQ53RE}LH76@<6_9K#uv3X5xibe zsAa2)8z3mwuog2c<|MIbaO0Go+;ip^S1D`m|8p4g)YnpZb>>zk5aMI+S2R3#QmHc3 z=53D#Xn?5vp;Dpbpupj7Pm4QJ3)(0Y!K3$H`lpP zhB|CHVOq4I3P6&H3d>gApp>CZV&IZ9I$v4llO!$t#l6?Barc?h$}fi)+OD~^+jGuk zlmUAua6$3It5PK&cNL6G*fgchA+}2jkRZV$(5E^rr`lXv636$(-d`zs?Ah9>=e8VY z%roqiU%RY=D6Z8B!DEx$UNioqV5Jd5eb$EfAh{eYCI#< zvB%Fn<#pnTHKbLR5i_@!085He1{`X%?@}NrqR7|=K`inRU?kAjLu@UM1~oOxn5lI4 zsqd^b->v8O(QbTUzkO5x>pH;kxzURDnM9|6x;ZO%YBh) z*?_aWyj!bn-2diSA(h?lt-oDC*S6=0hmbc(iP9_(Ct=M4K2UGYSj4g)tR@o_yaQ=Qfy!=u~QEksf-33)^ZHx+-`&0}Wtw0b$v3+QJ1?mzZ z7iyxkwAe#y7S)?aZDIcR!|QjQ+j?pH+*JE!r)=!?YD<*mg$+My(qt*9HDwHf61~oR z;nQP~Nkp2cNxu-#(H^VbQO+y5^<7dvC%qMWJ0X`n46+$}cb0aLfkH@88@pZy=n-9f zglsZF=a~fPC9d6QF`*KPA>V6`q3jr5k72pv>*2KC#~j~KN%J$(O3#%kp(?3>vvFYu zlOuZ5Zv?3hQNkE2a(pSSAo!3e+D-4iKh~~moOXMN@4p+>1si_JJnYhPY1kkjb7UV`{&m?R&C|XU3x$J%=-8EJ+xuJAtYWY@u8s3 zBehn5LScbchX(`i6C4~kb9%NsGqs?P?={|C{oI`5nKA7gXWXH#x7(iUokrBp3NH<5 z%b<8E0XKn%x#Xutfj@FOs8BX^jKvYMU1oK_GP%rp#yPqCw{Fh&$DK#tE9DqQ8ps3m zyS1wpBm>kas+G2HrF^ zKW{9{{1tw91h7!5Se`N(+;{*B(q_pbW|?wbu|NVH2 zjM~rb^M`y>KyL={tE_A>P)Y)(5qB?&MSw6RxB=BiM>H4_px}TPyCF zV$J{WUvCKGmw5N@HKI))a^*=;_9G1-HMmS!D5v41r)bj51^|>!vbjV*DA=0o((AT! zPrqU8{8Bq1r1e(`w}#Yfe&W~$zfnoxWT45@h67y%JO&Rah?#SVP$WD?usli-_bX5g zy%a{|jxk?-J>3vaS##tW-)nKyR9-k=y#T-uO_CB!>ZmMI;AQ}>_&`z~RFDdg(Wx%R zAy0Cf9PEk$LGV32DmpaGo=Z63CO>qM|3x04ySzt5gg`fs?wl?HKAxeV= zQgA+GP}}9Dt{Fmf=S>MeBcvF{&2#*jR{3+s+U8t0tkL=xOC1%wL@-sFJfQ6MJ=(~M}x99T`eryPH|?f&=5sqg%@-_LEQ zZf^9>>@max5`!=S$QjzpW+bsG1jbdRL@|mXDR4q0TaPMJVDH+080-A^+d6gp-@^Md z>{f}CWqmXtWcq@{1Q<+e3qnykgC`H9P7B0vGxh3Cp4yl=K7J6`@fzDtvH#Xzz!H^2^bQjHY~Ha9`4QvvS2|2l^}7Hyfw0g3^Hydopx53);g?0$in8J7AI&9+kq((8Ots1-emw7Px2t zex7NKe)iwv#Wh~tVdZ~QiAaH4eTNQkdNSke>C|?FR`%fMhk!d%I`I%wqmKf*5^^}; zOnJDLPFf+46UJX}mUC|mEx&)>sbfV_fmRISiBN#XW<(e}liqY;uxmZy7t5GVchzq)_(zb;lat9{2wqE(|#R0u)ArOO15-ttY4Z)M7R=2=i-Vz@- zI_8n$9Q3k?cIc8+fs1u0?vF6WY3%6-3c;tIe2|5h6Dh7{gt+4|V@-2u{L$b>49 z-0+N)sb4~sup=8-kwxr{ra^4X-VEs@=HAKS_J6_(Y2W_)m?yT?_K9h&s?hY!a#O_% zAxU)P?A{Qg#G-*xYQl`wC<8}I(IqxOIWR5bif)a)&cE@U-ODKZg`aY}V+s&u`rs6C zMz4^W(}vC!krZ&XfM`+E*2)16G?o0)P-xPVAWqj*<_>NDI>vov%{T6vIfU^?ewW?l z_2~l9422K`Z7$uC5p908;Kab*a)Vz{q)Ck047Iy4wGhQlZNwGN8|S_}?!IO0w(DH? z?W5*P9~KdCgy>)+2)DH%0LC%Pfx_AbnV`=kxinL5gA^Up`x|yWIktZGy>+g8S3W!4 zaN3=(9a~%a7O6sJM@e2D7dTz9Lg62$YF%j-D}x^gi8!&9aa75QrY>98{r$iB(hIMQ z{_g)Tmaz6aVW!IvRWZhb1_$J4&!DtMBG3R6G)7jlRHAkS0zlH6-Jk@j<{XKW>x2?| zJ}a(RU!5`Bf6tFO)K`1jZAIx5rK6DmZ4&69iQCen`w$>b!Vt$yvB-pu3pbd^CPGsQ z&sZhhR7!g*wlHhhHJ>-vFDw0rra0+4Lu3yUC?eR*rOiomk*7sbo+dzd^2n+qrVO4J zJ7H&Z2{3d2SZ?cg_r2rFAp6ou}Y+z?b~uw=;V@FviS zx4JYaz_{hflvh6cg>vF6uavf0dGFs^Pk4wylD-@Vg_NXAVq&ea2(^MtA0&U`^!d1; z0PYNpI$#YP5p-&9VjORke)?*E|J(1JsjgS{s_9Q+$M^L)G^#@yrL&2R%>hLTUhG*B zi~^v91#i=)Ix{VNogBA)R;@3`JzMR6pOHdn=f79mjB7-oY4<5$apVCPUYu~?aXiC# z`I){Y*t0ata1tufF61!rntbAEm1p673Qv{FW?EtOJtm|>UuPs@IA5KRv#eXZUsZ+vjd&6f!&d63DQs zAPAA9NA5$3Ph3JhvNge#q7F~sB}{XE_;ewIB`?R2MjkbUvF02tw>Wq1xu5ywe02aZ zps#Ke3{VZQ@G;f4VY_PfU=5uSNusq`}xIoleU2m72zk zGmgL1@cO10Q9z_u5k0r&Rv8d&)-n|)8aoz1T=U|IEz++_c$Eh6&ad@5)9B;P*UG(j zr4jc}tjO>L9>M5fM^N;&eIsu=fR$A-Du^Hr`&4q4dJM z>(<<3c{8Mc_T44l*HidAp9^k%DvoWE#ma*j8h0%oR9e;xUnw;xQV>878s&K7_3AH( zms1dH{kyJpc1S(u5XRYMh5kpbspL~SrS&~R^8$>XDP;z!d8mY;HWPE_hNmYm=s2jv zC=GxfT@cJklP}l!$Bnb@9OK{Rhga@zd+a#UAZ=#!<+u>()GQT^e6;BZLJ3KvJReOG zG*J^Vz@`*#G%B&6p@WmpQhNXWhndfaxu%^%m~GBHQXJ_59kwr4hXv8bwKM^*4y?RT zWI*I~u=68mZcw8Lcfk1cn)7Rt@ugGVX}O2jL#^@b@=uQ~*VVF`n|)^76uHIZ1*Au5 z!}R1GP-KD^3`WvE9zIJ$=GHQ7QzOjJlTvCm=GS9C>!$Y3KIi^4$_{5F5iV;Ib=fuO@>1635MGM`9trt4@a11e>pT4N$Nhh&_ENZarG3URc+pp< zNv1oyQ+1xrg~C-L21~Q+O#@IDOqDT|B<;n8J5_ew{V2WkFze1Ite9$DbF4eg4t^{8 zVobo%Ck&CE6?L3uhflkXb(mpFk*j5T${2dHp<$370MGocns>Q7k1=NZ<<``~dh_mE zYA41{^dXtsb!r)lJu%(_4Uniwv?(YE7-S?@F;R_rtqI}a$yt^%I8y7s?NCNcb)Q&% zNF&soXIZ1QNJ#XBs^IJfEDU3~(5*nzXptfmbU=O~IZ($5CKPeZ2+(lX$I~vqhP6ft zd!IeuYqOux!d!osyE=L3lf>%?5t#tZnwu{;HgCg2D@=iGbrN-e(FnXJe=fy|OTz2b zoOWA3j=kplbF8sSs&Vdi(4-B0jEsFFGV=CJn#+*}PYGBih}FuF$_^2uJ6WX#ix;$T zgwbBQr-yz2X{m*f-yeUUwo~l@fed|VcTy>H0AkIZ+Oe<81rR+)60p)nr%-Xp(VK!y zjyTDozT*00&U;FKq5S(}%(}*Tc)Y=4D{?J;zJAW!f4z81tS`6JfBLgjv=^Zdfj&;q;&Q~`B*2`SX4k?PU8>lK zNYMc^Vq+?fn56+Ag^D%K>g&cj!>uvJm}XD)&yxN)f7T|vgFYskoPmuSHY_SY#b)f6K6@GJ-;9!V~h zSOkj>nSvd1X5!AE2$3R-p;cfcU9+}2ABJ@H*yH-S#T-_;p|uiT7^95&PYS{ZA0I3p zGb%GtmY~q>i3x;gPb(`hT=6+Z$CVxYX5m?iOfV+!TcO5#R=B6nm)}oerg{IpbHBA- zHRDTBYKHWb_}E%QfM?e#)?tbPZ z@G-;X21^l@2<{F@5>oYG>*i19rTm1if_A0Hia%Jx>g-MItR=R%%L3iyd@v-&ie!9)O!>D3a}4&nCm z#tHMLKF7H2otsBoSc37<;gts3BsM`>;Bt77kZGBRW^!)BCh*b)=vf-7T3A>Zxmf28 zf5m+6D>>bER{1Z~w$93<-hQ0b&c3E}4c6$x^%{p@Fkc)56i?qC0GSH~ey_$mH(?5o;+b?H##<+P>M}VGW zT1=q8RognP{oc>2Ih1x|o~5)`)+>9~!Kzj$F}&6oXkb$&UTZ`iaY*)xm5s#40|GAh zr&61O9g8k_AVck+9!3new6?@KT&}!$#HwI`RJmw zMh?P>Q~@0D(5ML<3=bSs#Ubv&P{dIs`{ciD@ROkcWY3iJCouCPM+C_WjmZ`7Mv!TPrW7HTEy@#s2m^ zW)z%7DPs8Es2ggE4Dk^?t4aU(*+f>SH;xKj-kF*IrJ&#*0Yc=;}M?3D5ebJY>k z{~^55PfKU+T~aPiAJiigoWr9TYE@RaP0)+REYw|XB*`RtQffG^ky6()|~t3;sG=JMJY zG>~QgZ=?OTPt9}G(@(vB#=HN`zn>#MXN(FRvSsJxLEIir&L$z!i?U%2m{>UuaO!e5 z>c|IOd_?^JKc5hCZ6l=^+ll+1{8KM=h23Vdz=-b$vkF9n=m`VYX3EQ^JDb2P@F0eZ zH4KvOq`6fgBZ3Ncv*VmvPfz!}G5@IRjdUhzSZd0EfRW_}u+wyiho;Sd> zXn6t~EFe>)jsYJPY%mJKeU4DqS>fcnLV0QB8Pn`%g?p=Q62ymP%nwYO@I=#Y+jG@@ z+Usx4U2`fc-}!&oSmWwbgSnad;*`90YhO{a}=7%N!mBwp6(jWB&WD`F@?F$50|@c;=?VFvQKtlr*&3u&nx$ z%+#@0YgkU8QbOzuC%j0^_(bJJ3a!-i{{HEmJMQY^{98*qio%3vD9Es2QU%$WT5MVL zk_`+p4y7*?m1VVC%xNh3O&6$<670$ z^!ceUVZ{WvICypmumG|L2eU%ZG=vVqg!vh!*1(A)I1Ehiw_f;Z-M0I?b=}w6xuwK3 zLO%P7?aWXM1#%o51xZ8)=k^IYz{D3~Nk-;`&y>8H#-R%!NP_%$?j7g3sr?&E4ZWoN zdmrKUzvd4Y@G1M#^C^dhNrDobHYzOcq(}wHPXj7c>{Wm!MnfD9Qy1i@Qf0CKa^D%h zy|wy^eZ_fdZt;}k#n_*sFSDp5pk~aCx)efg?Ir5O>xUj-ea~QQ^3j&z6tgvo?A)rY z%vRs6Ki{<<+kONElKurz?03lSGuQLzBVnXy6kR04Stv=NYu zj4Me8aMrd%c>SL_R`_TB@NevQEb{fIXn+u~WOZQ_f=#sx*V>ui>t8;1YijP`_Mx|-~ZRV#WOigirWpWLfH z#)t_^;wY6MK4j63-PvozE)xfE@+^k_-c9AtJ6j9!o_qS8HO)3=`9-Vx^Yc?CDgz=) z#MpeOA^WvN02#iP^by9Si6B#K!mOx`!L#b?Z87Bjc3WqL8*h21-JVx@Km9$FpQ*n} z$L9P*BSYxI3DdGFS6v>PO`vuGT2+itq&OFsbchJCXiFUL?E3S6tNvQrJ9)nOcRL!R z{uC(zMWPjpGi-9OF2EUaBLyNe2-*YyXP>EatAm7vl^!^n#+rMrUiKa%wRKxB;jjG4 zo-M2!_32O0H3Maj1fzhhYha{ci=ZAnw#bPQVTc@Ib!{?2ZTVs+R+?qr_WrytryfH~ zJ^cOmk7a#y=}%LfJ#cPe+Hi7K9kcTbNiKXU-PZrj<>WbgIDKFA7i_@?XEsu``M_4o5+0YY>cN#k77m9DibUKU`(k4Tgdw}m z%yHy3R}S;1SX0iq?zLkJK>#}R=V!zh1sib?d1zAwNC%M_fS}>{rAESrEOlJGQQEO0 z&?R5KYMC+2J=%Z0oIB#|^|yFZ9X;N(OVFPHF#@6Lh|(leyXz&z!JA;jQbe0F7lB0` zO>mFGq{&i&gw)+iO5^u+MlCbP`p+!4v{GXc^5+kUk|h)Trt}Hsb|8*NDrcT@UD?q~ zuLChcr{W&;B7oFR@Z2k|+5dm>hMW5vJGZ${+;#P!L=O=$n@SuoDi0T&9~EZ-&>->Y zd=3FnT`Ke^N32VsT@lj6-949Za~^-q-C{atmo?K1^{q3TGGyoPQkfYc_Ki0%)+8}; zlgV)S4YRO@3!S4r%F?vpU?nEZ98WL4noG4yq%AQY1-31%NNn;_?KWWTZ>H8hQ!mkA2n&tBibJX(_g!N}>6KuwalV z$;=IIBNc>>u1O|n1ld9fMU7C1$6JRRX%6+_5+j}*bKCoeJ4259#Zk|`Yvy#HkYt`U z0J5l7B~6mv=#-&SiX|X(r(p~UT364PI81of!n~<^;@ksz*E>_0G0!~yUHL{8 z=13{$K=esk+JnQN4r#{<> z{Z{lQ%ijYBJf!s5^M@`H95e{3im9?9hyouM-w&^~C1WJ0L0}2y{ARv2#4yHME6&w> z3OR;ceyfEQl|K%zh&^Ev>BX7_5YJ-4aAnB}2pd;2_>vRq?Lej+P9%WoSO2-`tTlFS zZ>74+=_}vd#=midE>ZqUpbB!!PpmI;>X1zd_NGgV6^+(fb(u0Fk|4KEk&anm!@~RB z^8WcV+%|qrF~@oCOQGHnlcFYnadimYJKO^rWq4bl;Mp+(B5)za31VH4kZHAciN_rx zKQ35G&H3)U`&m8q-1nH_r9Ed^f9KLblE1o5FJv_tmS)u37-&S!l+k3w)R9e-OyJG= z@&*|oQ>y`F_~ZMx#r4m7|BTgB>>>8qa&4{WcSioYgbFc=2CF$x)~+nWvnCD{EUorD z$gMdNr`8>(EewRru|@N)7DGMn&2wW3Ym~G445ifA+Sq@9ya-!GNEI6{YbcH-;X@}+ zF?H$zhE(eaXx0l4rV~=`xux_sa@~2f^6Sm*gxvN#Kbj$b!kpwKY9>Wd6Sp8=9#ldC z!-PL`Rjx~TN$~`@nYFC#&~A<@i#;ZFkcs}+&KbQ z0gNRTQ!-eIfPj(>s#tM?aED8TXj0_>&(~g__m+Lre)au#^Ubra*=pYXvgr6*z$O+F zHm3}muQ(|H;1C5-0!v-|$ht6QjN1xEygZ0m;GxWZZ=U_fFzbK&{5Rh_d)%JhJTnFS zF$pyxfgJKCM%y1OF;?hirC@{BsILJbg^I1Ug;22ORL5CDtMj(mZ`tRpyW(Ce)K@|Y zd-S6F^OelWjy!FR+PFdZBB5C(N~xfPXh8Rh%cTcxTB5{NPIYE|b;Z0&+@tQBav5Ru zz0Y4gqP@012Xk&o7-16?dGqn6rPhQXZzlxU;3yUwq#t;rV6{oWMzAjNl$uwF^Y{Jh zZztUua&GDFeweZwf6WBYpyw~s+^_pH7 zyOy>0%KPS7-wbEQnmUO#<4??)KdoN8g0v#e*D4&BwtRSnEXT1SYz(BaGPeN`6qe`0 z|GTm5klSu!lo#IoFTGZBNO%5y!qh+c1WPHhRtZJh5MNAu2!MBi1urYJuh5$za zY+`&ht9FQVyCd!1``h=OUDjE(?h{%m<+XQ?t`>iwB0YkmsBg_2IV!CzY`6n)hpN>I zOa$EOBNwNoN2^dI57p9J2=BcUW9ezVmTC?8)scR=3kr(ANM=;jc-4naue)7#bXFum z;FOYG5eB7^AW6|3T^u@rSlG&w)7+!IvUZNE?HuOZKi)G-xDPQI@khi`16*H1lp#^q zTVc3Bj98*sq6Eg3DNGu<2Jl!>kkGH)pX%&=kJN52XSWbi4d?DLJp46k280;`9f#7^ zHvA$)7vl~V8V~wdfda5;(Z;7sRrcfcJB3sCinr!hR#0zySW#AewHJqH!W7 zw+IF#GZ4_gQAC1{lQyj`wW-}1fTBU72s+a$^UgZU33va~L+kUN)@}-Qv@{HVu+9!l zi$!y=;N@YpmJC*3B&>!?0v&P5Oqaa8vQrWv%O^iSbS;r!#|s^%G!E|mJOf}@s1{JuoiVpOa!9|Sk@rR`sCiLr;D!zm-zabB(= zty)%?i@_hC7z_uLAOnbjow7oXJf#5;#S2Md?C#MXT62_$XqokcPq%mLxM9Tc`s#I! z^Gm`uzM#i*$(+p9rL~>@?PWx8y%o z4f&@&ds=_)IAT7np80tq$BHFW5Jw2xvmd%@sF1-nfzF`=?r=$z)*eApu~z-MnNMtM zwY1MK_4klleyjcRM=V(ZuxG(XkW7vmc~MK#Ys16A4hleGQSe-h&}pKi)?n6#y$F+j z$9~q{rL{cTZSD0_Mhbt2+;aUFkkWu(9;P>;ko{3qNo@-oEOVG7`MpR}n^dG0nOJLo zHZ>BGecbeNNUo`hdt2jlq1Qs-osjN^qxvN`R$eNTC0^3;#_;3CAs_>rRC~yU&g#(`*Y+E zGhq)%SW!TcFvujwueHi#t69u)^X!@8>idjS+ka<_eOkF`&y?%^j7Tj%GBKEu*m2R! zN!#_T0Rkp_ia}fSDHA3=c(B~AU>BqqXw%Eexz-tO{_X#hdb#iIzf1c$h9#}Elxj=n z!xJC7HA`wYI>>@x=-H(%DC9&F6aj8GNSnc>M}GOlxK5a}mGyh=Z`@PPtg~d3%8yg& zv(^^O(V;tEUoeqLW=gyr>}{lxx43h(w7_CKM9 zTY9){lz-ddw8?MI9c0All*;7C?uQyu#Lz%XLuIpIfR!STPo0B1bS`~ZL!pMfTd%*i zl}8_g7M8qyCJ;yHd z{_^%bZ@p3SZR3R9`f9OH^C!tqR4_C$gcd`!;+{dH#ivXo0^WQCS6C`JVn!EC7BN+` zM$ozcm(*AI;rDsNIzRvX&It47R4e-9$b&5Qgb^k(R1HG`GVCr@vnIZ11kK#;xMxS8p_1MQiN~Tb!RGFP^Jf4lw7o`2jYl~_k8t+Y7aEj!i717;PVDMe3_Lvd*X zjM-zwXj!1W1b=l}ULAsQM{AJ?Rix7B*!hj$Ua7m3zSo;O|8-LdyZp80AVbGbS|Vi<>y-zin4%;;o?U=|hDKT^N2`=|S%7z(W1oK7&mq^9#|~reT=&`Ksbz5d zKyi5cVn8%N66AQ%u~G%lf=3sW*wC=1Oqe=uQ%bz$5pu`wye;4QTA6pQHBSEZcs#IiVIpAx34*G|7HgjM zobX0?zps8;iSxGD>J5#;_@PPk!q_4;a3-~7a<;1lv?EKD+3JJws|_16aq~xs320E^ zIYSXH`_AcP;@AO(2RtaUO5 zZ1A)UPEL8AU2-0E#PoWqtG88h?=OZDDU07RPHs2_pCo2?b7uJ3tW5Oj<0wHOIDq&Z zn?sW#QYB9aSh>55o#*X;wbQ~0XO`Pn7to5|r7UYQ?b#q_FAWNqHnl*(FtI{nl0oT> zqCIkM)D(~=uh`Zyb9`&$yxuvztv*VOVdNA0&v%@iD1ISUz`?TUlqOqqBk=GYN|EFR z0}L2iz^2BcO?8;CxR{`Zg#ViE-KEw)Z@VX@xn`R;=ekOKcbazM_e*Y3qE&4!tzctO zND!)8bqWD+Koi@eIfpQ~-F2!`?9zgmZmF@n82d=~?0o*IdF?WO?(Z%FD1e11w8@mC zJvWL;J1macY^pru6#_D~_OIalZ+^ zkA)ut5OJZ~Q0If0U5W)H#_V}AD%u^nIbME#7yw|7Ykwh)eRG&2wYX=AZ|2)~=zpK^ z%H9yaIdUeP=_V*STJxr2kf;oQ*vMT0#ApyHMP@3cVQN;Y7$`(i{wZtLch~#<#`|GtOthq25VWNX2t}jrtnh_KN!tKAE!CaBe5v&O&p+$8 z($4=e&i&F4Z>Cy8@bKdUTdl#eNl}zzwnvs9P^LcFvV){AS%y3DPEs^UJ|KmXaav8c z&D3A1FTdM%&bftlPYJyo;78M>g((B&izd6dS$#4%AW+6Oec4+CWep0s6x7rqwpKUN zouQ@iPnt9Rcv9_sl-ttYE#6Q<;Q7(uB8$!tQDkz^X(^?Hj)Yk(kU$G0sVbBxbAN!_gUUsKix5Jto!V!5YhShnpFXp6Dnic?((qEvElne1VkkEX$BKX zZfOfBz^WLfJyS*x?UvVKE4Pl6$4`Iu6w2Kus!bK%C_`BpUzr?hm-u{$Fo*}318ko> zbrG>9kX?=l%)-<^Qr_G1_gGFkIpkEvhJL7H001n|00000000082s>Y32-1EdW(e#+ zFfb${o6`CL6Ho%6M<4(MelUdrAn>06Kx5Z>R9&ft!FZ>U;( zq(K*CRbm5ke6nnt0yip&UopvI&T{yhy*Z@#;`H(62mojX2(0$tr==)$*^<*sRUraDX3?#m_agWNepJc z=w?H_s7sQg20BE*;Pn%=vk$1D#Rg}SMmFRF@uW5w4ZqUOtdr;xOYaTF(%bLhQkcqb zcWh8TG|;G)O;%$GV{S6^9z>3wE4C?ceu0Lm280@IUri*kYWOf3J|tX;)ft8*+zc#c zUNhr_x-+o>C3dPrw@Ev?;~orAfgGjDt(RtrrRfi2>2m}$V6Y@I@cP@rqdc$zVZ%gX zC6Q`}v>VKepoY9Cj{#0{zRDlW)I!HOI-t$I42NgLfn{ExVIol$u$E5NZk{%0E?$9d zg;d!NIjRkIXAurep@Pfkfs-m> z7e9uRm>*MA7O8}BIW8N{ywbiEM3C?RoY8qmhB{6ze};Z2vfn^bN7!kE>Du+$tHZH#FvoW`x@$0|fuN3rH=j0UgJ)+9X)EwPI+aFp%322tS?4 zUbyu^gKctO3g@p(AeomnbdwXD)muM%q&V7Ykyz27ve;?>B?P1U3_!yfomRY#`3iyD zNr3@v3h`?3K@mE<+>m#+#Gg`G{UsWDf0!ld8n^;%eQd~!(mfK$4rKJqV8S1|qB2m7 z47GE^sWs{djRFFO0)WT`s(>zqoPn@p>${@FA0+O$9b8pz{&INIquh}1#Ils1q5Cxg z#@W#NWX!?)l`=drbE}a8SL?=x9>>TAB^5PsLipIA%rnVTSZm6!tSaCGk0>(0PnOg&1%&ghf5pM@A;FP~K42${FhnGb5Kf0&RX*?Se5W9B(O zO^igtMn}UZaPrM*ftVT!0H!|;CDx46)jWgto9qb9o$*M@rmCo=)DW8D|EOBkc}T5EQ>RXrPQNRct*ufgFJWb;0A5b>3*Ef$}r_ib8b2 z<1)jmiUbj|T06f2C|Y+cWoobw;?=4+q(~(V^JH)rX%R*(9+3{@FJgn_D3Jp$7gBi~&udkD2 zf;*1wSR*{mK<=c~s#leiyCDWrm~zJuuSwe9rN9A9%dtJ8*b4K-?KHta%Ga;mk@UR&*S0-WFzbL+wCa`sD>_ z6TiWu`yvNiu#*jDpmb%XgfbHSzRwhLLP`uzV-WlCH-Ts7P*kfM0k>{o1zs5e6|e#Z z21(aMJBDo|0K6VTgETn}dE7wR27~_UfeLv9v0vpuMqfD|O1#NXYZNSf3j8H#)dx$4 zj;;Z6LI@`Yc+uPda=hoNszBX6XP{odE_{XKnmR1yR-AWlpgV8iAGG;1ITa05tLdB- z@GLvdh*&-5nGDmA7;!=2{mK};$%wR!vl)LuLt+EVhL;+90H)AwD9?CVOZRsD5KHD7 zS|ANR6O*{;3xVgLm5q*}(&fo*7it568e+s?P#v#LPG&9#DqajjLx|jy+;T9M?i(UL z+Nfa_pxz=nunXB8PJ&05Hv}xFKb|f1>1Sv$z{!ShP@#tYR$nH6i49V=7B*&xbPxhLn`p0LlO`G^bHoX#1e=Dbu2I_T^1-UG@wEWfJoW! zT0bdN3N^??)N%}J6A#j=c;*^}pBq~1HcQ6^&2eMH8iH63)9NLE0~8aiedxhS%(20N zk|fs)|CNS;?(8TFJpeBfFnE0dgklo6tNaBtz+W$E@B%l3#mu542&&17@cjmFa)}8= zId-a^vx-VkaSdb1_t$V<>ga>H0b@?MfF*|&^LWFNINhojsHJ{}xFJq9_*49=_-mMN zHnSU!#Bqt&-{YF_$vPVv)JPhYZ88&h#xYzM@kmwE5V)}QuYj?{70`KXWHIKC=@Gm6{L-mWO_BrE~D(fEd|kltz)PG6yPD@7_u}F5lr~KAwiv3Ca?dRI&O%8DxjjTDx<>}0pM>K z!f&{0o`EPuBHsXvW~@Lf;L$mSNt^mb?*wvO4g9OA8`MdGOo$H{2%4>w50ylrA+iHR zG;e2<@S7p(Kgh%scNj~dVmz6kj%^N{_eKv!10IVF!JQB%8X5wlW^=^wTCZ@0B%qtr z81|xtFAOC%v9!r%hj?!IN+RcBz_@eup+i{H|0LjXCWc!L zObpzR^ap8Jmf*^us|w-+Q3GA$6fpDph9hxe2W>(Af*Gc4%$J7EQu^B}x*Djre}g~R z&R{Fa1jL3$!a(E@26*Z2AU3*i-{b5Rm|8xB@{riY#@J3)F7d zGfbF3|KL1^1u<9^&st%+0#-1iE4N`M>l)s`lb}vn#d{k zqzar3{a_Ov!i&`NkrCgwbCNgE2M`+of`|k^!=n>baLd8rje;c9`A&ui0CL09h13QH z6#y*s2BKji)@OHe^fQbN+4_sKGE3h%LCtFrqx%W)s6c|$e+Di9Ip`RG+voC+@_`*t zs681tP-yD>NG5nbSM<>Ve8ht2#BfKvy=X1gyZq3_kIRiS+(FEslR=g%!VsQs*w&X?f^}d^1jDH{rJ~Th&L*nW_rQ5y zj(}=jI9aPNY#MXyh6kz^PX$wWPsPDkuqwt3IvEDy2o_B0Qpg`#Aj%c6IV7lYp%^M` zBMg}F=m#b-yn$*3sLteoHh2RdY^w&N0FT86sDRK7Dg~<6{D%oIxrSKKJ0mtwZ!rZ0^7*V&KF;wcy*prLZ_2c9EJ6?--inDt;RjlXHZ+ z0P-Q83o0JLk`7hyu_9_gLyse4tIVBvRbAu_h1y{|49Y${O|rE>!3Mr4TXS#))rHuQ z_%M+KJix>Z1=b|^0?84~pn*~fCZUaHA60W%oL{e`ljKTx6v6xB0*OJztVt~(qXP%~E0p1L)7jEl8C1L>FRGFM96kelU;~5xs$f1!2sxGwvEOM9 z?t*5MX5e4FNI7*}5iMnJFaVqwf*arm1%{Q4LT0?-IZn1=Jj0i+!hX>p84d2xAvQi2 zq>J?LUUcGTK89{GaG!xd+`(@cmY}+nt@ocBGSGk$`%A(HHBX@C)ioRdnC#Sg`&~NO z)$mP5%U2*ruX0tobU19q7Qfl ziD}${@nK+q3aFS}KeX0S8+gp0OX{z5&z!6ev_M<~)%v}v!BW-x`uPm8-|c^d`qB1`-u!<<-pbgD{Du|DIyP1P02n!a1}G7T#=d%uk33-!56mvCj~_ zeVL#=_+$kJ0E-PUpasZ$Y>53Tj*zu>ys8aFXw}XFbPW|u;j{cJZ7^#2A|;+p*bRr6lm`ZNyaNWJ_<5e_ z%nN1!geNe{&%lww520c4i>Hp^+j{}YO}aP?QZ^d$p5lG9<~vOI&&wb3P#qtvhBNQU zIE1jW=`+;mPAEVJi4O&a^)}2~1;wmM@eOuI8j5TX`;n1>AV|mY8~DM|<$1p_y0?!R z8golB4CJu>ribu)FiZIkhetityQmROcrt@6Woe_7E(rx#+6@=;5;I+`->n5Q3W~c? zha*vB5QHONRCbZtidt4-bQ|@6Y7H26vWl|n2OLt`fUSFFqX)izVuKyW1kPB2{}|Lq z1rM20UBFa>CCl(Pf~Elw5gFdEja=c|ZBPsJWBA^gW&=9!7gWK7_jvtJNn!T zsBPE~MrW@v3Y5gXAw)vS4s5|~vX}uA!^F~D>nKVE@&+l}%NOA|$uY!RqAnYvAhlEz zp1M#>s^JQx-J!t%=cRv`rHLtXLvnZ7hz-vsALRE9fX6Kz4xiCa8%|;Z4WYp_8U4V_ zOBf!g9mp4C^!Ik&d4S>bDmX2tx`29syE#Lo%l5wk?$TY%hX&i^bctM`59V=*Gvtt% zJSq^*BO%-?XuZ{PJQN$!-%yD;C?E<@Bs#;$F)h3DoqaGE!$1IoXh!$w2!8R*W|3XbF{R>JfgG~gzk>=^Agnc76zut+48sd|BeAp`b1 z;0GUt;zKd0hv{!;U?{aJVige{8@ctt5B_CgDYI03)gisYbf0Sh(nU1D6hdeUR^`?a z1a_#rDBS=w+}x`K)us1qYM_RY`jmz5zEpuxQ&Lh=0**BEKKjOYR4;V{a|3q+7FW$T-rY{AtH-lP3^Awt!fQs3$(Myxk`X@M z^lUKZA&(3t>^==_0yQX=oU5<6GB~47m|15p^Y=DK+UJh_etUD3o!2g{lP!GE2x!=j zj+H()#71Q;;TKpB9QXnQ5EE=_T9}dPamvsDQ*Mv=POuqz!NEO$d<=;DlJ3HRZiq+-ud_UaD!u(sS8B z^6hc6LJ}J^A9$^CDRsAohbH5XAuJY`G@rM%GOEUg79T2q##B>Cb%tC^x@YZIYwWL= zxz9&XihN#EBDE>Pvf~0vlM(Hm>Hw$=1U3$dq;$YQT96wSNTSFC=ic*ZbM>-jo$J)q zOIqQN6l%;#sA0%=A_-QQ4ypRgv`3Oj54}rruF&zifk|APPHAX#KulrKq-TwxkDSUZ zF_xcWjAN&n^XWaT_)U%4H>xbssZf>|9V8(jqlOH+K4oe0 zf(oGyU@F)F8RPQ1%IxEo@Y?vXjWFVkBepmWk^LI@@F61mhSGpU8YMU#u zP<^^Hr^Nn2ub>MCf)lC<*k=zy6c7dcDSL9Zurk<#KvAJHZXBo zbm6_>gH@FvkXz~{ngM<=zpZ1Pfzov z-AgTXrW4JDH#Q_|mFAErqt~@!rqGLfd!a`NITz?%*$MFl$Eik}Fr2WiJ$q{B#98_p zfBw17ZME$;(k`DH3rR}XF;S-q4MrrlClYWa@-t=01cN}IxiB6j+&}_hQ_OXD-l^Wy zdVk^1e1CW=t=+>pYothNzC&rNj5yIl2PJGz+*Sg1hms;&90gg8E_PkMF3dE6ZFX!s z=T!51d;QyD-93a`>J>8cSx|>{waCWY7}zEjI`~k+S|~eAaD>ny*M&^km0A)fcr@(9 zer8PT=5kLxbL13T+^wH*%et%$K-k3)5gl@~;>!lDoQle-CIQ5lBLJfwcUMLOO7!r6 zjz6{UxNF?At`=Y4^`x-+o2!j^6`wC33|3Y(GQiWGv#%0E2H?Pn5)>?Xt+J%ifQ}Ye zl)NI>`c^iu1g_VrW$@1@JC@?q71(X;7=F?`JA!Uzx}&=6m0 z4Av!@G{U1VvkWp7C9N=@|LdJKzS#Qc)pL&mq2)QcOLAef$<1YqSFhRw9HUsf@s1M z2DYNj$d4jH=A9QtjeW+oX6WJNJI)BP&(nWuEfH1UR)-&AXfnv7y|C7xA|pWwoC{ch zga!^PLUXNy)ZlFsD6Z|K+d~+!jrwP+ecu;P={1jB$5t$$`c@DLkymQQ`5=o?gNFyR z7%_oVS_V*agbbQ8C<+7@O_-vOc783Pp1n@}_mn}O3BD<=_ZjczLLnPohfQ9B(Qp$Ii)N2~|&C_4`ea{P~*V)t$SP+*O zm1nMb|Fe`F7QQ`+e4xoAL5CMcgQ+wvNpA1Uq!@&lkwNHCv@B%$5>$ znI`uc$GUm$KT9p?rN8ccE8P3f?Z(XA=ZFhTA}>H|Zh&o>VCf10Ms1CF4<4u7Aq=f9%`ZXnoK6Ln*iS z`vi=zy%!zw#2nz=9---AA%mq*YYJcxdKm_)E_k|H$fWo7X9cbI&Xa!|JN1=n=_{2R z?pUdgv7-X}v1iGT)#QSY9ZnO#h88BOd~EPGd4ZwQ8)WJ7VL}Uf1c9Q`M>y}jlE?Wk zjP~DbW8d^wJHnE@cNa%a4wn$xxHyB+hl&YboYe|5>Rrkj7oEia!%}WMjUoKhl1TlrW>jA?mt`}=Spk2z7qx0p6v|M{C z?e+RYEB)0q&uXK#R8DVY6ASnB!12@uRF*DqVo*r@km1S+eo`AFW1|p&b0d56>5U2% zf5Ixg{rpQEdDPg(Ot-%I_gXWDV1xpDIAl10f)-I^AP57&WvUB?5IUAWZWNgLA~aX_ zRY0nbnm6(09P2lG-^v|9NJQcg$|WaO@F!(S`(2C;YJTq8f-uF$%~TIkOS$ zu3WiFUaC69QAlEVytVc<_WU)c&|m8@)R*VZz5X>1NmOHx(OsPqCS=*Y@G9ZQR2v6N zgk(fHLm|)^GE9ZhDLGMc_x9SZJ+_#0dOe;TTN|zHyYKB8*i^t?5-$d7D@_jsV>mL+ z*aGoE%ZU#94$hGu8h~uIBrCE?LHX~#JXa1OpB``8|IM00SUwmM0I$3); ze0cfu+WIK<^mX1` z^&A2w_B?Uv#6brRg>IDXbwb4TAq>%%mWVV*e9Wno#>L7BdZdLk#~iVw8b^(5|MBl` zZKoT@e2Rn-do;+j0$|}JkQXx-MpU_S`-H^rxInw^QFV5M)u1a^jpC2|c0Ox`l42|? zg}8Rfd%hoR`mpx~#uk-j$dn38@`#O?jT92afJJK=@qR>giIx3`fKm+(t>pIm`ah1e z=P0Soz1tsY^;mu>NILAj`C>|sjU{PZ0OOYUtd(iO1|$Gdolt;Jm_}=FfkvDyGW*c` zsp+(o@{jTTan@|>&zfUiA%}qpr~w23g98EpfFKa;0RZ%Yk?;T@KshKK5&&Qb=nO;f z*AN2`{09UKSV;OH6<8SvOPHRi16nG50|8(VZbIh4p?RS(I8Y&jiC_udIom<#fx2!O zEI}e50VsU84AI*-*RzTCP3xG%( z8AkYp1R#LR1}#GqbEt@m0(!a(?zlCRs+=2E!t zOiCBu)ic;Jqz{5yo-wp8RyUCFheL)}g5U1OuzavJQ-w$nr3?(Qw1=qa3TESoq11W` zEPyb$BxDX-lE7i*&i8rVfEzBG04kIMUSDNMiP0FL0^td}lf9^{ifzyj+Sz1L+zAh8 zBsPJ8OM&A+X_`&o)Nw%!>TL-KML^uK8RGhVDj-r&`=#PAmKY*R;&VIt8Oj}Q2!%L+yq!I-%QCf;#@U z(@g;l7lmSYfZ=peG6TjOswf`}DZ-Lcik`7yz%ciyY*nFbBZl+Ap+mtO-oAejK?J}+ z&xVqyZ~+)VM*qvOJE!1^!TUvw!BE>(gec$wy)wjp%p!`^TB`^j-p~);*nS$eUn>R@ z$t!^g^oDYz>}Y0)fx^ybbN~Ra5Dkl>TR%Ibo8i$>i~%SA_6DR$!?Ovt0p#ewqUOMK w-fJ+H=F#VE@dWF$6&6>32))) + return tmp +} + // XL metadata constants. const ( // XL meta version. diff --git a/cmd/xl-storage-format-v2-legacy.go b/cmd/xl-storage-format-v2-legacy.go new file mode 100644 index 000000000..56e3d909e --- /dev/null +++ b/cmd/xl-storage-format-v2-legacy.go @@ -0,0 +1,89 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "fmt" + + "github.com/tinylib/msgp/msgp" +) + +// unmarshalV unmarshals with a specific header version. +func (x *xlMetaV2VersionHeader) unmarshalV(v uint8, bts []byte) (o []byte, err error) { + switch v { + case 1: + return x.unmarshalV1(bts) + case xlHeaderVersion: + return x.UnmarshalMsg(bts) + } + return bts, fmt.Errorf("unknown xlHeaderVersion: %d", v) +} + +// unmarshalV1 decodes version 1, never released. +func (x *xlMetaV2VersionHeader) unmarshalV1(bts []byte) (o []byte, err error) { + var zb0001 uint32 + zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 != 4 { + err = msgp.ArrayError{Wanted: 4, Got: zb0001} + return + } + bts, err = msgp.ReadExactBytes(bts, (x.VersionID)[:]) + if err != nil { + err = msgp.WrapError(err, "VersionID") + return + } + x.ModTime, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ModTime") + return + } + { + var zb0002 uint8 + zb0002, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Type") + return + } + x.Type = VersionType(zb0002) + } + { + var zb0003 uint8 + zb0003, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Flags") + return + } + x.Flags = xlFlags(zb0003) + } + o = bts + return +} + +// unmarshalV unmarshals with a specific metadata version. +func (j *xlMetaV2Version) unmarshalV(v uint8, bts []byte) (o []byte, err error) { + switch v { + // We accept un-set as latest version. + case 0, xlMetaVersion: + return j.UnmarshalMsg(bts) + } + return bts, fmt.Errorf("unknown xlMetaVersion: %d", v) +} diff --git a/cmd/xl-storage-format-v2.go b/cmd/xl-storage-format-v2.go index 78ab0187d..0674609c5 100644 --- a/cmd/xl-storage-format-v2.go +++ b/cmd/xl-storage-format-v2.go @@ -19,7 +19,9 @@ package cmd import ( "bytes" + "context" "encoding/binary" + "encoding/hex" "errors" "fmt" "io" @@ -46,6 +48,9 @@ var ( xlVersionCurrent [4]byte ) +//go:generate msgp -file=$GOFILE -unexported +//go:generate stringer -type VersionType -output=xl-storage-format-v2_string.go $GOFILE + const ( // Breaking changes. // Newer versions cannot be read by older software. @@ -56,7 +61,7 @@ const ( // Bumping this is informational, but should be done // if any change is made to the data stored, bumping this // will allow to detect the exact version later. - xlVersionMinor = 2 + xlVersionMinor = 3 ) func init() { @@ -64,35 +69,6 @@ func init() { binary.LittleEndian.PutUint16(xlVersionCurrent[2:4], xlVersionMinor) } -// checkXL2V1 will check if the metadata has correct header and is a known major version. -// The remaining payload and versions are returned. -func checkXL2V1(buf []byte) (payload []byte, major, minor uint16, err error) { - if len(buf) <= 8 { - return payload, 0, 0, fmt.Errorf("xlMeta: no data") - } - - if !bytes.Equal(buf[:4], xlHeader[:]) { - return payload, 0, 0, fmt.Errorf("xlMeta: unknown XLv2 header, expected %v, got %v", xlHeader[:4], buf[:4]) - } - - if bytes.Equal(buf[4:8], []byte("1 ")) { - // Set as 1,0. - major, minor = 1, 0 - } else { - major, minor = binary.LittleEndian.Uint16(buf[4:6]), binary.LittleEndian.Uint16(buf[6:8]) - } - if major > xlVersionMajor { - return buf[8:], major, minor, fmt.Errorf("xlMeta: unknown major version %d found", major) - } - - return buf[8:], major, minor, nil -} - -func isXL2V1Format(buf []byte) bool { - _, _, _, err := checkXL2V1(buf) - return err == nil -} - // The []journal contains all the different versions of the object. // // This array can have 3 kinds of objects: @@ -124,8 +100,6 @@ func isXL2V1Format(buf []byte) bool { // │ └── part.1 // └── xl.meta -//go:generate msgp -file=$GOFILE -unexported - // VersionType defines the type of journal type of the current entry. type VersionType uint8 @@ -187,26 +161,26 @@ type xlMetaV2DeleteMarker struct { // xlMetaV2Object defines the data struct for object journal type type xlMetaV2Object struct { - VersionID [16]byte `json:"ID" msg:"ID"` // Version ID - DataDir [16]byte `json:"DDir" msg:"DDir"` // Data dir ID - ErasureAlgorithm ErasureAlgo `json:"EcAlgo" msg:"EcAlgo"` // Erasure coding algorithm - ErasureM int `json:"EcM" msg:"EcM"` // Erasure data blocks - ErasureN int `json:"EcN" msg:"EcN"` // Erasure parity blocks - ErasureBlockSize int64 `json:"EcBSize" msg:"EcBSize"` // Erasure block size - ErasureIndex int `json:"EcIndex" msg:"EcIndex"` // Erasure disk index - ErasureDist []uint8 `json:"EcDist" msg:"EcDist"` // Erasure distribution - BitrotChecksumAlgo ChecksumAlgo `json:"CSumAlgo" msg:"CSumAlgo"` // Bitrot checksum algo - PartNumbers []int `json:"PartNums" msg:"PartNums"` // Part Numbers - PartETags []string `json:"PartETags" msg:"PartETags"` // Part ETags - PartSizes []int64 `json:"PartSizes" msg:"PartSizes"` // Part Sizes - PartActualSizes []int64 `json:"PartASizes,omitempty" msg:"PartASizes,omitempty"` // Part ActualSizes (compression) - Size int64 `json:"Size" msg:"Size"` // Object version size - ModTime int64 `json:"MTime" msg:"MTime"` // Object version modified time - MetaSys map[string][]byte `json:"MetaSys,omitempty" msg:"MetaSys,omitempty"` // Object version internal metadata - MetaUser map[string]string `json:"MetaUsr,omitempty" msg:"MetaUsr,omitempty"` // Object version metadata set by user + VersionID [16]byte `json:"ID" msg:"ID"` // Version ID + DataDir [16]byte `json:"DDir" msg:"DDir"` // Data dir ID + ErasureAlgorithm ErasureAlgo `json:"EcAlgo" msg:"EcAlgo"` // Erasure coding algorithm + ErasureM int `json:"EcM" msg:"EcM"` // Erasure data blocks + ErasureN int `json:"EcN" msg:"EcN"` // Erasure parity blocks + ErasureBlockSize int64 `json:"EcBSize" msg:"EcBSize"` // Erasure block size + ErasureIndex int `json:"EcIndex" msg:"EcIndex"` // Erasure disk index + ErasureDist []uint8 `json:"EcDist" msg:"EcDist"` // Erasure distribution + BitrotChecksumAlgo ChecksumAlgo `json:"CSumAlgo" msg:"CSumAlgo"` // Bitrot checksum algo + PartNumbers []int `json:"PartNums" msg:"PartNums"` // Part Numbers + PartETags []string `json:"PartETags" msg:"PartETags,allownil"` // Part ETags + PartSizes []int64 `json:"PartSizes" msg:"PartSizes"` // Part Sizes + PartActualSizes []int64 `json:"PartASizes,omitempty" msg:"PartASizes,allownil"` // Part ActualSizes (compression) + Size int64 `json:"Size" msg:"Size"` // Object version size + ModTime int64 `json:"MTime" msg:"MTime"` // Object version modified time + MetaSys map[string][]byte `json:"MetaSys,omitempty" msg:"MetaSys,allownil"` // Object version internal metadata + MetaUser map[string]string `json:"MetaUsr,omitempty" msg:"MetaUsr,allownil"` // Object version metadata set by user } -// xlMetaV2Version describes the jouranal entry, Type defines +// xlMetaV2Version describes the journal entry, Type defines // the current journal entry type other types might be nil based // on what Type field carries, it is imperative for the caller // to verify which journal type first before accessing rest of the fields. @@ -217,6 +191,84 @@ type xlMetaV2Version struct { DeleteMarker *xlMetaV2DeleteMarker `json:"DelObj,omitempty" msg:"DelObj,omitempty"` } +// xlFlags contains flags on the object. +// This can be extended up to 64 bits without breaking compatibility. +type xlFlags uint8 + +const ( + xlFlagFreeVersion xlFlags = 1 << iota + xlFlagUsesDataDir + xlFlagInlineData +) + +func (x xlFlags) String() string { + var s strings.Builder + if x&xlFlagFreeVersion != 0 { + s.WriteString("FreeVersion") + } + if x&xlFlagUsesDataDir != 0 { + if s.Len() > 0 { + s.WriteByte(',') + } + s.WriteString("UsesDD") + } + if x&xlFlagInlineData != 0 { + if s.Len() > 0 { + s.WriteByte(',') + } + s.WriteString("Inline") + } + return s.String() +} + +// checkXL2V1 will check if the metadata has correct header and is a known major version. +// The remaining payload and versions are returned. +func checkXL2V1(buf []byte) (payload []byte, major, minor uint16, err error) { + if len(buf) <= 8 { + return payload, 0, 0, fmt.Errorf("xlMeta: no data") + } + + if !bytes.Equal(buf[:4], xlHeader[:]) { + return payload, 0, 0, fmt.Errorf("xlMeta: unknown XLv2 header, expected %v, got %v", xlHeader[:4], buf[:4]) + } + + if bytes.Equal(buf[4:8], []byte("1 ")) { + // Set as 1,0. + major, minor = 1, 0 + } else { + major, minor = binary.LittleEndian.Uint16(buf[4:6]), binary.LittleEndian.Uint16(buf[6:8]) + } + if major > xlVersionMajor { + return buf[8:], major, minor, fmt.Errorf("xlMeta: unknown major version %d found", major) + } + + return buf[8:], major, minor, nil +} + +func isXL2V1Format(buf []byte) bool { + _, _, _, err := checkXL2V1(buf) + return err == nil +} + +//msgp:tuple xlMetaV2VersionHeader +type xlMetaV2VersionHeader struct { + VersionID [16]byte + ModTime int64 + Signature [4]byte + Type VersionType + Flags xlFlags +} + +func (x xlMetaV2VersionHeader) String() string { + return fmt.Sprintf("Type: %s, VersionID: %s, Signature: %s, ModTime: %s, Flags: %s", + x.Type.String(), + hex.EncodeToString(x.VersionID[:]), + hex.EncodeToString(x.Signature[:]), + time.Unix(0, x.ModTime), + x.Flags.String(), + ) +} + // Valid xl meta xlMetaV2Version is valid func (j xlMetaV2Version) Valid() bool { if !j.Type.valid() { @@ -239,6 +291,61 @@ func (j xlMetaV2Version) Valid() bool { return false } +// header will return a shallow header of the version. +func (j *xlMetaV2Version) header() xlMetaV2VersionHeader { + var flags xlFlags + if j.FreeVersion() { + flags |= xlFlagFreeVersion + } + if j.Type == ObjectType && j.ObjectV2.UsesDataDir() { + flags |= xlFlagUsesDataDir + } + if j.Type == ObjectType && j.ObjectV2.InlineData() { + flags |= xlFlagInlineData + } + return xlMetaV2VersionHeader{ + VersionID: j.getVersionID(), + ModTime: j.getModTime().UnixNano(), + Signature: j.getSignature(), + Type: j.Type, + Flags: flags, + } +} + +// FreeVersion returns true if x represents a free-version, false otherwise. +func (x xlMetaV2VersionHeader) FreeVersion() bool { + return x.Flags&xlFlagFreeVersion != 0 +} + +// UsesDataDir returns true if this object version uses its data directory for +// its contents and false otherwise. +func (x xlMetaV2VersionHeader) UsesDataDir() bool { + return x.Flags&xlFlagUsesDataDir != 0 +} + +// InlineData returns whether inline data has been set. +// Note that false does not mean there is no inline data, +// only that it is unlikely. +func (x xlMetaV2VersionHeader) InlineData() bool { + return x.Flags&xlFlagInlineData != 0 +} + +// signatureErr is a signature returned when an error occurs. +var signatureErr = [4]byte{'e', 'r', 'r', 0} + +// getSignature will return a signature that is expected to be the same across all disks. +func (j xlMetaV2Version) getSignature() [4]byte { + switch j.Type { + case ObjectType: + return j.ObjectV2.Signature() + case DeleteType: + return j.DeleteMarker.Signature() + case LegacyType: + return j.ObjectV1.Signature() + } + return signatureErr +} + // getModTime will return the ModTime of the underlying version. func (j xlMetaV2Version) getModTime() time.Time { switch j.Type { @@ -252,705 +359,35 @@ func (j xlMetaV2Version) getModTime() time.Time { return time.Time{} } -// xlMetaV2 - object meta structure defines the format and list of -// the journals for the object. -type xlMetaV2 struct { - Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` - - // data will contain raw data if any. - // data will be one or more versions indexed by versionID. - // To remove all data set to nil. - data xlMetaInlineData `msg:"-"` +// getVersionID will return the versionID of the underlying version. +func (j xlMetaV2Version) getVersionID() [16]byte { + switch j.Type { + case ObjectType: + return j.ObjectV2.VersionID + case DeleteType: + return j.DeleteMarker.VersionID + case LegacyType: + return [16]byte{} + } + return [16]byte{} } -// xlMetaInlineData is serialized data in [string][]byte pairs. -// -//msgp:ignore xlMetaInlineData -type xlMetaInlineData []byte - -// xlMetaInlineDataVer indicates the version of the inline data structure. -const xlMetaInlineDataVer = 1 - -// versionOK returns whether the version is ok. -func (x xlMetaInlineData) versionOK() bool { - if len(x) == 0 { - return true +func (j *xlMetaV2Version) ToFileInfo(volume, path string) (FileInfo, error) { + switch j.Type { + case ObjectType: + return j.ObjectV2.ToFileInfo(volume, path) + case DeleteType: + return j.DeleteMarker.ToFileInfo(volume, path) + case LegacyType: + return j.ObjectV1.ToFileInfo(volume, path) } - return x[0] > 0 && x[0] <= xlMetaInlineDataVer + return FileInfo{}, errFileNotFound } -// afterVersion returns the payload after the version, if any. -func (x xlMetaInlineData) afterVersion() []byte { - if len(x) == 0 { - return x - } - return x[1:] -} - -// find the data with key s. -// Returns nil if not for or an error occurs. -func (x xlMetaInlineData) find(key string) []byte { - if len(x) == 0 || !x.versionOK() { - return nil - } - sz, buf, err := msgp.ReadMapHeaderBytes(x.afterVersion()) - if err != nil || sz == 0 { - return nil - } - for i := uint32(0); i < sz; i++ { - var found []byte - found, buf, err = msgp.ReadMapKeyZC(buf) - if err != nil || sz == 0 { - return nil - } - if string(found) == key { - val, _, _ := msgp.ReadBytesZC(buf) - return val - } - // Skip it - _, buf, err = msgp.ReadBytesZC(buf) - if err != nil { - return nil - } - } - return nil -} - -// validate checks if the data is valid. -// It does not check integrity of the stored data. -func (x xlMetaInlineData) validate() error { - if len(x) == 0 { - return nil - } - - if !x.versionOK() { - return fmt.Errorf("xlMetaInlineData: unknown version 0x%x", x[0]) - } - - sz, buf, err := msgp.ReadMapHeaderBytes(x.afterVersion()) - if err != nil { - return fmt.Errorf("xlMetaInlineData: %w", err) - } - - for i := uint32(0); i < sz; i++ { - var key []byte - key, buf, err = msgp.ReadMapKeyZC(buf) - if err != nil { - return fmt.Errorf("xlMetaInlineData: %w", err) - } - if len(key) == 0 { - return fmt.Errorf("xlMetaInlineData: key %d is length 0", i) - } - _, buf, err = msgp.ReadBytesZC(buf) - if err != nil { - return fmt.Errorf("xlMetaInlineData: %w", err) - } - } - - return nil -} - -// repair will copy all seemingly valid data entries from a corrupted set. -// This does not ensure that data is correct, but will allow all operations to complete. -func (x *xlMetaInlineData) repair() { - data := *x - if len(data) == 0 { - return - } - - if !data.versionOK() { - *x = nil - return - } - - sz, buf, err := msgp.ReadMapHeaderBytes(data.afterVersion()) - if err != nil { - *x = nil - return - } - - // Remove all current data - keys := make([][]byte, 0, sz) - vals := make([][]byte, 0, sz) - for i := uint32(0); i < sz; i++ { - var key, val []byte - key, buf, err = msgp.ReadMapKeyZC(buf) - if err != nil { - break - } - if len(key) == 0 { - break - } - val, buf, err = msgp.ReadBytesZC(buf) - if err != nil { - break - } - keys = append(keys, key) - vals = append(vals, val) - } - x.serialize(-1, keys, vals) -} - -// validate checks if the data is valid. -// It does not check integrity of the stored data. -func (x xlMetaInlineData) list() ([]string, error) { - if len(x) == 0 { - return nil, nil - } - if !x.versionOK() { - return nil, errors.New("xlMetaInlineData: unknown version") - } - - sz, buf, err := msgp.ReadMapHeaderBytes(x.afterVersion()) - if err != nil { - return nil, err - } - keys := make([]string, 0, sz) - for i := uint32(0); i < sz; i++ { - var key []byte - key, buf, err = msgp.ReadMapKeyZC(buf) - if err != nil { - return keys, err - } - if len(key) == 0 { - return keys, fmt.Errorf("xlMetaInlineData: key %d is length 0", i) - } - keys = append(keys, string(key)) - // Skip data... - _, buf, err = msgp.ReadBytesZC(buf) - if err != nil { - return keys, err - } - } - return keys, nil -} - -// serialize will serialize the provided keys and values. -// The function will panic if keys/value slices aren't of equal length. -// Payload size can give an indication of expected payload size. -// If plSize is <= 0 it will be calculated. -func (x *xlMetaInlineData) serialize(plSize int, keys [][]byte, vals [][]byte) { - if len(keys) != len(vals) { - panic(fmt.Errorf("xlMetaInlineData.serialize: keys/value number mismatch")) - } - if len(keys) == 0 { - *x = nil - return - } - if plSize <= 0 { - plSize = 1 + msgp.MapHeaderSize - for i := range keys { - plSize += len(keys[i]) + len(vals[i]) + msgp.StringPrefixSize + msgp.ArrayHeaderSize - } - } - payload := make([]byte, 1, plSize) - payload[0] = xlMetaInlineDataVer - payload = msgp.AppendMapHeader(payload, uint32(len(keys))) - for i := range keys { - payload = msgp.AppendStringFromBytes(payload, keys[i]) - payload = msgp.AppendBytes(payload, vals[i]) - } - *x = payload -} - -// entries returns the number of entries in the data. -func (x xlMetaInlineData) entries() int { - if len(x) == 0 || !x.versionOK() { - return 0 - } - sz, _, _ := msgp.ReadMapHeaderBytes(x.afterVersion()) - return int(sz) -} - -// replace will add or replace a key/value pair. -func (x *xlMetaInlineData) replace(key string, value []byte) { - in := x.afterVersion() - sz, buf, _ := msgp.ReadMapHeaderBytes(in) - keys := make([][]byte, 0, sz+1) - vals := make([][]byte, 0, sz+1) - - // Version plus header... - plSize := 1 + msgp.MapHeaderSize - replaced := false - for i := uint32(0); i < sz; i++ { - var found, foundVal []byte - var err error - found, buf, err = msgp.ReadMapKeyZC(buf) - if err != nil { - break - } - foundVal, buf, err = msgp.ReadBytesZC(buf) - if err != nil { - break - } - plSize += len(found) + msgp.StringPrefixSize + msgp.ArrayHeaderSize - keys = append(keys, found) - if string(found) == key { - vals = append(vals, value) - plSize += len(value) - replaced = true - } else { - vals = append(vals, foundVal) - plSize += len(foundVal) - } - } - - // Add one more. - if !replaced { - keys = append(keys, []byte(key)) - vals = append(vals, value) - plSize += len(key) + len(value) + msgp.StringPrefixSize + msgp.ArrayHeaderSize - } - - // Reserialize... - x.serialize(plSize, keys, vals) -} - -// rename will rename a key. -// Returns whether the key was found. -func (x *xlMetaInlineData) rename(oldKey, newKey string) bool { - in := x.afterVersion() - sz, buf, _ := msgp.ReadMapHeaderBytes(in) - keys := make([][]byte, 0, sz) - vals := make([][]byte, 0, sz) - - // Version plus header... - plSize := 1 + msgp.MapHeaderSize - found := false - for i := uint32(0); i < sz; i++ { - var foundKey, foundVal []byte - var err error - foundKey, buf, err = msgp.ReadMapKeyZC(buf) - if err != nil { - break - } - foundVal, buf, err = msgp.ReadBytesZC(buf) - if err != nil { - break - } - plSize += len(foundVal) + msgp.StringPrefixSize + msgp.ArrayHeaderSize - vals = append(vals, foundVal) - if string(foundKey) != oldKey { - keys = append(keys, foundKey) - plSize += len(foundKey) - } else { - keys = append(keys, []byte(newKey)) - plSize += len(newKey) - found = true - } - } - // If not found, just return. - if !found { - return false - } - - // Reserialize... - x.serialize(plSize, keys, vals) - return true -} - -// remove will remove one or more keys. -// Returns true if any key was found. -func (x *xlMetaInlineData) remove(keys ...string) bool { - in := x.afterVersion() - sz, buf, _ := msgp.ReadMapHeaderBytes(in) - newKeys := make([][]byte, 0, sz) - newVals := make([][]byte, 0, sz) - var removeKey func(s []byte) bool - - // Copy if big number of compares... - if len(keys) > 5 && sz > 5 { - mKeys := make(map[string]struct{}, len(keys)) - for _, key := range keys { - mKeys[key] = struct{}{} - } - removeKey = func(s []byte) bool { - _, ok := mKeys[string(s)] - return ok - } - } else { - removeKey = func(s []byte) bool { - for _, key := range keys { - if key == string(s) { - return true - } - } - return false - } - } - - // Version plus header... - plSize := 1 + msgp.MapHeaderSize - found := false - for i := uint32(0); i < sz; i++ { - var foundKey, foundVal []byte - var err error - foundKey, buf, err = msgp.ReadMapKeyZC(buf) - if err != nil { - break - } - foundVal, buf, err = msgp.ReadBytesZC(buf) - if err != nil { - break - } - if !removeKey(foundKey) { - plSize += msgp.StringPrefixSize + msgp.ArrayHeaderSize + len(foundKey) + len(foundVal) - newKeys = append(newKeys, foundKey) - newVals = append(newVals, foundVal) - } else { - found = true - } - } - // If not found, just return. - if !found { - return false - } - // If none left... - if len(newKeys) == 0 { - *x = nil - return true - } - - // Reserialize... - x.serialize(plSize, newKeys, newVals) - return true -} - -// xlMetaV2TrimData will trim any data from the metadata without unmarshalling it. -// If any error occurs the unmodified data is returned. -func xlMetaV2TrimData(buf []byte) []byte { - metaBuf, min, maj, err := checkXL2V1(buf) - if err != nil { - return buf - } - if maj == 1 && min < 1 { - // First version to carry data. - return buf - } - // Skip header - _, metaBuf, err = msgp.ReadBytesZC(metaBuf) - if err != nil { - logger.LogIf(GlobalContext, err) - return buf - } - // Skip CRC - if maj > 1 || min >= 2 { - _, metaBuf, err = msgp.ReadUint32Bytes(metaBuf) - logger.LogIf(GlobalContext, err) - } - // = input - current pos - ends := len(buf) - len(metaBuf) - if ends > len(buf) { - return buf - } - - return buf[:ends] -} - -// AddLegacy adds a legacy version, is only called when no prior -// versions exist, safe to use it by only one function in xl-storage(RenameData) -func (z *xlMetaV2) AddLegacy(m *xlMetaV1Object) error { - if !m.valid() { - return errFileCorrupt - } - m.VersionID = nullVersionID - m.DataDir = legacyDataDir - z.Versions = []xlMetaV2Version{ - { - Type: LegacyType, - ObjectV1: m, - }, - } - return nil -} - -// Load unmarshal and load the entire message pack. -// Note that references to the incoming buffer may be kept as data. -func (z *xlMetaV2) Load(buf []byte) error { - buf, major, minor, err := checkXL2V1(buf) - if err != nil { - return fmt.Errorf("xlMetaV2.Load %w", err) - } - switch major { - case 1: - switch minor { - case 0: - _, err = z.UnmarshalMsg(buf) - if err != nil { - return fmt.Errorf("xlMetaV2.Load %w", err) - } - return nil - case 1, 2: - v, buf, err := msgp.ReadBytesZC(buf) - if err != nil { - return fmt.Errorf("xlMetaV2.Load version(%d), bufLen(%d) %w", minor, len(buf), err) - } - if minor >= 2 { - if crc, nbuf, err := msgp.ReadUint32Bytes(buf); err == nil { - // Read metadata CRC (added in v2) - buf = nbuf - if got := uint32(xxhash.Sum64(v)); got != crc { - return fmt.Errorf("xlMetaV2.Load version(%d), CRC mismatch, want 0x%x, got 0x%x", minor, crc, got) - } - } else { - return fmt.Errorf("xlMetaV2.Load version(%d), loading CRC: %w", minor, err) - } - } - - if _, err = z.UnmarshalMsg(v); err != nil { - return fmt.Errorf("xlMetaV2.Load version(%d), vLen(%d), %w", minor, len(v), err) - } - // Add remaining data. - z.data = buf - if err = z.data.validate(); err != nil { - z.data.repair() - logger.Info("xlMetaV2.Load: data validation failed: %v. %d entries after repair", err, z.data.entries()) - } - default: - return errors.New("unknown minor metadata version") - } - default: - return errors.New("unknown major metadata version") - } - return nil -} - -// AppendTo will marshal the data in z and append it to the provided slice. -func (z *xlMetaV2) AppendTo(dst []byte) ([]byte, error) { - sz := len(xlHeader) + len(xlVersionCurrent) + msgp.ArrayHeaderSize + z.Msgsize() + len(z.data) + len(dst) + msgp.Uint32Size - if cap(dst) < sz { - buf := make([]byte, len(dst), sz) - copy(buf, dst) - dst = buf - } - if err := z.data.validate(); err != nil { - return nil, err - } - - dst = append(dst, xlHeader[:]...) - dst = append(dst, xlVersionCurrent[:]...) - // Add "bin 32" type header to always have enough space. - // We will fill out the correct size when we know it. - dst = append(dst, 0xc6, 0, 0, 0, 0) - dataOffset := len(dst) - dst, err := z.MarshalMsg(dst) - if err != nil { - return nil, err - } - - // Update size... - binary.BigEndian.PutUint32(dst[dataOffset-4:dataOffset], uint32(len(dst)-dataOffset)) - - // Add CRC of metadata. - dst = msgp.AppendUint32(dst, uint32(xxhash.Sum64(dst[dataOffset:]))) - return append(dst, z.data...), nil -} - -// UpdateObjectVersion updates metadata and modTime for a given -// versionID, NOTE: versionID must be valid and should exist - -// and must not be a DeleteMarker or legacy object, if no -// versionID is specified 'null' versionID is updated instead. -// -// It is callers responsibility to set correct versionID, this -// function shouldn't be further extended to update immutable -// values such as ErasureInfo, ChecksumInfo. -// -// Metadata is only updated to new values, existing values -// stay as is, if you wish to update all values you should -// update all metadata freshly before calling this function -// in-case you wish to clear existing metadata. -func (z *xlMetaV2) UpdateObjectVersion(fi FileInfo) error { - if fi.VersionID == "" { - // this means versioning is not yet - // enabled or suspend i.e all versions - // are basically default value i.e "null" - fi.VersionID = nullVersionID - } - - var uv uuid.UUID - var err error - if fi.VersionID != "" && fi.VersionID != nullVersionID { - uv, err = uuid.Parse(fi.VersionID) - if err != nil { - return err - } - } - - for i, version := range z.Versions { - if !version.Valid() { - return errFileCorrupt - } - switch version.Type { - case LegacyType: - if version.ObjectV1.VersionID == fi.VersionID { - return errMethodNotAllowed - } - case ObjectType: - if version.ObjectV2.VersionID == uv { - for k, v := range fi.Metadata { - if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) { - z.Versions[i].ObjectV2.MetaSys[k] = []byte(v) - } else { - z.Versions[i].ObjectV2.MetaUser[k] = v - } - } - if !fi.ModTime.IsZero() { - z.Versions[i].ObjectV2.ModTime = fi.ModTime.UnixNano() - } - return nil - } - case DeleteType: - if version.DeleteMarker.VersionID == uv { - return errMethodNotAllowed - } - } - } - - return errFileVersionNotFound -} - -// AddVersion adds a new version -func (z *xlMetaV2) AddVersion(fi FileInfo) error { - if fi.VersionID == "" { - // this means versioning is not yet - // enabled or suspend i.e all versions - // are basically default value i.e "null" - fi.VersionID = nullVersionID - } - - var uv uuid.UUID - var err error - if fi.VersionID != "" && fi.VersionID != nullVersionID { - uv, err = uuid.Parse(fi.VersionID) - if err != nil { - return err - } - } - - var dd uuid.UUID - if fi.DataDir != "" { - dd, err = uuid.Parse(fi.DataDir) - if err != nil { - return err - } - } - - ventry := xlMetaV2Version{} - - if fi.Deleted { - ventry.Type = DeleteType - ventry.DeleteMarker = &xlMetaV2DeleteMarker{ - VersionID: uv, - ModTime: fi.ModTime.UnixNano(), - MetaSys: make(map[string][]byte), - } - } else { - ventry.Type = ObjectType - ventry.ObjectV2 = &xlMetaV2Object{ - VersionID: uv, - DataDir: dd, - Size: fi.Size, - ModTime: fi.ModTime.UnixNano(), - ErasureAlgorithm: ReedSolomon, - ErasureM: fi.Erasure.DataBlocks, - ErasureN: fi.Erasure.ParityBlocks, - ErasureBlockSize: fi.Erasure.BlockSize, - ErasureIndex: fi.Erasure.Index, - BitrotChecksumAlgo: HighwayHash, - ErasureDist: make([]uint8, len(fi.Erasure.Distribution)), - PartNumbers: make([]int, len(fi.Parts)), - PartETags: make([]string, len(fi.Parts)), - PartSizes: make([]int64, len(fi.Parts)), - PartActualSizes: make([]int64, len(fi.Parts)), - MetaSys: make(map[string][]byte), - MetaUser: make(map[string]string, len(fi.Metadata)), - } - - for i := range fi.Erasure.Distribution { - ventry.ObjectV2.ErasureDist[i] = uint8(fi.Erasure.Distribution[i]) - } - - for i := range fi.Parts { - ventry.ObjectV2.PartSizes[i] = fi.Parts[i].Size - if fi.Parts[i].ETag != "" { - ventry.ObjectV2.PartETags[i] = fi.Parts[i].ETag - } - ventry.ObjectV2.PartNumbers[i] = fi.Parts[i].Number - ventry.ObjectV2.PartActualSizes[i] = fi.Parts[i].ActualSize - } - - tierFVIDKey := ReservedMetadataPrefixLower + tierFVID - tierFVMarkerKey := ReservedMetadataPrefixLower + tierFVMarker - for k, v := range fi.Metadata { - if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) { - // Skip tierFVID, tierFVMarker keys; it's used - // only for creating free-version. - switch k { - case tierFVIDKey, tierFVMarkerKey: - continue - } - - ventry.ObjectV2.MetaSys[k] = []byte(v) - } else { - ventry.ObjectV2.MetaUser[k] = v - } - } - - // If asked to save data. - if len(fi.Data) > 0 || fi.Size == 0 { - z.data.replace(fi.VersionID, fi.Data) - } - - if fi.TransitionStatus != "" { - ventry.ObjectV2.MetaSys[ReservedMetadataPrefixLower+TransitionStatus] = []byte(fi.TransitionStatus) - } - if fi.TransitionedObjName != "" { - ventry.ObjectV2.MetaSys[ReservedMetadataPrefixLower+TransitionedObjectName] = []byte(fi.TransitionedObjName) - } - if fi.TransitionVersionID != "" { - ventry.ObjectV2.MetaSys[ReservedMetadataPrefixLower+TransitionedVersionID] = []byte(fi.TransitionVersionID) - } - if fi.TransitionTier != "" { - ventry.ObjectV2.MetaSys[ReservedMetadataPrefixLower+TransitionTier] = []byte(fi.TransitionTier) - } - } - - if !ventry.Valid() { - return errors.New("internal error: invalid version entry generated") - } - - for i, version := range z.Versions { - if !version.Valid() { - return errFileCorrupt - } - switch version.Type { - case LegacyType: - // This would convert legacy type into new ObjectType - // this means that we are basically purging the `null` - // version of the object. - if version.ObjectV1.VersionID == fi.VersionID { - z.Versions[i] = ventry - return nil - } - case ObjectType: - if version.ObjectV2.VersionID == uv { - z.Versions[i] = ventry - return nil - } - case DeleteType: - // Allowing delete marker to replaced with an proper - // object data type as well, this is not S3 complaint - // behavior but kept here for future flexibility. - if version.DeleteMarker.VersionID == uv { - z.Versions[i] = ventry - return nil - } - } - } - - z.Versions = append(z.Versions, ventry) - return nil -} +const ( + xlHeaderVersion = 2 + xlMetaVersion = 1 +) func (j xlMetaV2DeleteMarker) ToFileInfo(volume, path string) (FileInfo, error) { versionID := "" @@ -978,6 +415,25 @@ func (j xlMetaV2DeleteMarker) ToFileInfo(volume, path string) (FileInfo, error) return fi, nil } +// Signature will return a signature that is expected to be the same across all disks. +func (j *xlMetaV2DeleteMarker) Signature() [4]byte { + // Shallow copy + c := *j + + // Marshal metadata + crc := hashDeterministicBytes(c.MetaSys) + c.MetaSys = nil + if bts, err := c.MarshalMsg(metaDataPoolGet()); err == nil { + crc ^= xxhash.Sum64(bts) + metaDataPoolPut(bts) + } + + // Combine upper and lower part + var tmp [4]byte + binary.LittleEndian.PutUint32(tmp[:], uint32(crc^(crc>>32))) + return tmp +} + // UsesDataDir returns true if this object version uses its data directory for // its contents and false otherwise. func (j xlMetaV2Object) UsesDataDir() bool { @@ -990,6 +446,14 @@ func (j xlMetaV2Object) UsesDataDir() bool { return isRestoredObjectOnDisk(j.MetaUser) } +// InlineData returns whether inline data has been set. +// Note that false does not mean there is no inline data, +// only that it is unlikely. +func (j xlMetaV2Object) InlineData() bool { + _, ok := j.MetaSys[ReservedMetadataPrefixLower+"inline-data"] + return ok +} + func (j *xlMetaV2Object) SetTransition(fi FileInfo) { j.MetaSys[ReservedMetadataPrefixLower+TransitionStatus] = []byte(fi.TransitionStatus) j.MetaSys[ReservedMetadataPrefixLower+TransitionedObjectName] = []byte(fi.TransitionedObjName) @@ -1003,6 +467,40 @@ func (j *xlMetaV2Object) RemoveRestoreHdrs() { delete(j.MetaUser, xhttp.AmzRestoreRequestDate) } +// Signature will return a signature that is expected to be the same across all disks. +func (j *xlMetaV2Object) Signature() [4]byte { + // Shallow copy + c := *j + // Zero fields that will vary across disks + c.ErasureIndex = 0 + + // Nil 0 size allownil, so we don't differentiate between nil and 0 len. + if len(c.PartETags) == 0 { + c.PartETags = nil + } + if len(c.PartActualSizes) == 0 { + c.PartActualSizes = nil + } + + // Get a 64 bit CRC + crc := hashDeterministicString(c.MetaUser) + crc ^= hashDeterministicBytes(c.MetaSys) + + // Nil fields. + c.MetaSys = nil + c.MetaUser = nil + + if bts, err := c.MarshalMsg(metaDataPoolGet()); err == nil { + crc ^= xxhash.Sum64(bts) + metaDataPoolPut(bts) + } + + // Combine upper and lower part + var tmp [4]byte + binary.LittleEndian.PutUint32(tmp[:], uint32(crc^(crc>>32))) + return tmp +} + func (j xlMetaV2Object) ToFileInfo(volume, path string) (FileInfo, error) { versionID := "" var uv uuid.UUID @@ -1021,7 +519,9 @@ func (j xlMetaV2Object) ToFileInfo(volume, path string) (FileInfo, error) { for i := range fi.Parts { fi.Parts[i].Number = j.PartNumbers[i] fi.Parts[i].Size = j.PartSizes[i] - fi.Parts[i].ETag = j.PartETags[i] + if len(j.PartETags) > 0 { + fi.Parts[i].ETag = j.PartETags[i] + } fi.Parts[i].ActualSize = j.PartActualSizes[i] } fi.Erasure.Checksums = make([]ChecksumInfo, len(j.PartSizes)) @@ -1081,370 +581,6 @@ func (j xlMetaV2Object) ToFileInfo(volume, path string) (FileInfo, error) { return fi, nil } -func (z *xlMetaV2) SharedDataDirCountStr(versionID, dataDir string) int { - var ( - uv uuid.UUID - ddir uuid.UUID - err error - ) - if versionID == nullVersionID { - versionID = "" - } - if versionID != "" { - uv, err = uuid.Parse(versionID) - if err != nil { - return 0 - } - } - ddir, err = uuid.Parse(dataDir) - if err != nil { - return 0 - } - return z.SharedDataDirCount(uv, ddir) -} - -func (z *xlMetaV2) SharedDataDirCount(versionID [16]byte, dataDir [16]byte) int { - // v2 object is inlined, if it is skip dataDir share check. - if z.data.find(uuid.UUID(versionID).String()) != nil { - return 0 - } - var sameDataDirCount int - for _, version := range z.Versions { - switch version.Type { - case ObjectType: - if version.ObjectV2.VersionID == versionID { - continue - } - if version.ObjectV2.DataDir != dataDir { - continue - } - if version.ObjectV2.UsesDataDir() { - sameDataDirCount++ - } - } - } - return sameDataDirCount -} - -// DeleteVersion deletes the version specified by version id. -// returns to the caller which dataDir to delete, also -// indicates if this is the last version. -func (z *xlMetaV2) DeleteVersion(fi FileInfo) (string, bool, error) { - // This is a situation where versionId is explicitly - // specified as "null", as we do not save "null" - // string it is considered empty. But empty also - // means the version which matches will be purged. - if fi.VersionID == nullVersionID { - fi.VersionID = "" - } - - var uv uuid.UUID - var err error - if fi.VersionID != "" { - uv, err = uuid.Parse(fi.VersionID) - if err != nil { - return "", false, errFileVersionNotFound - } - } - - var ventry xlMetaV2Version - if fi.Deleted { - ventry = xlMetaV2Version{ - Type: DeleteType, - DeleteMarker: &xlMetaV2DeleteMarker{ - VersionID: uv, - ModTime: fi.ModTime.UnixNano(), - MetaSys: make(map[string][]byte), - }, - } - if !ventry.Valid() { - return "", false, errors.New("internal error: invalid version entry generated") - } - } - updateVersion := false - if fi.VersionPurgeStatus().Empty() && (fi.DeleteMarkerReplicationStatus() == "REPLICA" || fi.DeleteMarkerReplicationStatus().Empty()) { - updateVersion = fi.MarkDeleted - } else { - // for replication scenario - if fi.Deleted && fi.VersionPurgeStatus() != Complete { - if !fi.VersionPurgeStatus().Empty() || fi.DeleteMarkerReplicationStatus().Empty() { - updateVersion = true - } - } - // object or delete-marker versioned delete is not complete - if !fi.VersionPurgeStatus().Empty() && fi.VersionPurgeStatus() != Complete { - updateVersion = true - } - } - - if fi.Deleted { - if !fi.DeleteMarkerReplicationStatus().Empty() { - switch fi.DeleteMarkerReplicationStatus() { - case replication.Replica: - ventry.DeleteMarker.MetaSys[ReservedMetadataPrefixLower+ReplicaStatus] = []byte(string(fi.ReplicationState.ReplicaStatus)) - ventry.DeleteMarker.MetaSys[ReservedMetadataPrefixLower+ReplicaTimestamp] = []byte(fi.ReplicationState.ReplicaTimeStamp.Format(http.TimeFormat)) - default: - ventry.DeleteMarker.MetaSys[ReservedMetadataPrefixLower+ReplicationStatus] = []byte(fi.ReplicationState.ReplicationStatusInternal) - ventry.DeleteMarker.MetaSys[ReservedMetadataPrefixLower+ReplicationTimestamp] = []byte(fi.ReplicationState.ReplicationTimeStamp.Format(http.TimeFormat)) - } - } - if !fi.VersionPurgeStatus().Empty() { - ventry.DeleteMarker.MetaSys[VersionPurgeStatusKey] = []byte(fi.ReplicationState.VersionPurgeStatusInternal) - } - for k, v := range fi.ReplicationState.ResetStatusesMap { - ventry.DeleteMarker.MetaSys[k] = []byte(v) - } - } - - for i, version := range z.Versions { - if !version.Valid() { - return "", false, errFileCorrupt - } - switch version.Type { - case LegacyType: - if version.ObjectV1.VersionID == fi.VersionID { - z.Versions = append(z.Versions[:i], z.Versions[i+1:]...) - if fi.Deleted { - z.Versions = append(z.Versions, ventry) - } - return version.ObjectV1.DataDir, len(z.Versions) == 0, nil - } - case DeleteType: - if version.DeleteMarker.VersionID == uv { - if updateVersion { - if len(z.Versions[i].DeleteMarker.MetaSys) == 0 { - z.Versions[i].DeleteMarker.MetaSys = make(map[string][]byte) - } - if !fi.DeleteMarkerReplicationStatus().Empty() { - switch fi.DeleteMarkerReplicationStatus() { - case replication.Replica: - z.Versions[i].DeleteMarker.MetaSys[ReservedMetadataPrefixLower+ReplicaStatus] = []byte(string(fi.ReplicationState.ReplicaStatus)) - z.Versions[i].DeleteMarker.MetaSys[ReservedMetadataPrefixLower+ReplicaTimestamp] = []byte(fi.ReplicationState.ReplicaTimeStamp.Format(http.TimeFormat)) - default: - z.Versions[i].DeleteMarker.MetaSys[ReservedMetadataPrefixLower+ReplicationStatus] = []byte(fi.ReplicationState.ReplicationStatusInternal) - z.Versions[i].DeleteMarker.MetaSys[ReservedMetadataPrefixLower+ReplicationTimestamp] = []byte(fi.ReplicationState.ReplicationTimeStamp.Format(http.TimeFormat)) - } - } - if !fi.VersionPurgeStatus().Empty() { - z.Versions[i].DeleteMarker.MetaSys[VersionPurgeStatusKey] = []byte(fi.ReplicationState.VersionPurgeStatusInternal) - } - for k, v := range fi.ReplicationState.ResetStatusesMap { - z.Versions[i].DeleteMarker.MetaSys[k] = []byte(v) - } - } else { - z.Versions = append(z.Versions[:i], z.Versions[i+1:]...) - if fi.MarkDeleted && (fi.VersionPurgeStatus().Empty() || (fi.VersionPurgeStatus() != Complete)) { - z.Versions = append(z.Versions, ventry) - } - } - return "", len(z.Versions) == 0, nil - } - case ObjectType: - if version.ObjectV2.VersionID == uv && updateVersion { - z.Versions[i].ObjectV2.MetaSys[VersionPurgeStatusKey] = []byte(fi.ReplicationState.VersionPurgeStatusInternal) - for k, v := range fi.ReplicationState.ResetStatusesMap { - z.Versions[i].ObjectV2.MetaSys[k] = []byte(v) - } - return "", len(z.Versions) == 0, nil - } - } - } - - for i, version := range z.Versions { - if !version.Valid() { - return "", false, errFileCorrupt - } - switch version.Type { - case ObjectType: - if version.ObjectV2.VersionID == uv { - switch { - case fi.ExpireRestored: - z.Versions[i].ObjectV2.RemoveRestoreHdrs() - - case fi.TransitionStatus == lifecycle.TransitionComplete: - z.Versions[i].ObjectV2.SetTransition(fi) - - default: - z.Versions = append(z.Versions[:i], z.Versions[i+1:]...) - // if uv has tiered content we add a - // free-version to track it for - // asynchronous deletion via scanner. - if freeVersion, toFree := version.ObjectV2.InitFreeVersion(fi); toFree { - z.Versions = append(z.Versions, freeVersion) - } - } - - if fi.Deleted { - z.Versions = append(z.Versions, ventry) - } - if z.SharedDataDirCount(version.ObjectV2.VersionID, version.ObjectV2.DataDir) > 0 { - // Found that another version references the same dataDir - // we shouldn't remove it, and only remove the version instead - return "", len(z.Versions) == 0, nil - } - return uuid.UUID(version.ObjectV2.DataDir).String(), len(z.Versions) == 0, nil - } - } - } - - if fi.Deleted { - z.Versions = append(z.Versions, ventry) - return "", false, nil - } - return "", false, errFileVersionNotFound -} - -// TotalSize returns the total size of all versions. -func (z xlMetaV2) TotalSize() int64 { - var total int64 - for i := range z.Versions { - switch z.Versions[i].Type { - case ObjectType: - total += z.Versions[i].ObjectV2.Size - case LegacyType: - total += z.Versions[i].ObjectV1.Stat.Size - } - } - return total -} - -// ListVersions lists current versions, and current deleted -// versions returns error for unexpected entries. -// showPendingDeletes is set to true if ListVersions needs to list objects marked deleted -// but waiting to be replicated -func (z xlMetaV2) ListVersions(volume, path string) ([]FileInfo, time.Time, error) { - versions := make([]FileInfo, 0, len(z.Versions)) - var err error - - for _, version := range z.Versions { - if !version.Valid() { - return nil, time.Time{}, errFileCorrupt - } - var fi FileInfo - switch version.Type { - case ObjectType: - fi, err = version.ObjectV2.ToFileInfo(volume, path) - case DeleteType: - fi, err = version.DeleteMarker.ToFileInfo(volume, path) - case LegacyType: - fi, err = version.ObjectV1.ToFileInfo(volume, path) - } - if err != nil { - return nil, time.Time{}, err - } - versions = append(versions, fi) - } - - versionsSorter(versions).sort() - - for i := range versions { - versions[i].NumVersions = len(versions) - if i > 0 { - versions[i].SuccessorModTime = versions[i-1].ModTime - } - } - - versions[0].IsLatest = true - return versions, versions[0].ModTime, nil -} - -// ToFileInfo converts xlMetaV2 into a common FileInfo datastructure -// for consumption across callers. -func (z xlMetaV2) ToFileInfo(volume, path, versionID string) (fi FileInfo, err error) { - var uv uuid.UUID - if versionID != "" && versionID != nullVersionID { - uv, err = uuid.Parse(versionID) - if err != nil { - logger.LogIf(GlobalContext, fmt.Errorf("invalid versionID specified %s", versionID)) - return FileInfo{}, errFileVersionNotFound - } - } - - orderedVersions := make([]xlMetaV2Version, 0, len(z.Versions)) - for _, version := range z.Versions { - if !version.Valid() { - logger.LogIf(GlobalContext, fmt.Errorf("invalid version detected %#v", version)) - if versionID == "" { - return FileInfo{}, errFileNotFound - } - return FileInfo{}, errFileVersionNotFound - - } - // skip listing free-version unless explicitly requested via versionID - if version.FreeVersion() && version.DeleteMarker.VersionID != uv { - continue - } - orderedVersions = append(orderedVersions, version) - } - - if len(orderedVersions) > 1 { - sort.Slice(orderedVersions, func(i, j int) bool { - return orderedVersions[i].getModTime().After(orderedVersions[j].getModTime()) - }) - } - - if versionID == "" { - if len(orderedVersions) >= 1 { - switch orderedVersions[0].Type { - case ObjectType: - fi, err = orderedVersions[0].ObjectV2.ToFileInfo(volume, path) - case DeleteType: - fi, err = orderedVersions[0].DeleteMarker.ToFileInfo(volume, path) - case LegacyType: - fi, err = orderedVersions[0].ObjectV1.ToFileInfo(volume, path) - } - fi.IsLatest = true - fi.NumVersions = len(orderedVersions) - return fi, err - } - return FileInfo{}, errFileNotFound - } - - var foundIndex = -1 - - for i := range orderedVersions { - switch orderedVersions[i].Type { - case ObjectType: - if orderedVersions[i].ObjectV2.VersionID == uv { - fi, err = orderedVersions[i].ObjectV2.ToFileInfo(volume, path) - foundIndex = i - break - } - case LegacyType: - if orderedVersions[i].ObjectV1.VersionID == versionID { - fi, err = orderedVersions[i].ObjectV1.ToFileInfo(volume, path) - foundIndex = i - break - } - case DeleteType: - if orderedVersions[i].DeleteMarker.VersionID == uv { - fi, err = orderedVersions[i].DeleteMarker.ToFileInfo(volume, path) - foundIndex = i - break - } - } - } - if err != nil { - return fi, err - } - - if foundIndex >= 0 { - // A version is found, fill dynamic fields - fi.IsLatest = foundIndex == 0 - fi.NumVersions = len(z.Versions) - if foundIndex > 0 { - fi.SuccessorModTime = orderedVersions[foundIndex-1].getModTime() - } - return fi, nil - } - - if versionID == "" { - return FileInfo{}, errFileNotFound - } - - return FileInfo{}, errFileVersionNotFound -} - // Read at most this much on initial read. const metaDataReadDefault = 4 << 10 @@ -1511,7 +647,7 @@ func readXLMetaNoData(r io.Reader, size int64) ([]byte, error) { case 0: err = readMore(size) return buf, err - case 1, 2: + case 1, 2, 3: sz, tmp, err := msgp.ReadBytesHeader(tmp) if err != nil { return nil, err @@ -1551,3 +687,1051 @@ func readXLMetaNoData(r io.Reader, size int64) ([]byte, error) { return nil, errors.New("unknown major metadata version") } } + +func decodeXLHeaders(buf []byte) (versions int, headerV, metaV uint8, b []byte, err error) { + hdrVer, buf, err := msgp.ReadUint8Bytes(buf) + if err != nil { + return 0, 0, 0, buf, err + } + metaVer, buf, err := msgp.ReadUint8Bytes(buf) + if err != nil { + return 0, 0, 0, buf, err + } + if hdrVer > xlHeaderVersion { + return 0, 0, 0, buf, fmt.Errorf("decodeXLHeaders: Unknown xl header version %d", metaVer) + } + if metaVer > xlMetaVersion { + return 0, 0, 0, buf, fmt.Errorf("decodeXLHeaders: Unknown xl meta version %d", metaVer) + } + versions, buf, err = msgp.ReadIntBytes(buf) + if err != nil { + return 0, 0, 0, buf, err + } + if versions < 0 { + return 0, 0, 0, buf, fmt.Errorf("decodeXLHeaders: Negative version count %d", versions) + } + return versions, hdrVer, metaVer, buf, nil +} + +// decodeVersions will decode a number of versions from a buffer +// and perform a callback for each version in order, newest first. +// Return errDoneForNow to stop processing and return nil. +// Any non-nil error is returned. +func decodeVersions(buf []byte, versions int, fn func(idx int, hdr, meta []byte) error) (err error) { + var tHdr, tMeta []byte // Zero copy bytes + for i := 0; i < versions; i++ { + tHdr, buf, err = msgp.ReadBytesZC(buf) + if err != nil { + return err + } + tMeta, buf, err = msgp.ReadBytesZC(buf) + if err != nil { + return err + } + if err = fn(i, tHdr, tMeta); err != nil { + if err == errDoneForNow { + err = nil + } + return err + } + } + return nil +} + +// isIndexedMetaV2 returns non-nil result if metadata is indexed. +// If data doesn't validate nil is also returned. +func isIndexedMetaV2(buf []byte) (meta xlMetaBuf, data xlMetaInlineData) { + buf, major, minor, err := checkXL2V1(buf) + if err != nil { + return nil, nil + } + if major != 1 || minor < 3 { + return nil, nil + } + meta, buf, err = msgp.ReadBytesZC(buf) + if err != nil { + return nil, nil + } + if crc, nbuf, err := msgp.ReadUint32Bytes(buf); err == nil { + // Read metadata CRC + buf = nbuf + if got := uint32(xxhash.Sum64(meta)); got != crc { + return nil, nil + } + } else { + return nil, nil + } + data = buf + if data.validate() != nil { + data.repair() + } + + return meta, data +} + +type xlMetaV2ShallowVersion struct { + header xlMetaV2VersionHeader + meta []byte +} + +//msgp:ignore xlMetaV2 xlMetaV2ShallowVersion + +type xlMetaV2 struct { + versions []xlMetaV2ShallowVersion + + // data will contain raw data if any. + // data will be one or more versions indexed by versionID. + // To remove all data set to nil. + data xlMetaInlineData + + // metadata version. + metaV uint8 +} + +// Load all versions of the stored data. +// Note that references to the incoming buffer will be kept. +func (x *xlMetaV2) Load(buf []byte) error { + if meta, data := isIndexedMetaV2(buf); meta != nil { + return x.loadIndexed(meta, data) + } + // Convert older format. + return x.loadLegacy(buf) +} + +func (x *xlMetaV2) loadIndexed(buf xlMetaBuf, data xlMetaInlineData) error { + versions, headerV, metaV, buf, err := decodeXLHeaders(buf) + if err != nil { + return err + } + if cap(x.versions) < versions { + x.versions = make([]xlMetaV2ShallowVersion, 0, versions+1) + } + x.versions = x.versions[:versions] + x.data = data + x.metaV = metaV + if err = x.data.validate(); err != nil { + x.data.repair() + logger.Info("xlMetaV2.loadIndexed: data validation failed: %v. %d entries after repair", err, x.data.entries()) + } + + return decodeVersions(buf, versions, func(i int, hdr, meta []byte) error { + ver := &x.versions[i] + _, err = ver.header.unmarshalV(headerV, hdr) + if err != nil { + return err + } + ver.meta = meta + return nil + }) +} + +// loadLegacy will load content prior to v1.3 +// Note that references to the incoming buffer will be kept. +func (x *xlMetaV2) loadLegacy(buf []byte) error { + buf, major, minor, err := checkXL2V1(buf) + if err != nil { + return fmt.Errorf("xlMetaV2.Load %w", err) + } + var allMeta []byte + switch major { + case 1: + switch minor { + case 0: + allMeta = buf + case 1, 2: + v, buf, err := msgp.ReadBytesZC(buf) + if err != nil { + return fmt.Errorf("xlMetaV2.Load version(%d), bufLen(%d) %w", minor, len(buf), err) + } + if minor >= 2 { + if crc, nbuf, err := msgp.ReadUint32Bytes(buf); err == nil { + // Read metadata CRC (added in v2) + buf = nbuf + if got := uint32(xxhash.Sum64(v)); got != crc { + return fmt.Errorf("xlMetaV2.Load version(%d), CRC mismatch, want 0x%x, got 0x%x", minor, crc, got) + } + } else { + return fmt.Errorf("xlMetaV2.Load version(%d), loading CRC: %w", minor, err) + } + } + + allMeta = v + // Add remaining data. + x.data = buf + if err = x.data.validate(); err != nil { + x.data.repair() + logger.Info("xlMetaV2.Load: data validation failed: %v. %d entries after repair", err, x.data.entries()) + } + default: + return errors.New("unknown minor metadata version") + } + default: + return errors.New("unknown major metadata version") + } + if allMeta == nil { + return errCorruptedFormat + } + // bts will shrink as we decode. + bts := allMeta + var field []byte + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + return msgp.WrapError(err, "loadLegacy.ReadMapHeader") + } + + var tmp xlMetaV2Version + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + return msgp.WrapError(err, "loadLegacy.ReadMapKey") + } + switch msgp.UnsafeString(field) { + case "Versions": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + return msgp.WrapError(err, "Versions") + } + if cap(x.versions) >= int(zb0002) { + x.versions = (x.versions)[:zb0002] + } else { + x.versions = make([]xlMetaV2ShallowVersion, zb0002, zb0002+1) + } + for za0001 := range x.versions { + start := len(allMeta) - len(bts) + bts, err = tmp.unmarshalV(1, bts) + if err != nil { + return msgp.WrapError(err, "Versions", za0001) + } + end := len(allMeta) - len(bts) + // We reference the marshaled data, so we don't have to re-marshal. + x.versions[za0001] = xlMetaV2ShallowVersion{ + header: tmp.header(), + meta: allMeta[start:end], + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + return msgp.WrapError(err, "loadLegacy.Skip") + } + } + } + x.metaV = 1 // Fixed for legacy conversions. + x.sortByModTime() + return nil +} + +func (x *xlMetaV2) addVersion(ver xlMetaV2Version) error { + modTime := ver.getModTime().UnixNano() + if !ver.Valid() { + return errors.New("attempted to add invalid version") + } + encoded, err := ver.MarshalMsg(nil) + if err != nil { + return err + } + // Add space at the end. + // Will have -1 modtime, so it will be inserted there. + x.versions = append(x.versions, xlMetaV2ShallowVersion{header: xlMetaV2VersionHeader{ModTime: -1}}) + + // Linear search, we likely have to insert at front. + for i, existing := range x.versions { + if existing.header.ModTime <= modTime { + // Insert at current idx. First move current back. + copy(x.versions[i+1:], x.versions[i:]) + x.versions[i] = xlMetaV2ShallowVersion{ + header: ver.header(), + meta: encoded, + } + return nil + } + } + return fmt.Errorf("addVersion: Internal error, unable to add version") +} + +// AppendTo will marshal the data in z and append it to the provided slice. +func (x *xlMetaV2) AppendTo(dst []byte) ([]byte, error) { + // Header... + sz := len(xlHeader) + len(xlVersionCurrent) + msgp.ArrayHeaderSize + len(dst) + 3*msgp.Uint32Size + // Existing + Inline data + sz += len(dst) + len(x.data) + // Versions... + for _, ver := range x.versions { + sz += 32 + len(ver.meta) + } + if cap(dst) < sz { + buf := make([]byte, len(dst), sz) + copy(buf, dst) + dst = buf + } + if err := x.data.validate(); err != nil { + return nil, err + } + + dst = append(dst, xlHeader[:]...) + dst = append(dst, xlVersionCurrent[:]...) + // Add "bin 32" type header to always have enough space. + // We will fill out the correct size when we know it. + dst = append(dst, 0xc6, 0, 0, 0, 0) + dataOffset := len(dst) + + dst = msgp.AppendUint(dst, xlHeaderVersion) + dst = msgp.AppendUint(dst, xlMetaVersion) + dst = msgp.AppendInt(dst, len(x.versions)) + + tmp := metaDataPoolGet() + defer metaDataPoolPut(tmp) + for _, ver := range x.versions { + var err error + + // Add header + tmp, err = ver.header.MarshalMsg(tmp[:0]) + if err != nil { + return nil, err + } + dst = msgp.AppendBytes(dst, tmp) + + // Add full meta + dst = msgp.AppendBytes(dst, ver.meta) + } + + // Update size... + binary.BigEndian.PutUint32(dst[dataOffset-4:dataOffset], uint32(len(dst)-dataOffset)) + + // Add CRC of metadata as fixed size (5 bytes) + // Prior to v1.3 this was variable sized. + tmp = tmp[:5] + tmp[0] = 0xce // muint32 + binary.BigEndian.PutUint32(tmp[1:], uint32(xxhash.Sum64(dst[dataOffset:]))) + dst = append(dst, tmp[:5]...) + return append(dst, x.data...), nil +} + +func (x *xlMetaV2) findVersion(key [16]byte) (idx int, ver *xlMetaV2Version, err error) { + for i, ver := range x.versions { + if key == ver.header.VersionID { + obj, err := x.getIdx(i) + return i, obj, err + } + } + return -1, nil, errFileVersionNotFound +} + +func (x *xlMetaV2) getIdx(idx int) (ver *xlMetaV2Version, err error) { + if idx < 0 || idx >= len(x.versions) { + return nil, errFileNotFound + } + var dst xlMetaV2Version + _, err = dst.unmarshalV(x.metaV, x.versions[idx].meta) + if false { + if err == nil && x.versions[idx].header.VersionID != dst.getVersionID() { + panic(fmt.Sprintf("header: %x != object id: %x", x.versions[idx].header.VersionID, dst.getVersionID())) + } + } + return &dst, err +} + +// setIdx will replace a version at a given index. +// Note that versions may become re-sorted if modtime changes. +func (x *xlMetaV2) setIdx(idx int, ver xlMetaV2Version) (err error) { + if idx < 0 || idx >= len(x.versions) { + return errFileNotFound + } + update := &x.versions[idx] + prevMod := update.header.ModTime + update.meta, err = ver.MarshalMsg(update.meta[:0:len(update.meta)]) + if err != nil { + update.meta = nil + return err + } + update.header = ver.header() + if prevMod != update.header.ModTime { + x.sortByModTime() + } + return nil +} + +// sortByModTime will sort versions by modtime in descending order, +// meaning index 0 will be latest version. +func (x *xlMetaV2) sortByModTime() { + // Quick check + if len(x.versions) <= 1 || sort.SliceIsSorted(x.versions, func(i, j int) bool { + return x.versions[i].header.ModTime > x.versions[j].header.ModTime + }) { + return + } + + // We should sort. + sort.Slice(x.versions, func(i, j int) bool { + return x.versions[i].header.ModTime > x.versions[j].header.ModTime + }) +} + +// DeleteVersion deletes the version specified by version id. +// returns to the caller which dataDir to delete, also +// indicates if this is the last version. +func (x *xlMetaV2) DeleteVersion(fi FileInfo) (string, bool, error) { + // This is a situation where versionId is explicitly + // specified as "null", as we do not save "null" + // string it is considered empty. But empty also + // means the version which matches will be purged. + if fi.VersionID == nullVersionID { + fi.VersionID = "" + } + + var uv uuid.UUID + var err error + if fi.VersionID != "" { + uv, err = uuid.Parse(fi.VersionID) + if err != nil { + return "", false, errFileVersionNotFound + } + } + + var ventry xlMetaV2Version + if fi.Deleted { + ventry = xlMetaV2Version{ + Type: DeleteType, + DeleteMarker: &xlMetaV2DeleteMarker{ + VersionID: uv, + ModTime: fi.ModTime.UnixNano(), + MetaSys: make(map[string][]byte), + }, + } + if !ventry.Valid() { + return "", false, errors.New("internal error: invalid version entry generated") + } + } + updateVersion := false + if fi.VersionPurgeStatus().Empty() && (fi.DeleteMarkerReplicationStatus() == "REPLICA" || fi.DeleteMarkerReplicationStatus().Empty()) { + updateVersion = fi.MarkDeleted + } else { + // for replication scenario + if fi.Deleted && fi.VersionPurgeStatus() != Complete { + if !fi.VersionPurgeStatus().Empty() || fi.DeleteMarkerReplicationStatus().Empty() { + updateVersion = true + } + } + // object or delete-marker versioned delete is not complete + if !fi.VersionPurgeStatus().Empty() && fi.VersionPurgeStatus() != Complete { + updateVersion = true + } + } + + if fi.Deleted { + if !fi.DeleteMarkerReplicationStatus().Empty() { + switch fi.DeleteMarkerReplicationStatus() { + case replication.Replica: + ventry.DeleteMarker.MetaSys[ReservedMetadataPrefixLower+ReplicaStatus] = []byte(string(fi.ReplicationState.ReplicaStatus)) + ventry.DeleteMarker.MetaSys[ReservedMetadataPrefixLower+ReplicaTimestamp] = []byte(fi.ReplicationState.ReplicaTimeStamp.Format(http.TimeFormat)) + default: + ventry.DeleteMarker.MetaSys[ReservedMetadataPrefixLower+ReplicationStatus] = []byte(fi.ReplicationState.ReplicationStatusInternal) + ventry.DeleteMarker.MetaSys[ReservedMetadataPrefixLower+ReplicationTimestamp] = []byte(fi.ReplicationState.ReplicationTimeStamp.Format(http.TimeFormat)) + } + } + if !fi.VersionPurgeStatus().Empty() { + ventry.DeleteMarker.MetaSys[VersionPurgeStatusKey] = []byte(fi.ReplicationState.VersionPurgeStatusInternal) + } + for k, v := range fi.ReplicationState.ResetStatusesMap { + ventry.DeleteMarker.MetaSys[k] = []byte(v) + } + } + + for i, ver := range x.versions { + if ver.header.VersionID != uv { + continue + } + switch ver.header.Type { + case LegacyType: + ver, err := x.getIdx(i) + if err != nil { + return "", false, err + } + x.versions = append(x.versions[:i], x.versions[i+1:]...) + if fi.Deleted { + err = x.addVersion(ventry) + } + return ver.ObjectV1.DataDir, len(x.versions) == 0, err + case DeleteType: + if updateVersion { + ver, err := x.getIdx(i) + if err != nil { + return "", false, err + } + if len(ver.DeleteMarker.MetaSys) == 0 { + ver.DeleteMarker.MetaSys = make(map[string][]byte) + } + if !fi.DeleteMarkerReplicationStatus().Empty() { + switch fi.DeleteMarkerReplicationStatus() { + case replication.Replica: + ver.DeleteMarker.MetaSys[ReservedMetadataPrefixLower+ReplicaStatus] = []byte(string(fi.ReplicationState.ReplicaStatus)) + ver.DeleteMarker.MetaSys[ReservedMetadataPrefixLower+ReplicaTimestamp] = []byte(fi.ReplicationState.ReplicaTimeStamp.Format(http.TimeFormat)) + default: + ver.DeleteMarker.MetaSys[ReservedMetadataPrefixLower+ReplicationStatus] = []byte(fi.ReplicationState.ReplicationStatusInternal) + ver.DeleteMarker.MetaSys[ReservedMetadataPrefixLower+ReplicationTimestamp] = []byte(fi.ReplicationState.ReplicationTimeStamp.Format(http.TimeFormat)) + } + } + if !fi.VersionPurgeStatus().Empty() { + ver.DeleteMarker.MetaSys[VersionPurgeStatusKey] = []byte(fi.ReplicationState.VersionPurgeStatusInternal) + } + for k, v := range fi.ReplicationState.ResetStatusesMap { + ver.DeleteMarker.MetaSys[k] = []byte(v) + } + err = x.setIdx(i, *ver) + return "", len(x.versions) == 0, err + } + var err error + x.versions = append(x.versions[:i], x.versions[i+1:]...) + if fi.MarkDeleted && (fi.VersionPurgeStatus().Empty() || (fi.VersionPurgeStatus() != Complete)) { + err = x.addVersion(ventry) + } + return "", len(x.versions) == 0, err + case ObjectType: + if updateVersion { + ver, err := x.getIdx(i) + if err != nil { + return "", false, err + } + ver.ObjectV2.MetaSys[VersionPurgeStatusKey] = []byte(fi.ReplicationState.VersionPurgeStatusInternal) + for k, v := range fi.ReplicationState.ResetStatusesMap { + ver.ObjectV2.MetaSys[k] = []byte(v) + } + err = x.setIdx(i, *ver) + return "", len(x.versions) == 0, err + } + } + } + + for i, version := range x.versions { + if version.header.Type != ObjectType || version.header.VersionID != uv { + continue + } + ver, err := x.getIdx(i) + if err != nil { + return "", false, err + } + switch { + case fi.ExpireRestored: + ver.ObjectV2.RemoveRestoreHdrs() + err = x.setIdx(i, *ver) + case fi.TransitionStatus == lifecycle.TransitionComplete: + ver.ObjectV2.SetTransition(fi) + err = x.setIdx(i, *ver) + default: + x.versions = append(x.versions[:i], x.versions[i+1:]...) + // if uv has tiered content we add a + // free-version to track it for + // asynchronous deletion via scanner. + if freeVersion, toFree := ver.ObjectV2.InitFreeVersion(fi); toFree { + err = x.addVersion(freeVersion) + } + } + logger.LogIf(context.Background(), err) + + if fi.Deleted { + err = x.addVersion(ventry) + } + if x.SharedDataDirCount(ver.ObjectV2.VersionID, ver.ObjectV2.DataDir) > 0 { + // Found that another version references the same dataDir + // we shouldn't remove it, and only remove the version instead + return "", len(x.versions) == 0, nil + } + return uuid.UUID(ver.ObjectV2.DataDir).String(), len(x.versions) == 0, err + } + + if fi.Deleted { + err = x.addVersion(ventry) + return "", false, err + } + return "", false, errFileVersionNotFound +} + +// xlMetaDataDirDecoder is a shallow decoder for decoding object datadir only. +type xlMetaDataDirDecoder struct { + ObjectV2 *struct { + DataDir [16]byte `msg:"DDir"` // Data dir ID + } `msg:"V2Obj,omitempty"` +} + +// UpdateObjectVersion updates metadata and modTime for a given +// versionID, NOTE: versionID must be valid and should exist - +// and must not be a DeleteMarker or legacy object, if no +// versionID is specified 'null' versionID is updated instead. +// +// It is callers responsibility to set correct versionID, this +// function shouldn't be further extended to update immutable +// values such as ErasureInfo, ChecksumInfo. +// +// Metadata is only updated to new values, existing values +// stay as is, if you wish to update all values you should +// update all metadata freshly before calling this function +// in-case you wish to clear existing metadata. +func (x *xlMetaV2) UpdateObjectVersion(fi FileInfo) error { + if fi.VersionID == "" { + // this means versioning is not yet + // enabled or suspend i.e all versions + // are basically default value i.e "null" + fi.VersionID = nullVersionID + } + + var uv uuid.UUID + var err error + if fi.VersionID != "" && fi.VersionID != nullVersionID { + uv, err = uuid.Parse(fi.VersionID) + if err != nil { + return err + } + } + + for i, version := range x.versions { + switch version.header.Type { + case LegacyType, DeleteType: + if version.header.VersionID == uv { + return errMethodNotAllowed + } + case ObjectType: + if version.header.VersionID == uv { + ver, err := x.getIdx(i) + if err != nil { + return err + } + for k, v := range fi.Metadata { + if len(k) > len(ReservedMetadataPrefixLower) && strings.EqualFold(k[:len(ReservedMetadataPrefixLower)], ReservedMetadataPrefixLower) { + ver.ObjectV2.MetaSys[k] = []byte(v) + } else { + ver.ObjectV2.MetaUser[k] = v + } + } + if !fi.ModTime.IsZero() { + ver.ObjectV2.ModTime = fi.ModTime.UnixNano() + } + return x.setIdx(i, *ver) + } + } + } + + return errFileVersionNotFound +} + +// AddVersion adds a new version +func (x *xlMetaV2) AddVersion(fi FileInfo) error { + if fi.VersionID == "" { + // this means versioning is not yet + // enabled or suspend i.e all versions + // are basically default value i.e "null" + fi.VersionID = nullVersionID + } + + var uv uuid.UUID + var err error + if fi.VersionID != "" && fi.VersionID != nullVersionID { + uv, err = uuid.Parse(fi.VersionID) + if err != nil { + return err + } + } + + var dd uuid.UUID + if fi.DataDir != "" { + dd, err = uuid.Parse(fi.DataDir) + if err != nil { + return err + } + } + + ventry := xlMetaV2Version{} + + if fi.Deleted { + ventry.Type = DeleteType + ventry.DeleteMarker = &xlMetaV2DeleteMarker{ + VersionID: uv, + ModTime: fi.ModTime.UnixNano(), + MetaSys: make(map[string][]byte), + } + } else { + ventry.Type = ObjectType + ventry.ObjectV2 = &xlMetaV2Object{ + VersionID: uv, + DataDir: dd, + Size: fi.Size, + ModTime: fi.ModTime.UnixNano(), + ErasureAlgorithm: ReedSolomon, + ErasureM: fi.Erasure.DataBlocks, + ErasureN: fi.Erasure.ParityBlocks, + ErasureBlockSize: fi.Erasure.BlockSize, + ErasureIndex: fi.Erasure.Index, + BitrotChecksumAlgo: HighwayHash, + ErasureDist: make([]uint8, len(fi.Erasure.Distribution)), + PartNumbers: make([]int, len(fi.Parts)), + PartETags: nil, + PartSizes: make([]int64, len(fi.Parts)), + PartActualSizes: make([]int64, len(fi.Parts)), + MetaSys: make(map[string][]byte), + MetaUser: make(map[string]string, len(fi.Metadata)), + } + for i := range fi.Parts { + // Only add etags if any. + if fi.Parts[i].ETag != "" { + ventry.ObjectV2.PartETags = make([]string, len(fi.Parts)) + break + } + } + for i := range fi.Erasure.Distribution { + ventry.ObjectV2.ErasureDist[i] = uint8(fi.Erasure.Distribution[i]) + } + + for i := range fi.Parts { + ventry.ObjectV2.PartSizes[i] = fi.Parts[i].Size + if len(ventry.ObjectV2.PartETags) > 0 && fi.Parts[i].ETag != "" { + ventry.ObjectV2.PartETags[i] = fi.Parts[i].ETag + } + ventry.ObjectV2.PartNumbers[i] = fi.Parts[i].Number + ventry.ObjectV2.PartActualSizes[i] = fi.Parts[i].ActualSize + } + + tierFVIDKey := ReservedMetadataPrefixLower + tierFVID + tierFVMarkerKey := ReservedMetadataPrefixLower + tierFVMarker + for k, v := range fi.Metadata { + if len(k) > len(ReservedMetadataPrefixLower) && strings.EqualFold(k[:len(ReservedMetadataPrefixLower)], ReservedMetadataPrefixLower) { + // Skip tierFVID, tierFVMarker keys; it's used + // only for creating free-version. + switch k { + case tierFVIDKey, tierFVMarkerKey: + continue + } + + ventry.ObjectV2.MetaSys[k] = []byte(v) + } else { + ventry.ObjectV2.MetaUser[k] = v + } + } + + // If asked to save data. + if len(fi.Data) > 0 || fi.Size == 0 { + x.data.replace(fi.VersionID, fi.Data) + } + + if fi.TransitionStatus != "" { + ventry.ObjectV2.MetaSys[ReservedMetadataPrefixLower+TransitionStatus] = []byte(fi.TransitionStatus) + } + if fi.TransitionedObjName != "" { + ventry.ObjectV2.MetaSys[ReservedMetadataPrefixLower+TransitionedObjectName] = []byte(fi.TransitionedObjName) + } + if fi.TransitionVersionID != "" { + ventry.ObjectV2.MetaSys[ReservedMetadataPrefixLower+TransitionedVersionID] = []byte(fi.TransitionVersionID) + } + if fi.TransitionTier != "" { + ventry.ObjectV2.MetaSys[ReservedMetadataPrefixLower+TransitionTier] = []byte(fi.TransitionTier) + } + } + + if !ventry.Valid() { + return errors.New("internal error: invalid version entry generated") + } + + // Check if we should replace first. + for i := range x.versions { + if x.versions[i].header.VersionID != uv { + continue + } + switch x.versions[i].header.Type { + case LegacyType: + // This would convert legacy type into new ObjectType + // this means that we are basically purging the `null` + // version of the object. + return x.setIdx(i, ventry) + case ObjectType: + return x.setIdx(i, ventry) + case DeleteType: + // Allowing delete marker to replaced with proper + // object data type as well, this is not S3 complaint + // behavior but kept here for future flexibility. + return x.setIdx(i, ventry) + } + } + + // We did not find it, add it. + return x.addVersion(ventry) +} + +func (x *xlMetaV2) SharedDataDirCount(versionID [16]byte, dataDir [16]byte) int { + // v2 object is inlined, if it is skip dataDir share check. + if x.data.entries() > 0 && x.data.find(uuid.UUID(versionID).String()) != nil { + return 0 + } + var sameDataDirCount int + var decoded xlMetaDataDirDecoder + for _, version := range x.versions { + if version.header.Type != ObjectType || version.header.VersionID == versionID || !version.header.UsesDataDir() { + continue + } + _, err := decoded.UnmarshalMsg(version.meta) + if err != nil || decoded.ObjectV2 == nil || decoded.ObjectV2.DataDir != dataDir { + continue + } + sameDataDirCount++ + } + return sameDataDirCount +} + +func (x *xlMetaV2) SharedDataDirCountStr(versionID, dataDir string) int { + var ( + uv uuid.UUID + ddir uuid.UUID + err error + ) + if versionID == nullVersionID { + versionID = "" + } + if versionID != "" { + uv, err = uuid.Parse(versionID) + if err != nil { + return 0 + } + } + ddir, err = uuid.Parse(dataDir) + if err != nil { + return 0 + } + return x.SharedDataDirCount(uv, ddir) +} + +// AddLegacy adds a legacy version, is only called when no prior +// versions exist, safe to use it by only one function in xl-storage(RenameData) +func (x *xlMetaV2) AddLegacy(m *xlMetaV1Object) error { + if !m.valid() { + return errFileCorrupt + } + m.VersionID = nullVersionID + m.DataDir = legacyDataDir + + return x.addVersion(xlMetaV2Version{ObjectV1: m, Type: LegacyType}) +} + +// ToFileInfo converts xlMetaV2 into a common FileInfo datastructure +// for consumption across callers. +func (x xlMetaV2) ToFileInfo(volume, path, versionID string) (fi FileInfo, err error) { + var uv uuid.UUID + if versionID != "" && versionID != nullVersionID { + uv, err = uuid.Parse(versionID) + if err != nil { + logger.LogIf(GlobalContext, fmt.Errorf("invalid versionID specified %s", versionID)) + return fi, errFileVersionNotFound + } + } + var succModTime int64 + isLatest := true + nonFreeVersions := len(x.versions) + found := false + for _, ver := range x.versions { + header := &ver.header + // skip listing free-version unless explicitly requested via versionID + if header.FreeVersion() { + nonFreeVersions-- + if header.VersionID != uv { + continue + } + } + if found { + continue + } + + // We need a specific version, skip... + if versionID != "" && uv != header.VersionID { + isLatest = false + succModTime = header.ModTime + continue + } + + // We found what we need. + found = true + var version xlMetaV2Version + if _, err := version.unmarshalV(x.metaV, ver.meta); err != nil { + return fi, err + } + if fi, err = version.ToFileInfo(volume, path); err != nil { + return fi, err + } + fi.IsLatest = isLatest + if succModTime != 0 { + fi.SuccessorModTime = time.Unix(0, succModTime) + } + } + if !found { + if versionID == "" { + return FileInfo{}, errFileNotFound + } + + return FileInfo{}, errFileVersionNotFound + } + fi.NumVersions = nonFreeVersions + return fi, err +} + +// ListVersions lists current versions, and current deleted +// versions returns error for unexpected entries. +// showPendingDeletes is set to true if ListVersions needs to list objects marked deleted +// but waiting to be replicated +func (x xlMetaV2) ListVersions(volume, path string) ([]FileInfo, error) { + versions := make([]FileInfo, 0, len(x.versions)) + var err error + + var dst xlMetaV2Version + for _, version := range x.versions { + _, err = dst.unmarshalV(x.metaV, version.meta) + if err != nil { + return versions, err + } + fi, err := dst.ToFileInfo(volume, path) + if err != nil { + return versions, err + } + fi.NumVersions = len(x.versions) + versions = append(versions, fi) + } + + for i := range versions { + versions[i].NumVersions = len(versions) + if i > 0 { + versions[i].SuccessorModTime = versions[i-1].ModTime + } + } + if len(versions) > 0 { + versions[0].IsLatest = true + } + return versions, nil +} + +type xlMetaBuf []byte + +// ToFileInfo converts xlMetaV2 into a common FileInfo datastructure +// for consumption across callers. +func (x xlMetaBuf) ToFileInfo(volume, path, versionID string) (fi FileInfo, err error) { + var uv uuid.UUID + if versionID != "" && versionID != nullVersionID { + uv, err = uuid.Parse(versionID) + if err != nil { + logger.LogIf(GlobalContext, fmt.Errorf("invalid versionID specified %s", versionID)) + return fi, errFileVersionNotFound + } + } + versions, headerV, metaV, buf, err := decodeXLHeaders(x) + if err != nil { + return fi, err + } + var header xlMetaV2VersionHeader + var succModTime int64 + isLatest := true + nonFreeVersions := versions + found := false + err = decodeVersions(buf, versions, func(idx int, hdr, meta []byte) error { + if _, err := header.unmarshalV(headerV, hdr); err != nil { + return err + } + + // skip listing free-version unless explicitly requested via versionID + if header.FreeVersion() { + nonFreeVersions-- + if header.VersionID != uv { + return nil + } + } + if found { + return nil + } + + // We need a specific version, skip... + if versionID != "" && uv != header.VersionID { + isLatest = false + succModTime = header.ModTime + return nil + } + + // We found what we need. + found = true + var version xlMetaV2Version + if _, err := version.unmarshalV(metaV, meta); err != nil { + return err + } + if fi, err = version.ToFileInfo(volume, path); err != nil { + return err + } + fi.IsLatest = isLatest + if succModTime != 0 { + fi.SuccessorModTime = time.Unix(0, succModTime) + } + return nil + }) + if !found { + if versionID == "" { + return FileInfo{}, errFileNotFound + } + + return FileInfo{}, errFileVersionNotFound + } + fi.NumVersions = nonFreeVersions + return fi, err +} + +// ListVersions lists current versions, and current deleted +// versions returns error for unexpected entries. +// showPendingDeletes is set to true if ListVersions needs to list objects marked deleted +// but waiting to be replicated +func (x xlMetaBuf) ListVersions(volume, path string) ([]FileInfo, error) { + vers, _, metaV, buf, err := decodeXLHeaders(x) + if err != nil { + return nil, err + } + var succModTime time.Time + isLatest := true + dst := make([]FileInfo, 0, vers) + var xl xlMetaV2Version + err = decodeVersions(buf, vers, func(idx int, hdr, meta []byte) error { + if _, err := xl.unmarshalV(metaV, meta); err != nil { + return err + } + if !xl.Valid() { + return errFileCorrupt + } + fi, err := xl.ToFileInfo(volume, path) + if err != nil { + return err + } + fi.IsLatest = isLatest + fi.SuccessorModTime = succModTime + fi.NumVersions = vers + isLatest = false + succModTime = xl.getModTime() + + dst = append(dst, fi) + return nil + }) + return dst, err +} + +// IsLatestDeleteMarker returns true if latest version is a deletemarker or there are no versions. +// If any error occurs false is returned. +func (x xlMetaBuf) IsLatestDeleteMarker() bool { + vers, headerV, _, buf, err := decodeXLHeaders(x) + if err != nil { + return false + } + if vers == 0 { + return true + } + isDeleteMarker := false + + _ = decodeVersions(buf, vers, func(idx int, hdr, _ []byte) error { + var xl xlMetaV2VersionHeader + if _, err := xl.unmarshalV(headerV, hdr); err != nil { + return errDoneForNow + } + isDeleteMarker = xl.Type == DeleteType + return errDoneForNow + + }) + return isDeleteMarker +} diff --git a/cmd/xl-storage-format-v2_gen.go b/cmd/xl-storage-format-v2_gen.go index 307147ca1..0ee5d38df 100644 --- a/cmd/xl-storage-format-v2_gen.go +++ b/cmd/xl-storage-format-v2_gen.go @@ -163,7 +163,111 @@ func (z VersionType) Msgsize() (s int) { } // DecodeMsg implements msgp.Decodable -func (z *xlMetaV2) DecodeMsg(dc *msgp.Reader) (err error) { +func (z *xlFlags) DecodeMsg(dc *msgp.Reader) (err error) { + { + var zb0001 uint8 + zb0001, err = dc.ReadUint8() + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = xlFlags(zb0001) + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z xlFlags) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteUint8(uint8(z)) + if err != nil { + err = msgp.WrapError(err) + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z xlFlags) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendUint8(o, uint8(z)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *xlFlags) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 uint8 + zb0001, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = xlFlags(zb0001) + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z xlFlags) Msgsize() (s int) { + s = msgp.Uint8Size + return +} + +// DecodeMsg implements msgp.Decodable +func (z *xlMetaBuf) DecodeMsg(dc *msgp.Reader) (err error) { + { + var zb0001 []byte + zb0001, err = dc.ReadBytes([]byte((*z))) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = xlMetaBuf(zb0001) + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z xlMetaBuf) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteBytes([]byte(z)) + if err != nil { + err = msgp.WrapError(err) + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z xlMetaBuf) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendBytes(o, []byte(z)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *xlMetaBuf) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 []byte + zb0001, bts, err = msgp.ReadBytesBytes(bts, []byte((*z))) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = xlMetaBuf(zb0001) + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z xlMetaBuf) Msgsize() (s int) { + s = msgp.BytesPrefixSize + len([]byte(z)) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *xlMetaDataDirDecoder) DecodeMsg(dc *msgp.Reader) (err error) { var field []byte _ = field var zb0001 uint32 @@ -180,24 +284,48 @@ func (z *xlMetaV2) DecodeMsg(dc *msgp.Reader) (err error) { return } switch msgp.UnsafeString(field) { - case "Versions": - var zb0002 uint32 - zb0002, err = dc.ReadArrayHeader() - if err != nil { - err = msgp.WrapError(err, "Versions") - return - } - if cap(z.Versions) >= int(zb0002) { - z.Versions = (z.Versions)[:zb0002] - } else { - z.Versions = make([]xlMetaV2Version, zb0002) - } - for za0001 := range z.Versions { - err = z.Versions[za0001].DecodeMsg(dc) + case "V2Obj": + if dc.IsNil() { + err = dc.ReadNil() if err != nil { - err = msgp.WrapError(err, "Versions", za0001) + err = msgp.WrapError(err, "ObjectV2") return } + z.ObjectV2 = nil + } else { + if z.ObjectV2 == nil { + z.ObjectV2 = new(struct { + DataDir [16]byte `msg:"DDir"` + }) + } + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "ObjectV2") + return + } + for zb0002 > 0 { + zb0002-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err, "ObjectV2") + return + } + switch msgp.UnsafeString(field) { + case "DDir": + err = dc.ReadExactBytes((z.ObjectV2.DataDir)[:]) + if err != nil { + err = msgp.WrapError(err, "ObjectV2", "DataDir") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err, "ObjectV2") + return + } + } + } } default: err = dc.Skip() @@ -211,47 +339,82 @@ func (z *xlMetaV2) DecodeMsg(dc *msgp.Reader) (err error) { } // EncodeMsg implements msgp.Encodable -func (z *xlMetaV2) EncodeMsg(en *msgp.Writer) (err error) { - // map header, size 1 - // write "Versions" - err = en.Append(0x81, 0xa8, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73) +func (z *xlMetaDataDirDecoder) EncodeMsg(en *msgp.Writer) (err error) { + // omitempty: check for empty values + zb0001Len := uint32(1) + var zb0001Mask uint8 /* 1 bits */ + if z.ObjectV2 == nil { + zb0001Len-- + zb0001Mask |= 0x1 + } + // variable map header, size zb0001Len + err = en.Append(0x80 | uint8(zb0001Len)) if err != nil { return } - err = en.WriteArrayHeader(uint32(len(z.Versions))) - if err != nil { - err = msgp.WrapError(err, "Versions") + if zb0001Len == 0 { return } - for za0001 := range z.Versions { - err = z.Versions[za0001].EncodeMsg(en) + if (zb0001Mask & 0x1) == 0 { // if not empty + // write "V2Obj" + err = en.Append(0xa5, 0x56, 0x32, 0x4f, 0x62, 0x6a) if err != nil { - err = msgp.WrapError(err, "Versions", za0001) return } + if z.ObjectV2 == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + // map header, size 1 + // write "DDir" + err = en.Append(0x81, 0xa4, 0x44, 0x44, 0x69, 0x72) + if err != nil { + return + } + err = en.WriteBytes((z.ObjectV2.DataDir)[:]) + if err != nil { + err = msgp.WrapError(err, "ObjectV2", "DataDir") + return + } + } } return } // MarshalMsg implements msgp.Marshaler -func (z *xlMetaV2) MarshalMsg(b []byte) (o []byte, err error) { +func (z *xlMetaDataDirDecoder) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) - // map header, size 1 - // string "Versions" - o = append(o, 0x81, 0xa8, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73) - o = msgp.AppendArrayHeader(o, uint32(len(z.Versions))) - for za0001 := range z.Versions { - o, err = z.Versions[za0001].MarshalMsg(o) - if err != nil { - err = msgp.WrapError(err, "Versions", za0001) - return + // omitempty: check for empty values + zb0001Len := uint32(1) + var zb0001Mask uint8 /* 1 bits */ + if z.ObjectV2 == nil { + zb0001Len-- + zb0001Mask |= 0x1 + } + // variable map header, size zb0001Len + o = append(o, 0x80|uint8(zb0001Len)) + if zb0001Len == 0 { + return + } + if (zb0001Mask & 0x1) == 0 { // if not empty + // string "V2Obj" + o = append(o, 0xa5, 0x56, 0x32, 0x4f, 0x62, 0x6a) + if z.ObjectV2 == nil { + o = msgp.AppendNil(o) + } else { + // map header, size 1 + // string "DDir" + o = append(o, 0x81, 0xa4, 0x44, 0x44, 0x69, 0x72) + o = msgp.AppendBytes(o, (z.ObjectV2.DataDir)[:]) } } return } // UnmarshalMsg implements msgp.Unmarshaler -func (z *xlMetaV2) UnmarshalMsg(bts []byte) (o []byte, err error) { +func (z *xlMetaDataDirDecoder) UnmarshalMsg(bts []byte) (o []byte, err error) { var field []byte _ = field var zb0001 uint32 @@ -268,24 +431,47 @@ func (z *xlMetaV2) UnmarshalMsg(bts []byte) (o []byte, err error) { return } switch msgp.UnsafeString(field) { - case "Versions": - var zb0002 uint32 - zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) - if err != nil { - err = msgp.WrapError(err, "Versions") - return - } - if cap(z.Versions) >= int(zb0002) { - z.Versions = (z.Versions)[:zb0002] - } else { - z.Versions = make([]xlMetaV2Version, zb0002) - } - for za0001 := range z.Versions { - bts, err = z.Versions[za0001].UnmarshalMsg(bts) + case "V2Obj": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) if err != nil { - err = msgp.WrapError(err, "Versions", za0001) return } + z.ObjectV2 = nil + } else { + if z.ObjectV2 == nil { + z.ObjectV2 = new(struct { + DataDir [16]byte `msg:"DDir"` + }) + } + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ObjectV2") + return + } + for zb0002 > 0 { + zb0002-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "ObjectV2") + return + } + switch msgp.UnsafeString(field) { + case "DDir": + bts, err = msgp.ReadExactBytes(bts, (z.ObjectV2.DataDir)[:]) + if err != nil { + err = msgp.WrapError(err, "ObjectV2", "DataDir") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err, "ObjectV2") + return + } + } + } } default: bts, err = msgp.Skip(bts) @@ -300,10 +486,12 @@ func (z *xlMetaV2) UnmarshalMsg(bts []byte) (o []byte, err error) { } // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z *xlMetaV2) Msgsize() (s int) { - s = 1 + 9 + msgp.ArrayHeaderSize - for za0001 := range z.Versions { - s += z.Versions[za0001].Msgsize() +func (z *xlMetaDataDirDecoder) Msgsize() (s int) { + s = 1 + 6 + if z.ObjectV2 == nil { + s += msgp.NilSize + } else { + s += 1 + 5 + msgp.ArrayHeaderSize + (16 * (msgp.ByteSize)) } return } @@ -673,23 +861,32 @@ func (z *xlMetaV2Object) DecodeMsg(dc *msgp.Reader) (err error) { } } case "PartETags": - var zb0006 uint32 - zb0006, err = dc.ReadArrayHeader() - if err != nil { - err = msgp.WrapError(err, "PartETags") - return - } - if cap(z.PartETags) >= int(zb0006) { - z.PartETags = (z.PartETags)[:zb0006] - } else { - z.PartETags = make([]string, zb0006) - } - for za0005 := range z.PartETags { - z.PartETags[za0005], err = dc.ReadString() + if dc.IsNil() { + err = dc.ReadNil() if err != nil { - err = msgp.WrapError(err, "PartETags", za0005) + err = msgp.WrapError(err, "PartETags") return } + z.PartETags = nil + } else { + var zb0006 uint32 + zb0006, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "PartETags") + return + } + if cap(z.PartETags) >= int(zb0006) { + z.PartETags = (z.PartETags)[:zb0006] + } else { + z.PartETags = make([]string, zb0006) + } + for za0005 := range z.PartETags { + z.PartETags[za0005], err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "PartETags", za0005) + return + } + } } case "PartSizes": var zb0007 uint32 @@ -711,23 +908,32 @@ func (z *xlMetaV2Object) DecodeMsg(dc *msgp.Reader) (err error) { } } case "PartASizes": - var zb0008 uint32 - zb0008, err = dc.ReadArrayHeader() - if err != nil { - err = msgp.WrapError(err, "PartActualSizes") - return - } - if cap(z.PartActualSizes) >= int(zb0008) { - z.PartActualSizes = (z.PartActualSizes)[:zb0008] - } else { - z.PartActualSizes = make([]int64, zb0008) - } - for za0007 := range z.PartActualSizes { - z.PartActualSizes[za0007], err = dc.ReadInt64() + if dc.IsNil() { + err = dc.ReadNil() if err != nil { - err = msgp.WrapError(err, "PartActualSizes", za0007) + err = msgp.WrapError(err, "PartActualSizes") return } + z.PartActualSizes = nil + } else { + var zb0008 uint32 + zb0008, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "PartActualSizes") + return + } + if cap(z.PartActualSizes) >= int(zb0008) { + z.PartActualSizes = (z.PartActualSizes)[:zb0008] + } else { + z.PartActualSizes = make([]int64, zb0008) + } + for za0007 := range z.PartActualSizes { + z.PartActualSizes[za0007], err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "PartActualSizes", za0007) + return + } + } } case "Size": z.Size, err = dc.ReadInt64() @@ -742,64 +948,82 @@ func (z *xlMetaV2Object) DecodeMsg(dc *msgp.Reader) (err error) { return } case "MetaSys": - var zb0009 uint32 - zb0009, err = dc.ReadMapHeader() - if err != nil { - err = msgp.WrapError(err, "MetaSys") - return - } - if z.MetaSys == nil { - z.MetaSys = make(map[string][]byte, zb0009) - } else if len(z.MetaSys) > 0 { - for key := range z.MetaSys { - delete(z.MetaSys, key) - } - } - for zb0009 > 0 { - zb0009-- - var za0008 string - var za0009 []byte - za0008, err = dc.ReadString() + if dc.IsNil() { + err = dc.ReadNil() if err != nil { err = msgp.WrapError(err, "MetaSys") return } - za0009, err = dc.ReadBytes(za0009) + z.MetaSys = nil + } else { + var zb0009 uint32 + zb0009, err = dc.ReadMapHeader() if err != nil { - err = msgp.WrapError(err, "MetaSys", za0008) + err = msgp.WrapError(err, "MetaSys") return } - z.MetaSys[za0008] = za0009 - } - case "MetaUsr": - var zb0010 uint32 - zb0010, err = dc.ReadMapHeader() - if err != nil { - err = msgp.WrapError(err, "MetaUser") - return - } - if z.MetaUser == nil { - z.MetaUser = make(map[string]string, zb0010) - } else if len(z.MetaUser) > 0 { - for key := range z.MetaUser { - delete(z.MetaUser, key) + if z.MetaSys == nil { + z.MetaSys = make(map[string][]byte, zb0009) + } else if len(z.MetaSys) > 0 { + for key := range z.MetaSys { + delete(z.MetaSys, key) + } + } + for zb0009 > 0 { + zb0009-- + var za0008 string + var za0009 []byte + za0008, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "MetaSys") + return + } + za0009, err = dc.ReadBytes(za0009) + if err != nil { + err = msgp.WrapError(err, "MetaSys", za0008) + return + } + z.MetaSys[za0008] = za0009 } } - for zb0010 > 0 { - zb0010-- - var za0010 string - var za0011 string - za0010, err = dc.ReadString() + case "MetaUsr": + if dc.IsNil() { + err = dc.ReadNil() if err != nil { err = msgp.WrapError(err, "MetaUser") return } - za0011, err = dc.ReadString() + z.MetaUser = nil + } else { + var zb0010 uint32 + zb0010, err = dc.ReadMapHeader() if err != nil { - err = msgp.WrapError(err, "MetaUser", za0010) + err = msgp.WrapError(err, "MetaUser") return } - z.MetaUser[za0010] = za0011 + if z.MetaUser == nil { + z.MetaUser = make(map[string]string, zb0010) + } else if len(z.MetaUser) > 0 { + for key := range z.MetaUser { + delete(z.MetaUser, key) + } + } + for zb0010 > 0 { + zb0010-- + var za0010 string + var za0011 string + za0010, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "MetaUser") + return + } + za0011, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "MetaUser", za0010) + return + } + z.MetaUser[za0010] = za0011 + } } default: err = dc.Skip() @@ -814,31 +1038,9 @@ func (z *xlMetaV2Object) DecodeMsg(dc *msgp.Reader) (err error) { // EncodeMsg implements msgp.Encodable func (z *xlMetaV2Object) EncodeMsg(en *msgp.Writer) (err error) { - // omitempty: check for empty values - zb0001Len := uint32(17) - var zb0001Mask uint32 /* 17 bits */ - if z.PartActualSizes == nil { - zb0001Len-- - zb0001Mask |= 0x1000 - } - if z.MetaSys == nil { - zb0001Len-- - zb0001Mask |= 0x8000 - } - if z.MetaUser == nil { - zb0001Len-- - zb0001Mask |= 0x10000 - } - // variable map header, size zb0001Len - err = en.WriteMapHeader(zb0001Len) - if err != nil { - return - } - if zb0001Len == 0 { - return - } + // map header, size 17 // write "ID" - err = en.Append(0xa2, 0x49, 0x44) + err = en.Append(0xde, 0x0, 0x11, 0xa2, 0x49, 0x44) if err != nil { return } @@ -956,17 +1158,24 @@ func (z *xlMetaV2Object) EncodeMsg(en *msgp.Writer) (err error) { if err != nil { return } - err = en.WriteArrayHeader(uint32(len(z.PartETags))) - if err != nil { - err = msgp.WrapError(err, "PartETags") - return - } - for za0005 := range z.PartETags { - err = en.WriteString(z.PartETags[za0005]) + if z.PartETags == nil { // allownil: if nil + err = en.WriteNil() if err != nil { - err = msgp.WrapError(err, "PartETags", za0005) return } + } else { + err = en.WriteArrayHeader(uint32(len(z.PartETags))) + if err != nil { + err = msgp.WrapError(err, "PartETags") + return + } + for za0005 := range z.PartETags { + err = en.WriteString(z.PartETags[za0005]) + if err != nil { + err = msgp.WrapError(err, "PartETags", za0005) + return + } + } } // write "PartSizes" err = en.Append(0xa9, 0x50, 0x61, 0x72, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x73) @@ -985,12 +1194,17 @@ func (z *xlMetaV2Object) EncodeMsg(en *msgp.Writer) (err error) { return } } - if (zb0001Mask & 0x1000) == 0 { // if not empty - // write "PartASizes" - err = en.Append(0xaa, 0x50, 0x61, 0x72, 0x74, 0x41, 0x53, 0x69, 0x7a, 0x65, 0x73) + // write "PartASizes" + err = en.Append(0xaa, 0x50, 0x61, 0x72, 0x74, 0x41, 0x53, 0x69, 0x7a, 0x65, 0x73) + if err != nil { + return + } + if z.PartActualSizes == nil { // allownil: if nil + err = en.WriteNil() if err != nil { return } + } else { err = en.WriteArrayHeader(uint32(len(z.PartActualSizes))) if err != nil { err = msgp.WrapError(err, "PartActualSizes") @@ -1024,12 +1238,17 @@ func (z *xlMetaV2Object) EncodeMsg(en *msgp.Writer) (err error) { err = msgp.WrapError(err, "ModTime") return } - if (zb0001Mask & 0x8000) == 0 { // if not empty - // write "MetaSys" - err = en.Append(0xa7, 0x4d, 0x65, 0x74, 0x61, 0x53, 0x79, 0x73) + // write "MetaSys" + err = en.Append(0xa7, 0x4d, 0x65, 0x74, 0x61, 0x53, 0x79, 0x73) + if err != nil { + return + } + if z.MetaSys == nil { // allownil: if nil + err = en.WriteNil() if err != nil { return } + } else { err = en.WriteMapHeader(uint32(len(z.MetaSys))) if err != nil { err = msgp.WrapError(err, "MetaSys") @@ -1048,12 +1267,17 @@ func (z *xlMetaV2Object) EncodeMsg(en *msgp.Writer) (err error) { } } } - if (zb0001Mask & 0x10000) == 0 { // if not empty - // write "MetaUsr" - err = en.Append(0xa7, 0x4d, 0x65, 0x74, 0x61, 0x55, 0x73, 0x72) + // write "MetaUsr" + err = en.Append(0xa7, 0x4d, 0x65, 0x74, 0x61, 0x55, 0x73, 0x72) + if err != nil { + return + } + if z.MetaUser == nil { // allownil: if nil + err = en.WriteNil() if err != nil { return } + } else { err = en.WriteMapHeader(uint32(len(z.MetaUser))) if err != nil { err = msgp.WrapError(err, "MetaUser") @@ -1078,28 +1302,9 @@ func (z *xlMetaV2Object) EncodeMsg(en *msgp.Writer) (err error) { // MarshalMsg implements msgp.Marshaler func (z *xlMetaV2Object) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) - // omitempty: check for empty values - zb0001Len := uint32(17) - var zb0001Mask uint32 /* 17 bits */ - if z.PartActualSizes == nil { - zb0001Len-- - zb0001Mask |= 0x1000 - } - if z.MetaSys == nil { - zb0001Len-- - zb0001Mask |= 0x8000 - } - if z.MetaUser == nil { - zb0001Len-- - zb0001Mask |= 0x10000 - } - // variable map header, size zb0001Len - o = msgp.AppendMapHeader(o, zb0001Len) - if zb0001Len == 0 { - return - } + // map header, size 17 // string "ID" - o = append(o, 0xa2, 0x49, 0x44) + o = append(o, 0xde, 0x0, 0x11, 0xa2, 0x49, 0x44) o = msgp.AppendBytes(o, (z.VersionID)[:]) // string "DDir" o = append(o, 0xa4, 0x44, 0x44, 0x69, 0x72) @@ -1136,9 +1341,13 @@ func (z *xlMetaV2Object) MarshalMsg(b []byte) (o []byte, err error) { } // string "PartETags" o = append(o, 0xa9, 0x50, 0x61, 0x72, 0x74, 0x45, 0x54, 0x61, 0x67, 0x73) - o = msgp.AppendArrayHeader(o, uint32(len(z.PartETags))) - for za0005 := range z.PartETags { - o = msgp.AppendString(o, z.PartETags[za0005]) + if z.PartETags == nil { // allownil: if nil + o = msgp.AppendNil(o) + } else { + o = msgp.AppendArrayHeader(o, uint32(len(z.PartETags))) + for za0005 := range z.PartETags { + o = msgp.AppendString(o, z.PartETags[za0005]) + } } // string "PartSizes" o = append(o, 0xa9, 0x50, 0x61, 0x72, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x73) @@ -1146,9 +1355,11 @@ func (z *xlMetaV2Object) MarshalMsg(b []byte) (o []byte, err error) { for za0006 := range z.PartSizes { o = msgp.AppendInt64(o, z.PartSizes[za0006]) } - if (zb0001Mask & 0x1000) == 0 { // if not empty - // string "PartASizes" - o = append(o, 0xaa, 0x50, 0x61, 0x72, 0x74, 0x41, 0x53, 0x69, 0x7a, 0x65, 0x73) + // string "PartASizes" + o = append(o, 0xaa, 0x50, 0x61, 0x72, 0x74, 0x41, 0x53, 0x69, 0x7a, 0x65, 0x73) + if z.PartActualSizes == nil { // allownil: if nil + o = msgp.AppendNil(o) + } else { o = msgp.AppendArrayHeader(o, uint32(len(z.PartActualSizes))) for za0007 := range z.PartActualSizes { o = msgp.AppendInt64(o, z.PartActualSizes[za0007]) @@ -1160,18 +1371,22 @@ func (z *xlMetaV2Object) MarshalMsg(b []byte) (o []byte, err error) { // string "MTime" o = append(o, 0xa5, 0x4d, 0x54, 0x69, 0x6d, 0x65) o = msgp.AppendInt64(o, z.ModTime) - if (zb0001Mask & 0x8000) == 0 { // if not empty - // string "MetaSys" - o = append(o, 0xa7, 0x4d, 0x65, 0x74, 0x61, 0x53, 0x79, 0x73) + // string "MetaSys" + o = append(o, 0xa7, 0x4d, 0x65, 0x74, 0x61, 0x53, 0x79, 0x73) + if z.MetaSys == nil { // allownil: if nil + o = msgp.AppendNil(o) + } else { o = msgp.AppendMapHeader(o, uint32(len(z.MetaSys))) for za0008, za0009 := range z.MetaSys { o = msgp.AppendString(o, za0008) o = msgp.AppendBytes(o, za0009) } } - if (zb0001Mask & 0x10000) == 0 { // if not empty - // string "MetaUsr" - o = append(o, 0xa7, 0x4d, 0x65, 0x74, 0x61, 0x55, 0x73, 0x72) + // string "MetaUsr" + o = append(o, 0xa7, 0x4d, 0x65, 0x74, 0x61, 0x55, 0x73, 0x72) + if z.MetaUser == nil { // allownil: if nil + o = msgp.AppendNil(o) + } else { o = msgp.AppendMapHeader(o, uint32(len(z.MetaUser))) for za0010, za0011 := range z.MetaUser { o = msgp.AppendString(o, za0010) @@ -1294,23 +1509,28 @@ func (z *xlMetaV2Object) UnmarshalMsg(bts []byte) (o []byte, err error) { } } case "PartETags": - var zb0006 uint32 - zb0006, bts, err = msgp.ReadArrayHeaderBytes(bts) - if err != nil { - err = msgp.WrapError(err, "PartETags") - return - } - if cap(z.PartETags) >= int(zb0006) { - z.PartETags = (z.PartETags)[:zb0006] + if msgp.IsNil(bts) { + bts = bts[1:] + z.PartETags = nil } else { - z.PartETags = make([]string, zb0006) - } - for za0005 := range z.PartETags { - z.PartETags[za0005], bts, err = msgp.ReadStringBytes(bts) + var zb0006 uint32 + zb0006, bts, err = msgp.ReadArrayHeaderBytes(bts) if err != nil { - err = msgp.WrapError(err, "PartETags", za0005) + err = msgp.WrapError(err, "PartETags") return } + if cap(z.PartETags) >= int(zb0006) { + z.PartETags = (z.PartETags)[:zb0006] + } else { + z.PartETags = make([]string, zb0006) + } + for za0005 := range z.PartETags { + z.PartETags[za0005], bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "PartETags", za0005) + return + } + } } case "PartSizes": var zb0007 uint32 @@ -1332,23 +1552,28 @@ func (z *xlMetaV2Object) UnmarshalMsg(bts []byte) (o []byte, err error) { } } case "PartASizes": - var zb0008 uint32 - zb0008, bts, err = msgp.ReadArrayHeaderBytes(bts) - if err != nil { - err = msgp.WrapError(err, "PartActualSizes") - return - } - if cap(z.PartActualSizes) >= int(zb0008) { - z.PartActualSizes = (z.PartActualSizes)[:zb0008] + if msgp.IsNil(bts) { + bts = bts[1:] + z.PartActualSizes = nil } else { - z.PartActualSizes = make([]int64, zb0008) - } - for za0007 := range z.PartActualSizes { - z.PartActualSizes[za0007], bts, err = msgp.ReadInt64Bytes(bts) + var zb0008 uint32 + zb0008, bts, err = msgp.ReadArrayHeaderBytes(bts) if err != nil { - err = msgp.WrapError(err, "PartActualSizes", za0007) + err = msgp.WrapError(err, "PartActualSizes") return } + if cap(z.PartActualSizes) >= int(zb0008) { + z.PartActualSizes = (z.PartActualSizes)[:zb0008] + } else { + z.PartActualSizes = make([]int64, zb0008) + } + for za0007 := range z.PartActualSizes { + z.PartActualSizes[za0007], bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "PartActualSizes", za0007) + return + } + } } case "Size": z.Size, bts, err = msgp.ReadInt64Bytes(bts) @@ -1363,64 +1588,74 @@ func (z *xlMetaV2Object) UnmarshalMsg(bts []byte) (o []byte, err error) { return } case "MetaSys": - var zb0009 uint32 - zb0009, bts, err = msgp.ReadMapHeaderBytes(bts) - if err != nil { - err = msgp.WrapError(err, "MetaSys") - return - } - if z.MetaSys == nil { - z.MetaSys = make(map[string][]byte, zb0009) - } else if len(z.MetaSys) > 0 { - for key := range z.MetaSys { - delete(z.MetaSys, key) - } - } - for zb0009 > 0 { - var za0008 string - var za0009 []byte - zb0009-- - za0008, bts, err = msgp.ReadStringBytes(bts) + if msgp.IsNil(bts) { + bts = bts[1:] + z.MetaSys = nil + } else { + var zb0009 uint32 + zb0009, bts, err = msgp.ReadMapHeaderBytes(bts) if err != nil { err = msgp.WrapError(err, "MetaSys") return } - za0009, bts, err = msgp.ReadBytesBytes(bts, za0009) - if err != nil { - err = msgp.WrapError(err, "MetaSys", za0008) - return + if z.MetaSys == nil { + z.MetaSys = make(map[string][]byte, zb0009) + } else if len(z.MetaSys) > 0 { + for key := range z.MetaSys { + delete(z.MetaSys, key) + } + } + for zb0009 > 0 { + var za0008 string + var za0009 []byte + zb0009-- + za0008, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "MetaSys") + return + } + za0009, bts, err = msgp.ReadBytesBytes(bts, za0009) + if err != nil { + err = msgp.WrapError(err, "MetaSys", za0008) + return + } + z.MetaSys[za0008] = za0009 } - z.MetaSys[za0008] = za0009 } case "MetaUsr": - var zb0010 uint32 - zb0010, bts, err = msgp.ReadMapHeaderBytes(bts) - if err != nil { - err = msgp.WrapError(err, "MetaUser") - return - } - if z.MetaUser == nil { - z.MetaUser = make(map[string]string, zb0010) - } else if len(z.MetaUser) > 0 { - for key := range z.MetaUser { - delete(z.MetaUser, key) - } - } - for zb0010 > 0 { - var za0010 string - var za0011 string - zb0010-- - za0010, bts, err = msgp.ReadStringBytes(bts) + if msgp.IsNil(bts) { + bts = bts[1:] + z.MetaUser = nil + } else { + var zb0010 uint32 + zb0010, bts, err = msgp.ReadMapHeaderBytes(bts) if err != nil { err = msgp.WrapError(err, "MetaUser") return } - za0011, bts, err = msgp.ReadStringBytes(bts) - if err != nil { - err = msgp.WrapError(err, "MetaUser", za0010) - return + if z.MetaUser == nil { + z.MetaUser = make(map[string]string, zb0010) + } else if len(z.MetaUser) > 0 { + for key := range z.MetaUser { + delete(z.MetaUser, key) + } + } + for zb0010 > 0 { + var za0010 string + var za0011 string + zb0010-- + za0010, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "MetaUser") + return + } + za0011, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "MetaUser", za0010) + return + } + z.MetaUser[za0010] = za0011 } - z.MetaUser[za0010] = za0011 } default: bts, err = msgp.Skip(bts) @@ -1826,3 +2061,154 @@ func (z *xlMetaV2Version) Msgsize() (s int) { } return } + +// DecodeMsg implements msgp.Decodable +func (z *xlMetaV2VersionHeader) DecodeMsg(dc *msgp.Reader) (err error) { + var zb0001 uint32 + zb0001, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 != 5 { + err = msgp.ArrayError{Wanted: 5, Got: zb0001} + return + } + err = dc.ReadExactBytes((z.VersionID)[:]) + if err != nil { + err = msgp.WrapError(err, "VersionID") + return + } + z.ModTime, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "ModTime") + return + } + err = dc.ReadExactBytes((z.Signature)[:]) + if err != nil { + err = msgp.WrapError(err, "Signature") + return + } + { + var zb0002 uint8 + zb0002, err = dc.ReadUint8() + if err != nil { + err = msgp.WrapError(err, "Type") + return + } + z.Type = VersionType(zb0002) + } + { + var zb0003 uint8 + zb0003, err = dc.ReadUint8() + if err != nil { + err = msgp.WrapError(err, "Flags") + return + } + z.Flags = xlFlags(zb0003) + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *xlMetaV2VersionHeader) EncodeMsg(en *msgp.Writer) (err error) { + // array header, size 5 + err = en.Append(0x95) + if err != nil { + return + } + err = en.WriteBytes((z.VersionID)[:]) + if err != nil { + err = msgp.WrapError(err, "VersionID") + return + } + err = en.WriteInt64(z.ModTime) + if err != nil { + err = msgp.WrapError(err, "ModTime") + return + } + err = en.WriteBytes((z.Signature)[:]) + if err != nil { + err = msgp.WrapError(err, "Signature") + return + } + err = en.WriteUint8(uint8(z.Type)) + if err != nil { + err = msgp.WrapError(err, "Type") + return + } + err = en.WriteUint8(uint8(z.Flags)) + if err != nil { + err = msgp.WrapError(err, "Flags") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *xlMetaV2VersionHeader) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // array header, size 5 + o = append(o, 0x95) + o = msgp.AppendBytes(o, (z.VersionID)[:]) + o = msgp.AppendInt64(o, z.ModTime) + o = msgp.AppendBytes(o, (z.Signature)[:]) + o = msgp.AppendUint8(o, uint8(z.Type)) + o = msgp.AppendUint8(o, uint8(z.Flags)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *xlMetaV2VersionHeader) UnmarshalMsg(bts []byte) (o []byte, err error) { + var zb0001 uint32 + zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 != 5 { + err = msgp.ArrayError{Wanted: 5, Got: zb0001} + return + } + bts, err = msgp.ReadExactBytes(bts, (z.VersionID)[:]) + if err != nil { + err = msgp.WrapError(err, "VersionID") + return + } + z.ModTime, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ModTime") + return + } + bts, err = msgp.ReadExactBytes(bts, (z.Signature)[:]) + if err != nil { + err = msgp.WrapError(err, "Signature") + return + } + { + var zb0002 uint8 + zb0002, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Type") + return + } + z.Type = VersionType(zb0002) + } + { + var zb0003 uint8 + zb0003, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Flags") + return + } + z.Flags = xlFlags(zb0003) + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *xlMetaV2VersionHeader) Msgsize() (s int) { + s = 1 + msgp.ArrayHeaderSize + (16 * (msgp.ByteSize)) + msgp.Int64Size + msgp.ArrayHeaderSize + (4 * (msgp.ByteSize)) + msgp.Uint8Size + msgp.Uint8Size + return +} diff --git a/cmd/xl-storage-format-v2_gen_test.go b/cmd/xl-storage-format-v2_gen_test.go index 39f03c898..afcc74dae 100644 --- a/cmd/xl-storage-format-v2_gen_test.go +++ b/cmd/xl-storage-format-v2_gen_test.go @@ -9,8 +9,8 @@ import ( "github.com/tinylib/msgp/msgp" ) -func TestMarshalUnmarshalxlMetaV2(t *testing.T) { - v := xlMetaV2{} +func TestMarshalUnmarshalxlMetaDataDirDecoder(t *testing.T) { + v := xlMetaDataDirDecoder{} bts, err := v.MarshalMsg(nil) if err != nil { t.Fatal(err) @@ -32,8 +32,8 @@ func TestMarshalUnmarshalxlMetaV2(t *testing.T) { } } -func BenchmarkMarshalMsgxlMetaV2(b *testing.B) { - v := xlMetaV2{} +func BenchmarkMarshalMsgxlMetaDataDirDecoder(b *testing.B) { + v := xlMetaDataDirDecoder{} b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { @@ -41,8 +41,8 @@ func BenchmarkMarshalMsgxlMetaV2(b *testing.B) { } } -func BenchmarkAppendMsgxlMetaV2(b *testing.B) { - v := xlMetaV2{} +func BenchmarkAppendMsgxlMetaDataDirDecoder(b *testing.B) { + v := xlMetaDataDirDecoder{} bts := make([]byte, 0, v.Msgsize()) bts, _ = v.MarshalMsg(bts[0:0]) b.SetBytes(int64(len(bts))) @@ -53,8 +53,8 @@ func BenchmarkAppendMsgxlMetaV2(b *testing.B) { } } -func BenchmarkUnmarshalxlMetaV2(b *testing.B) { - v := xlMetaV2{} +func BenchmarkUnmarshalxlMetaDataDirDecoder(b *testing.B) { + v := xlMetaDataDirDecoder{} bts, _ := v.MarshalMsg(nil) b.ReportAllocs() b.SetBytes(int64(len(bts))) @@ -67,17 +67,17 @@ func BenchmarkUnmarshalxlMetaV2(b *testing.B) { } } -func TestEncodeDecodexlMetaV2(t *testing.T) { - v := xlMetaV2{} +func TestEncodeDecodexlMetaDataDirDecoder(t *testing.T) { + v := xlMetaDataDirDecoder{} var buf bytes.Buffer msgp.Encode(&buf, &v) m := v.Msgsize() if buf.Len() > m { - t.Log("WARNING: TestEncodeDecodexlMetaV2 Msgsize() is inaccurate") + t.Log("WARNING: TestEncodeDecodexlMetaDataDirDecoder Msgsize() is inaccurate") } - vn := xlMetaV2{} + vn := xlMetaDataDirDecoder{} err := msgp.Decode(&buf, &vn) if err != nil { t.Error(err) @@ -91,8 +91,8 @@ func TestEncodeDecodexlMetaV2(t *testing.T) { } } -func BenchmarkEncodexlMetaV2(b *testing.B) { - v := xlMetaV2{} +func BenchmarkEncodexlMetaDataDirDecoder(b *testing.B) { + v := xlMetaDataDirDecoder{} var buf bytes.Buffer msgp.Encode(&buf, &v) b.SetBytes(int64(buf.Len())) @@ -105,8 +105,8 @@ func BenchmarkEncodexlMetaV2(b *testing.B) { en.Flush() } -func BenchmarkDecodexlMetaV2(b *testing.B) { - v := xlMetaV2{} +func BenchmarkDecodexlMetaDataDirDecoder(b *testing.B) { + v := xlMetaDataDirDecoder{} var buf bytes.Buffer msgp.Encode(&buf, &v) b.SetBytes(int64(buf.Len())) @@ -460,3 +460,116 @@ func BenchmarkDecodexlMetaV2Version(b *testing.B) { } } } + +func TestMarshalUnmarshalxlMetaV2VersionHeader(t *testing.T) { + v := xlMetaV2VersionHeader{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgxlMetaV2VersionHeader(b *testing.B) { + v := xlMetaV2VersionHeader{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgxlMetaV2VersionHeader(b *testing.B) { + v := xlMetaV2VersionHeader{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalxlMetaV2VersionHeader(b *testing.B) { + v := xlMetaV2VersionHeader{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodexlMetaV2VersionHeader(t *testing.T) { + v := xlMetaV2VersionHeader{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodexlMetaV2VersionHeader Msgsize() is inaccurate") + } + + vn := xlMetaV2VersionHeader{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodexlMetaV2VersionHeader(b *testing.B) { + v := xlMetaV2VersionHeader{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodexlMetaV2VersionHeader(b *testing.B) { + v := xlMetaV2VersionHeader{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/cmd/xl-storage-format-v2_string.go b/cmd/xl-storage-format-v2_string.go new file mode 100644 index 000000000..8ebf4ace3 --- /dev/null +++ b/cmd/xl-storage-format-v2_string.go @@ -0,0 +1,27 @@ +// Code generated by "stringer -type VersionType -output=xl-storage-format-v2_string.go xl-storage-format-v2.go"; DO NOT EDIT. + +package cmd + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[invalidVersionType-0] + _ = x[ObjectType-1] + _ = x[DeleteType-2] + _ = x[LegacyType-3] + _ = x[lastVersionType-4] +} + +const _VersionType_name = "invalidVersionTypeObjectTypeDeleteTypeLegacyTypelastVersionType" + +var _VersionType_index = [...]uint8{0, 18, 28, 38, 48, 63} + +func (i VersionType) String() string { + if i >= VersionType(len(_VersionType_index)-1) { + return "VersionType(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _VersionType_name[_VersionType_index[i]:_VersionType_index[i+1]] +} diff --git a/cmd/xl-storage-format-v2_test.go b/cmd/xl-storage-format-v2_test.go index 4b55a4132..0c8548562 100644 --- a/cmd/xl-storage-format-v2_test.go +++ b/cmd/xl-storage-format-v2_test.go @@ -19,12 +19,15 @@ package cmd import ( "bytes" + "sort" "testing" "time" "github.com/google/uuid" + "github.com/klauspost/compress/zstd" "github.com/minio/minio/internal/bucket/lifecycle" xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/ioutil" ) func TestXLV2FormatData(t *testing.T) { @@ -341,15 +344,17 @@ func TestDeleteVersionWithSharedDataDir(t *testing.T) { } } fi.TransitionStatus = tc.transitionStatus + fi.ModTime = fi.ModTime.Add(time.Duration(i) * time.Second) failOnErr(i+1, xl.AddVersion(fi)) fi.ExpireRestored = tc.expireRestored fileInfos = append(fileInfos, fi) } for i, tc := range testCases { - version := xl.Versions[i] - if actual := xl.SharedDataDirCount(version.ObjectV2.VersionID, version.ObjectV2.DataDir); actual != tc.shares { - t.Fatalf("Test %d: For %#v, expected sharers of data directory %d got %d", i+1, version.ObjectV2, tc.shares, actual) + _, version, err := xl.findVersion(uuid.MustParse(tc.versionID)) + failOnErr(i+1, err) + if got := xl.SharedDataDirCount(version.getVersionID(), version.ObjectV2.DataDir); got != tc.shares { + t.Fatalf("Test %d: For %#v, expected sharers of data directory %d got %d", i+1, version.ObjectV2.VersionID, tc.shares, got) } } @@ -366,3 +371,110 @@ func TestDeleteVersionWithSharedDataDir(t *testing.T) { count++ } } + +func Benchmark_xlMetaV2Shallow_Load(b *testing.B) { + data, err := ioutil.ReadFile("testdata/xl.meta-v1.2.zst") + if err != nil { + b.Fatal(err) + } + dec, _ := zstd.NewReader(nil) + data, err = dec.DecodeAll(data, nil) + if err != nil { + b.Fatal(err) + } + + b.Run("legacy", func(b *testing.B) { + var xl xlMetaV2 + b.ReportAllocs() + b.ResetTimer() + b.SetBytes(855) // number of versions... + for i := 0; i < b.N; i++ { + err = xl.Load(data) + if err != nil { + b.Fatal(err) + } + } + }) + b.Run("indexed", func(b *testing.B) { + var xl xlMetaV2 + err = xl.Load(data) + if err != nil { + b.Fatal(err) + } + data, err := xl.AppendTo(nil) + if err != nil { + b.Fatal(err) + } + b.ReportAllocs() + b.ResetTimer() + b.SetBytes(855) // number of versions... + for i := 0; i < b.N; i++ { + err = xl.Load(data) + if err != nil { + b.Fatal(err) + } + } + }) + +} + +func Test_xlMetaV2Shallow_Load(t *testing.T) { + // Load Legacy + data, err := ioutil.ReadFile("testdata/xl.meta-v1.2.zst") + if err != nil { + t.Fatal(err) + } + dec, _ := zstd.NewReader(nil) + data, err = dec.DecodeAll(data, nil) + if err != nil { + t.Fatal(err) + } + test := func(t *testing.T, xl *xlMetaV2) { + if len(xl.versions) != 855 { + t.Errorf("want %d versions, got %d", 855, len(xl.versions)) + } + xl.sortByModTime() + if !sort.SliceIsSorted(xl.versions, func(i, j int) bool { + return xl.versions[i].header.ModTime > xl.versions[j].header.ModTime + }) { + t.Errorf("Contents not sorted") + } + for i := range xl.versions { + hdr := xl.versions[i].header + ver, err := xl.getIdx(i) + if err != nil { + t.Error(err) + continue + } + gotHdr := ver.header() + if hdr != gotHdr { + t.Errorf("Header does not match, index: %+v != meta: %+v", hdr, gotHdr) + } + } + } + t.Run("load-legacy", func(t *testing.T) { + var xl xlMetaV2 + err = xl.Load(data) + if err != nil { + t.Fatal(err) + } + test(t, &xl) + }) + t.Run("roundtrip", func(t *testing.T) { + var xl xlMetaV2 + err = xl.Load(data) + if err != nil { + t.Fatal(err) + } + data, err = xl.AppendTo(nil) + if err != nil { + t.Fatal(err) + } + xl = xlMetaV2{} + err = xl.Load(data) + if err != nil { + t.Fatal(err) + } + test(t, &xl) + }) +} diff --git a/cmd/xl-storage-format_test.go b/cmd/xl-storage-format_test.go index 4fd621234..8f4f82f36 100644 --- a/cmd/xl-storage-format_test.go +++ b/cmd/xl-storage-format_test.go @@ -21,10 +21,14 @@ import ( "bytes" "encoding/hex" "encoding/json" + "fmt" + "math/rand" "testing" + "time" "github.com/dustin/go-humanize" jsoniter "github.com/json-iterator/go" + xhttp "github.com/minio/minio/internal/http" ) func TestIsXLMetaFormatValid(t *testing.T) { @@ -317,3 +321,221 @@ func TestGetPartSizeFromIdx(t *testing.T) { } } } + +func BenchmarkXlMetaV2Shallow(b *testing.B) { + fi := FileInfo{ + Volume: "volume", + Name: "object-name", + VersionID: "756100c6-b393-4981-928a-d49bbc164741", + IsLatest: true, + Deleted: false, + TransitionStatus: "PENDING", + DataDir: "bffea160-ca7f-465f-98bc-9b4f1c3ba1ef", + XLV1: false, + ModTime: time.Now(), + Size: 1234456, + Mode: 0, + Metadata: map[string]string{ + xhttp.AmzRestore: "FAILED", + xhttp.ContentMD5: mustGetUUID(), + xhttp.AmzBucketReplicationStatus: "PENDING", + xhttp.ContentType: "application/json", + }, + Parts: []ObjectPartInfo{{ + Number: 1, + Size: 1234345, + ActualSize: 1234345, + }, + { + Number: 2, + Size: 1234345, + ActualSize: 1234345, + }, + }, + Erasure: ErasureInfo{ + Algorithm: ReedSolomon.String(), + DataBlocks: 4, + ParityBlocks: 2, + BlockSize: 10000, + Index: 1, + Distribution: []int{1, 2, 3, 4, 5, 6, 7, 8}, + Checksums: []ChecksumInfo{{ + PartNumber: 1, + Algorithm: HighwayHash256S, + Hash: nil, + }, + { + PartNumber: 2, + Algorithm: HighwayHash256S, + Hash: nil, + }, + }, + }, + } + for _, size := range []int{1, 10, 1000, 100_000} { + b.Run(fmt.Sprint(size, "-versions"), func(b *testing.B) { + var xl xlMetaV2 + ids := make([]string, size) + for i := 0; i < size; i++ { + fi.VersionID = mustGetUUID() + fi.DataDir = mustGetUUID() + ids[i] = fi.VersionID + fi.ModTime = fi.ModTime.Add(-time.Second) + xl.AddVersion(fi) + } + // Encode all. This is used for benchmarking. + enc, err := xl.AppendTo(nil) + if err != nil { + b.Fatal(err) + } + b.Logf("Serialized size: %d bytes", len(enc)) + rng := rand.New(rand.NewSource(0)) + var dump = make([]byte, len(enc)) + b.Run("UpdateObjectVersion", func(b *testing.B) { + b.SetBytes(int64(size)) + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + // Load... + xl = xlMetaV2{} + err := xl.Load(enc) + if err != nil { + b.Fatal(err) + } + // Update modtime for resorting... + fi.ModTime = fi.ModTime.Add(-time.Second) + // Update a random version. + fi.VersionID = ids[rng.Intn(size)] + // Update... + err = xl.UpdateObjectVersion(fi) + if err != nil { + b.Fatal(err) + } + // Save... + dump, err = xl.AppendTo(dump[:0]) + if err != nil { + b.Fatal(err) + } + } + }) + b.Run("DeleteVersion", func(b *testing.B) { + b.SetBytes(int64(size)) + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + // Load... + xl = xlMetaV2{} + err := xl.Load(enc) + if err != nil { + b.Fatal(err) + } + // Update a random version. + fi.VersionID = ids[rng.Intn(size)] + // Delete... + _, _, err = xl.DeleteVersion(fi) + if err != nil { + b.Fatal(err) + } + // Save... + dump, err = xl.AppendTo(dump[:0]) + if err != nil { + b.Fatal(err) + } + } + }) + b.Run("AddVersion", func(b *testing.B) { + b.SetBytes(int64(size)) + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + // Load... + xl = xlMetaV2{} + err := xl.Load(enc) + if err != nil { + b.Fatal(err) + } + // Update modtime for resorting... + fi.ModTime = fi.ModTime.Add(-time.Second) + // Update a random version. + fi.VersionID = mustGetUUID() + // Add... + err = xl.AddVersion(fi) + if err != nil { + b.Fatal(err) + } + // Save... + dump, err = xl.AppendTo(dump[:0]) + if err != nil { + b.Fatal(err) + } + } + }) + b.Run("ToFileInfo", func(b *testing.B) { + b.SetBytes(int64(size)) + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + // Load... + xl = xlMetaV2{} + err := xl.Load(enc) + if err != nil { + b.Fatal(err) + } + // List... + _, err = xl.ToFileInfo("volume", "path", ids[rng.Intn(size)]) + if err != nil { + b.Fatal(err) + } + } + }) + b.Run("ListVersions", func(b *testing.B) { + b.SetBytes(int64(size)) + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + // Load... + xl = xlMetaV2{} + err := xl.Load(enc) + if err != nil { + b.Fatal(err) + } + // List... + _, err = xl.ListVersions("volume", "path") + if err != nil { + b.Fatal(err) + } + } + }) + b.Run("ToFileInfoNew", func(b *testing.B) { + b.SetBytes(int64(size)) + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + buf, _ := isIndexedMetaV2(enc) + if buf == nil { + b.Fatal("buf == nil") + } + _, err = buf.ToFileInfo("volume", "path", ids[rng.Intn(size)]) + if err != nil { + b.Fatal(err) + } + } + }) + b.Run("ListVersionsNew", func(b *testing.B) { + b.SetBytes(int64(size)) + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + buf, _ := isIndexedMetaV2(enc) + if buf == nil { + b.Fatal("buf == nil") + } + _, err = buf.ListVersions("volume", "path") + if err != nil { + b.Fatal(err) + } + } + }) + }) + } +} diff --git a/cmd/xl-storage-free-version.go b/cmd/xl-storage-free-version.go index 44e55ffaa..3187595e1 100644 --- a/cmd/xl-storage-free-version.go +++ b/cmd/xl-storage-free-version.go @@ -75,7 +75,7 @@ func (j xlMetaV2Version) FreeVersion() bool { // AddFreeVersion adds a free-version if needed for fi.VersionID version. // Free-version will be added if fi.VersionID has transitioned. -func (z *xlMetaV2) AddFreeVersion(fi FileInfo) error { +func (x *xlMetaV2) AddFreeVersion(fi FileInfo) error { var uv uuid.UUID var err error switch fi.VersionID { @@ -87,19 +87,22 @@ func (z *xlMetaV2) AddFreeVersion(fi FileInfo) error { } } - for _, version := range z.Versions { - switch version.Type { - case ObjectType: - if version.ObjectV2.VersionID == uv { - // if uv has tiered content we add a - // free-version to track it for asynchronous - // deletion via scanner. - if freeVersion, toFree := version.ObjectV2.InitFreeVersion(fi); toFree { - z.Versions = append(z.Versions, freeVersion) - } - return nil - } + for i, version := range x.versions { + if version.header.VersionID != uv || version.header.Type != ObjectType { + continue } + // if uv has tiered content we add a + // free-version to track it for asynchronous + // deletion via scanner. + ver, err := x.getIdx(i) + if err != nil { + return err + } + + if freeVersion, toFree := ver.ObjectV2.InitFreeVersion(fi); toFree { + return x.addVersion(freeVersion) + } + return nil } return nil } diff --git a/cmd/xl-storage-free-version_test.go b/cmd/xl-storage-free-version_test.go index 40e6d1e22..ea81b2c24 100644 --- a/cmd/xl-storage-free-version_test.go +++ b/cmd/xl-storage-free-version_test.go @@ -24,8 +24,8 @@ import ( "github.com/minio/minio/internal/bucket/lifecycle" ) -func (z xlMetaV2) listFreeVersions(volume, path string) ([]FileInfo, error) { - fivs, _, err := z.ListVersions(volume, path) +func (x xlMetaV2) listFreeVersions(volume, path string) ([]FileInfo, error) { + fivs, err := x.ListVersions(volume, path) if err != nil { return nil, err } @@ -41,8 +41,21 @@ func (z xlMetaV2) listFreeVersions(volume, path string) ([]FileInfo, error) { } func TestFreeVersion(t *testing.T) { + fatalErr := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + // Add a version with tiered content, one with local content xl := xlMetaV2{} + counter := 1 + report := func() { + t.Helper() + // t.Logf("versions (%d): len = %d", counter, len(xl.versions)) + counter++ + } fi := FileInfo{ Volume: "volume", Name: "object-name", @@ -77,16 +90,21 @@ func TestFreeVersion(t *testing.T) { SuccessorModTime: time.Time{}, } // Add a version with local content - xl.AddVersion(fi) + fatalErr(xl.AddVersion(fi)) + report() // Add null version with tiered content tierfi := fi tierfi.VersionID = "" - xl.AddVersion(tierfi) + fatalErr(xl.AddVersion(tierfi)) + report() tierfi.TransitionStatus = lifecycle.TransitionComplete tierfi.TransitionedObjName = mustGetUUID() tierfi.TransitionTier = "MINIOTIER-1" - xl.DeleteVersion(tierfi) + var err error + _, _, err = xl.DeleteVersion(tierfi) + fatalErr(err) + report() fvIDs := []string{ "00000000-0000-0000-0000-0000000000f1", @@ -95,15 +113,20 @@ func TestFreeVersion(t *testing.T) { // Simulate overwrite of null version newtierfi := tierfi newtierfi.SetTierFreeVersionID(fvIDs[0]) - xl.AddFreeVersion(newtierfi) - xl.AddVersion(newtierfi) + fatalErr(xl.AddFreeVersion(newtierfi)) + report() + fatalErr(xl.AddVersion(newtierfi)) + report() // Simulate removal of null version newtierfi.TransitionTier = "" newtierfi.TransitionedObjName = "" newtierfi.TransitionStatus = "" newtierfi.SetTierFreeVersionID(fvIDs[1]) - xl.DeleteVersion(newtierfi) + report() + _, _, err = xl.DeleteVersion(newtierfi) + report() + fatalErr(err) // Check number of free-versions freeVersions, err := xl.listFreeVersions(newtierfi.Volume, newtierfi.Name) @@ -118,8 +141,10 @@ func TestFreeVersion(t *testing.T) { freefi := newtierfi for _, fvID := range fvIDs { freefi.VersionID = fvID - xl.DeleteVersion(freefi) + _, _, err = xl.DeleteVersion(freefi) + fatalErr(err) } + report() // Check number of free-versions freeVersions, err = xl.listFreeVersions(newtierfi.Volume, newtierfi.Name) @@ -129,11 +154,13 @@ func TestFreeVersion(t *testing.T) { if len(freeVersions) != 0 { t.Fatalf("Expected zero free version but got %d", len(freeVersions)) } + report() // Adding a free version to a version with no tiered content. newfi := fi newfi.SetTierFreeVersionID("00000000-0000-0000-0000-0000000000f3") - xl.AddFreeVersion(newfi) // this shouldn't add a free-version + fatalErr(xl.AddFreeVersion(newfi)) // this shouldn't add a free-version + report() // Check number of free-versions freeVersions, err = xl.listFreeVersions(newtierfi.Volume, newtierfi.Name) diff --git a/cmd/xl-storage-meta-inline.go b/cmd/xl-storage-meta-inline.go new file mode 100644 index 000000000..d8259c555 --- /dev/null +++ b/cmd/xl-storage-meta-inline.go @@ -0,0 +1,408 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "errors" + "fmt" + + "github.com/minio/minio/internal/logger" + "github.com/tinylib/msgp/msgp" +) + +// xlMetaInlineData is serialized data in [string][]byte pairs. +type xlMetaInlineData []byte + +// xlMetaInlineDataVer indicates the version of the inline data structure. +const xlMetaInlineDataVer = 1 + +// versionOK returns whether the version is ok. +func (x xlMetaInlineData) versionOK() bool { + if len(x) == 0 { + return true + } + return x[0] > 0 && x[0] <= xlMetaInlineDataVer +} + +// afterVersion returns the payload after the version, if any. +func (x xlMetaInlineData) afterVersion() []byte { + if len(x) == 0 { + return x + } + return x[1:] +} + +// find the data with key s. +// Returns nil if not for or an error occurs. +func (x xlMetaInlineData) find(key string) []byte { + if len(x) == 0 || !x.versionOK() { + return nil + } + sz, buf, err := msgp.ReadMapHeaderBytes(x.afterVersion()) + if err != nil || sz == 0 { + return nil + } + for i := uint32(0); i < sz; i++ { + var found []byte + found, buf, err = msgp.ReadMapKeyZC(buf) + if err != nil || sz == 0 { + return nil + } + if string(found) == key { + val, _, _ := msgp.ReadBytesZC(buf) + return val + } + // Skip it + _, buf, err = msgp.ReadBytesZC(buf) + if err != nil { + return nil + } + } + return nil +} + +// validate checks if the data is valid. +// It does not check integrity of the stored data. +func (x xlMetaInlineData) validate() error { + if len(x) == 0 { + return nil + } + + if !x.versionOK() { + return fmt.Errorf("xlMetaInlineData: unknown version 0x%x", x[0]) + } + + sz, buf, err := msgp.ReadMapHeaderBytes(x.afterVersion()) + if err != nil { + return fmt.Errorf("xlMetaInlineData: %w", err) + } + + for i := uint32(0); i < sz; i++ { + var key []byte + key, buf, err = msgp.ReadMapKeyZC(buf) + if err != nil { + return fmt.Errorf("xlMetaInlineData: %w", err) + } + if len(key) == 0 { + return fmt.Errorf("xlMetaInlineData: key %d is length 0", i) + } + _, buf, err = msgp.ReadBytesZC(buf) + if err != nil { + return fmt.Errorf("xlMetaInlineData: %w", err) + } + } + + return nil +} + +// repair will copy all seemingly valid data entries from a corrupted set. +// This does not ensure that data is correct, but will allow all operations to complete. +func (x *xlMetaInlineData) repair() { + data := *x + if len(data) == 0 { + return + } + + if !data.versionOK() { + *x = nil + return + } + + sz, buf, err := msgp.ReadMapHeaderBytes(data.afterVersion()) + if err != nil { + *x = nil + return + } + + // Remove all current data + keys := make([][]byte, 0, sz) + vals := make([][]byte, 0, sz) + for i := uint32(0); i < sz; i++ { + var key, val []byte + key, buf, err = msgp.ReadMapKeyZC(buf) + if err != nil { + break + } + if len(key) == 0 { + break + } + val, buf, err = msgp.ReadBytesZC(buf) + if err != nil { + break + } + keys = append(keys, key) + vals = append(vals, val) + } + x.serialize(-1, keys, vals) +} + +// validate checks if the data is valid. +// It does not check integrity of the stored data. +func (x xlMetaInlineData) list() ([]string, error) { + if len(x) == 0 { + return nil, nil + } + if !x.versionOK() { + return nil, errors.New("xlMetaInlineData: unknown version") + } + + sz, buf, err := msgp.ReadMapHeaderBytes(x.afterVersion()) + if err != nil { + return nil, err + } + keys := make([]string, 0, sz) + for i := uint32(0); i < sz; i++ { + var key []byte + key, buf, err = msgp.ReadMapKeyZC(buf) + if err != nil { + return keys, err + } + if len(key) == 0 { + return keys, fmt.Errorf("xlMetaInlineData: key %d is length 0", i) + } + keys = append(keys, string(key)) + // Skip data... + _, buf, err = msgp.ReadBytesZC(buf) + if err != nil { + return keys, err + } + } + return keys, nil +} + +// serialize will serialize the provided keys and values. +// The function will panic if keys/value slices aren't of equal length. +// Payload size can give an indication of expected payload size. +// If plSize is <= 0 it will be calculated. +func (x *xlMetaInlineData) serialize(plSize int, keys [][]byte, vals [][]byte) { + if len(keys) != len(vals) { + panic(fmt.Errorf("xlMetaInlineData.serialize: keys/value number mismatch")) + } + if len(keys) == 0 { + *x = nil + return + } + if plSize <= 0 { + plSize = 1 + msgp.MapHeaderSize + for i := range keys { + plSize += len(keys[i]) + len(vals[i]) + msgp.StringPrefixSize + msgp.ArrayHeaderSize + } + } + payload := make([]byte, 1, plSize) + payload[0] = xlMetaInlineDataVer + payload = msgp.AppendMapHeader(payload, uint32(len(keys))) + for i := range keys { + payload = msgp.AppendStringFromBytes(payload, keys[i]) + payload = msgp.AppendBytes(payload, vals[i]) + } + *x = payload +} + +// entries returns the number of entries in the data. +func (x xlMetaInlineData) entries() int { + if len(x) == 0 || !x.versionOK() { + return 0 + } + sz, _, _ := msgp.ReadMapHeaderBytes(x.afterVersion()) + return int(sz) +} + +// replace will add or replace a key/value pair. +func (x *xlMetaInlineData) replace(key string, value []byte) { + in := x.afterVersion() + sz, buf, _ := msgp.ReadMapHeaderBytes(in) + keys := make([][]byte, 0, sz+1) + vals := make([][]byte, 0, sz+1) + + // Version plus header... + plSize := 1 + msgp.MapHeaderSize + replaced := false + for i := uint32(0); i < sz; i++ { + var found, foundVal []byte + var err error + found, buf, err = msgp.ReadMapKeyZC(buf) + if err != nil { + break + } + foundVal, buf, err = msgp.ReadBytesZC(buf) + if err != nil { + break + } + plSize += len(found) + msgp.StringPrefixSize + msgp.ArrayHeaderSize + keys = append(keys, found) + if string(found) == key { + vals = append(vals, value) + plSize += len(value) + replaced = true + } else { + vals = append(vals, foundVal) + plSize += len(foundVal) + } + } + + // Add one more. + if !replaced { + keys = append(keys, []byte(key)) + vals = append(vals, value) + plSize += len(key) + len(value) + msgp.StringPrefixSize + msgp.ArrayHeaderSize + } + + // Reserialize... + x.serialize(plSize, keys, vals) +} + +// rename will rename a key. +// Returns whether the key was found. +func (x *xlMetaInlineData) rename(oldKey, newKey string) bool { + in := x.afterVersion() + sz, buf, _ := msgp.ReadMapHeaderBytes(in) + keys := make([][]byte, 0, sz) + vals := make([][]byte, 0, sz) + + // Version plus header... + plSize := 1 + msgp.MapHeaderSize + found := false + for i := uint32(0); i < sz; i++ { + var foundKey, foundVal []byte + var err error + foundKey, buf, err = msgp.ReadMapKeyZC(buf) + if err != nil { + break + } + foundVal, buf, err = msgp.ReadBytesZC(buf) + if err != nil { + break + } + plSize += len(foundVal) + msgp.StringPrefixSize + msgp.ArrayHeaderSize + vals = append(vals, foundVal) + if string(foundKey) != oldKey { + keys = append(keys, foundKey) + plSize += len(foundKey) + } else { + keys = append(keys, []byte(newKey)) + plSize += len(newKey) + found = true + } + } + // If not found, just return. + if !found { + return false + } + + // Reserialize... + x.serialize(plSize, keys, vals) + return true +} + +// remove will remove one or more keys. +// Returns true if any key was found. +func (x *xlMetaInlineData) remove(keys ...string) bool { + in := x.afterVersion() + sz, buf, _ := msgp.ReadMapHeaderBytes(in) + newKeys := make([][]byte, 0, sz) + newVals := make([][]byte, 0, sz) + var removeKey func(s []byte) bool + + // Copy if big number of compares... + if len(keys) > 5 && sz > 5 { + mKeys := make(map[string]struct{}, len(keys)) + for _, key := range keys { + mKeys[key] = struct{}{} + } + removeKey = func(s []byte) bool { + _, ok := mKeys[string(s)] + return ok + } + } else { + removeKey = func(s []byte) bool { + for _, key := range keys { + if key == string(s) { + return true + } + } + return false + } + } + + // Version plus header... + plSize := 1 + msgp.MapHeaderSize + found := false + for i := uint32(0); i < sz; i++ { + var foundKey, foundVal []byte + var err error + foundKey, buf, err = msgp.ReadMapKeyZC(buf) + if err != nil { + break + } + foundVal, buf, err = msgp.ReadBytesZC(buf) + if err != nil { + break + } + if !removeKey(foundKey) { + plSize += msgp.StringPrefixSize + msgp.ArrayHeaderSize + len(foundKey) + len(foundVal) + newKeys = append(newKeys, foundKey) + newVals = append(newVals, foundVal) + } else { + found = true + } + } + // If not found, just return. + if !found { + return false + } + // If none left... + if len(newKeys) == 0 { + *x = nil + return true + } + + // Reserialize... + x.serialize(plSize, newKeys, newVals) + return true +} + +// xlMetaV2TrimData will trim any data from the metadata without unmarshalling it. +// If any error occurs the unmodified data is returned. +func xlMetaV2TrimData(buf []byte) []byte { + metaBuf, min, maj, err := checkXL2V1(buf) + if err != nil { + return buf + } + if maj == 1 && min < 1 { + // First version to carry data. + return buf + } + // Skip header + _, metaBuf, err = msgp.ReadBytesZC(metaBuf) + if err != nil { + logger.LogIf(GlobalContext, err) + return buf + } + // Skip CRC + if maj > 1 || min >= 2 { + _, metaBuf, err = msgp.ReadUint32Bytes(metaBuf) + logger.LogIf(GlobalContext, err) + } + // = input - current pos + ends := len(buf) - len(metaBuf) + if ends > len(buf) { + return buf + } + + return buf[:ends] +} diff --git a/cmd/xl-storage.go b/cmd/xl-storage.go index f8b352749..9f6c9b93d 100644 --- a/cmd/xl-storage.go +++ b/cmd/xl-storage.go @@ -978,7 +978,7 @@ func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi F } var xlMeta xlMetaV2 - if err = xlMeta.Load(buf); err != nil { + if err := xlMeta.Load(buf); err != nil { return err } @@ -1044,6 +1044,7 @@ func (s *xlStorage) UpdateMetadata(ctx context.Context, volume, path string, fi } return err } + defer metaDataPoolPut(buf) if !isXL2V1Format(buf) { return errFileVersionNotFound @@ -1059,12 +1060,13 @@ func (s *xlStorage) UpdateMetadata(ctx context.Context, volume, path string, fi return err } - buf, err = xlMeta.AppendTo(nil) + wbuf, err := xlMeta.AppendTo(metaDataPoolGet()) if err != nil { return err } + defer metaDataPoolPut(wbuf) - return s.WriteAll(ctx, volume, pathJoin(path, xlStorageFormatFile), buf) + return s.WriteAll(ctx, volume, pathJoin(path, xlStorageFormatFile), wbuf) } // WriteMetadata - writes FileInfo metadata for path at `xl.meta` diff --git a/docs/debugging/xl-meta/main.go b/docs/debugging/xl-meta/main.go index 8c5d6e8b8..55da4ab53 100644 --- a/docs/debugging/xl-meta/main.go +++ b/docs/debugging/xl-meta/main.go @@ -114,6 +114,48 @@ FLAGS: return nil, err } data = b + case 3: + v, b, err := msgp.ReadBytesZC(b) + if err != nil { + return nil, err + } + if _, nbuf, err := msgp.ReadUint32Bytes(b); err == nil { + // Read metadata CRC (added in v2, ignore if not found) + b = nbuf + } + + nVers, v, err := decodeXLHeaders(v) + if err != nil { + return nil, err + } + var versions = struct { + Versions []json.RawMessage + Headers []json.RawMessage + }{ + Versions: make([]json.RawMessage, nVers), + Headers: make([]json.RawMessage, nVers), + } + err = decodeVersions(v, nVers, func(idx int, hdr, meta []byte) error { + var buf bytes.Buffer + if _, err := msgp.UnmarshalAsJSON(&buf, hdr); err != nil { + return err + } + versions.Headers[idx] = buf.Bytes() + buf = bytes.Buffer{} + if _, err := msgp.UnmarshalAsJSON(&buf, meta); err != nil { + return err + } + versions.Versions[idx] = buf.Bytes() + return nil + }) + if err != nil { + return nil, err + } + enc := json.NewEncoder(buf) + if err := enc.Encode(versions); err != nil { + return nil, err + } + data = b default: return nil, fmt.Errorf("unknown metadata version %d", minor) } @@ -416,3 +458,54 @@ func (x xlMetaInlineData) files(fn func(name string, data []byte)) error { return nil } + +const ( + xlHeaderVersion = 2 + xlMetaVersion = 1 +) + +func decodeXLHeaders(buf []byte) (versions int, b []byte, err error) { + hdrVer, buf, err := msgp.ReadUintBytes(buf) + if err != nil { + return 0, buf, err + } + metaVer, buf, err := msgp.ReadUintBytes(buf) + if err != nil { + return 0, buf, err + } + if hdrVer > xlHeaderVersion { + return 0, buf, fmt.Errorf("decodeXLHeaders: Unknown xl header version %d", metaVer) + } + if metaVer > xlMetaVersion { + return 0, buf, fmt.Errorf("decodeXLHeaders: Unknown xl meta version %d", metaVer) + } + versions, buf, err = msgp.ReadIntBytes(buf) + if err != nil { + return 0, buf, err + } + if versions < 0 { + return 0, buf, fmt.Errorf("decodeXLHeaders: Negative version count %d", versions) + } + return versions, buf, nil +} + +// decodeVersions will decode a number of versions from a buffer +// and perform a callback for each version in order, newest first. +// Any non-nil error is returned. +func decodeVersions(buf []byte, versions int, fn func(idx int, hdr, meta []byte) error) (err error) { + var tHdr, tMeta []byte // Zero copy bytes + for i := 0; i < versions; i++ { + tHdr, buf, err = msgp.ReadBytesZC(buf) + if err != nil { + return err + } + tMeta, buf, err = msgp.ReadBytesZC(buf) + if err != nil { + return err + } + if err = fn(i, tHdr, tMeta); err != nil { + return err + } + } + return nil +} diff --git a/go.mod b/go.mod index d00c79535..d7dc13b0c 100644 --- a/go.mod +++ b/go.mod @@ -81,6 +81,7 @@ require ( github.com/valyala/bytebufferpool v1.0.0 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c github.com/yargevad/filepathx v1.0.0 + github.com/zeebo/xxh3 v1.0.0 go.etcd.io/etcd/api/v3 v3.5.0 go.etcd.io/etcd/client/v3 v3.5.0 go.uber.org/atomic v1.9.0 diff --git a/go.sum b/go.sum index 0d7e987a4..ed747054f 100644 --- a/go.sum +++ b/go.sum @@ -1527,6 +1527,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/zeebo/xxh3 v1.0.0 h1:6eLPZCVXpsGnhv8RiWBEJs5kenm2W1CMwon19/l8ODc= +github.com/zeebo/xxh3 v1.0.0/go.mod h1:8VHV24/3AZLn3b6Mlp/KuC33LWH687Wq6EnziEB+rsA= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=