AWK is quicker than I thought, so:
for i in /storj/n*; do
echo $'\n\n'$i;
find $i -type f -print0 | xargs -0 -P 40 -n 100 stat -c "%s" | awk '{
n=log($1)/log(2);
if (n<12) { n=12; } else if (n > int(n)) { n = int(n) + 1; }
size[n]++;
r128 = int($1 / 131072) + (($1 % 131072) > 0);
r512 = int($1 / 524288) + (($1 % 524288) > 0);
records128[n] += r128;
records512[n] += r512;
loss128[n] += (n < 18? 2^n : r128 * 131072) - $1;
loss512[n] += (n < 20? 2^n : r512 * 524288) - $1;
totalsize += $1;
}
END {
t=0; t128=0; t512=0; l128=0; l512=0;
for (i in size){
printf("%d: %d files; 128k-records: %d (loss: %d), 512k-records: %d (loss: %d)\n", 2^i, size[i], records128[i], loss128[i], records512[i], loss512[i]);
t += size[i];
t128 += records128[i];
t512 += records512[i];
l128 += loss128[i];
l512 += loss512[i];
}
printf("Total amount of files: %d (totalsize %d), 128k-records: %d (loss: %d), 512k-records: %d (loss %d)\n", t, totalsize, t128, l128, t512, l512);
}' | sort -n;
done
Result:
/storj/nd10 Total amount of files: 14425945 (totalsize 2252536281582), 128k-records: 27616495 (loss: 357451568658), 512k-records: 16486900 (loss 592687535634)
4096: 2783759 files; 128k-records: 2783750 (loss: 3929095698), 512k-records: 2783750 (loss: 3929095698)
8192: 972035 files; 128k-records: 972035 (loss: 2072509015), 512k-records: 972035 (loss: 2072509015)
16384: 2114147 files; 128k-records: 2114147 (loss: 7625174970), 512k-records: 2114147 (loss: 7625174970)
32768: 1710650 files; 128k-records: 1710650 (loss: 20412948858), 512k-records: 1710650 (loss: 20412948858)
65536: 1925923 files; 128k-records: 1925923 (loss: 47420580630), 512k-records: 1925923 (loss: 47420580630)
131072: 560868 files; 128k-records: 560868 (loss: 23127283566), 512k-records: 560868 (loss: 23127283566)
262144: 3167139 files; 128k-records: 6334278 (loss: 179982391276), 512k-records: 3167139 (loss: 179982391276)
524288: 559257 files; 128k-records: 1752166 (loss: 44015135380), 512k-records: 559257 (loss: 107566967444)
1048576: 105338 files; 128k-records: 614007 (loss: 7292039212), 512k-records: 210676 (loss: 37267812396)
2097152: 97720 files; 128k-records: 1156117 (loss: 3826079040), 512k-records: 335095 (loss: 27977798976)
4194304: 429055 files; 128k-records: 7684684 (loss: 17744105993), 512k-records: 2145368 (loss: 135287902729)
8388608: 19 files; 128k-records: 719 (loss: 1750351), 512k-records: 192 (loss: 8172879)
16777216: 22 files; 128k-records: 1914 (loss: 1426085), 512k-records: 486 (loss: 5358245)
33554432: 11 files; 128k-records: 1847 (loss: 889608), 512k-records: 466 (loss: 3117832)
67108864: 1 files; 128k-records: 284 (loss: 118016), 512k-records: 71 (loss: 118016)
536870912: 1 files; 128k-records: 3106 (loss: 40960), 512k-records: 777 (loss: 303104)
/storj/nd11
Total amount of files: 2301357 (totalsize 423198041183), 128k-records: 4719088 (loss: 75008641953), 512k-records: 2581592 (loss 120390486945)
4096: 522815 files; 128k-records: 508937 (loss: 681816165), 512k-records: 508937 (loss: 681816165)
8192: 121198 files; 128k-records: 121198 (loss: 266938873), 512k-records: 121198 (loss: 266938873)
16384: 168942 files; 128k-records: 168942 (loss: 708726815), 512k-records: 168942 (loss: 708726815)
32768: 127609 files; 128k-records: 127609 (loss: 1323333488), 512k-records: 127609 (loss: 1323333488)
65536: 136639 files; 128k-records: 136639 (loss: 2631076813), 512k-records: 136639 (loss: 2631076813)
131072: 115151 files; 128k-records: 115151 (loss: 4945044794), 512k-records: 115151 (loss: 4945044794)
262144: 862808 files; 128k-records: 1725616 (loss: 47876053501), 512k-records: 862808 (loss: 47876053501)
524288: 148634 files; 128k-records: 461556 (loss: 12318094691), 512k-records: 148634 (loss: 29748049251)
1048576: 19696 files; 128k-records: 110366 (loss: 1391607981), 512k-records: 39392 (loss: 7578468525)
2097152: 20563 files; 128k-records: 213761 (loss: 485049953), 512k-records: 65168 (loss: 6633768545)
4194304: 57270 files; 128k-records: 1026507 (loss: 2378753074), 512k-records: 286398 (loss: 17987462194)
8388608: 17 files; 128k-records: 656 (loss: 1390647), 512k-records: 173 (loss: 6109239)
16777216: 14 files; 128k-records: 1170 (loss: 685526), 512k-records: 298 (loss: 3569110)
134217728: 1 files; 128k-records: 980 (loss: 69632), 512k-records: 245 (loss: 69632)
/storj/nd16
Total amount of files: 2682684 (totalsize 426638958102), 128k-records: 5173367 (loss: 82538615274), 512k-records: 3013476 (loss 139986817514)
4096: 457071 files; 128k-records: 457071 (loss: 701857559), 512k-records: 457071 (loss: 701857559)
8192: 159826 files; 128k-records: 159826 (loss: 344430429), 512k-records: 159826 (loss: 344430429)
16384: 369414 files; 128k-records: 369414 (loss: 1398463082), 512k-records: 369414 (loss: 1398463082)
32768: 285823 files; 128k-records: 285823 (loss: 2978327709), 512k-records: 285823 (loss: 2978327709)
65536: 316843 files; 128k-records: 316843 (loss: 6064901378), 512k-records: 316843 (loss: 6064901378)
131072: 148117 files; 128k-records: 148117 (loss: 6463225475), 512k-records: 148117 (loss: 6463225475)
262144: 615480 files; 128k-records: 1230960 (loss: 40556268085), 512k-records: 615480 (loss: 40556268085)
524288: 223578 files; 128k-records: 695439 (loss: 19384045069), 512k-records: 223578 (loss: 45450726925)
1048576: 19543 files; 128k-records: 107640 (loss: 1408859954), 512k-records: 39086 (loss: 7792590642)
2097152: 19906 files; 128k-records: 200461 (loss: 503447296), 512k-records: 62626 (loss: 7062683392)
4194304: 67073 files; 128k-records: 1200843 (loss: 2733856759), 512k-records: 335375 (loss: 21170051063)
8388608: 4 files; 128k-records: 176 (loss: 361343), 512k-records: 46 (loss: 1409919)
16777216: 3 files; 128k-records: 255 (loss: 337509), 512k-records: 65 (loss: 992869)
33554432: 3 files; 128k-records: 499 (loss: 233627), 512k-records: 126 (loss: 888987)
/storj/nd18
Total amount of files: 13603903 (totalsize 2516255704307), 128k-records: 27463891 (loss: 503069473549), 512k-records: 14852171 (loss 674692397837) 4096: 2985524 files; 128k-records: 2985083 (loss: 3169546327), 512k-records: 2985083 (loss: 3169546327)
8192: 473560 files; 128k-records: 473560 (loss: 1068433977), 512k-records: 473560 (loss: 1068433977)
16384: 586011 files; 128k-records: 586011 (loss: 2399812019), 512k-records: 586011 (loss: 2399812019)
32768: 468576 files; 128k-records: 468576 (loss: 4942098709), 512k-records: 468576 (loss: 4942098709)
65536: 456657 files; 128k-records: 456657 (loss: 8757065444), 512k-records: 456657 (loss: 8757065444)
131072: 342213 files; 128k-records: 342213 (loss: 13681375877), 512k-records: 342213 (loss: 13681375877)
262144: 7349557 files; 128k-records: 14699114 (loss: 407009279401), 512k-records: 7349557 (loss: 407009279401)
524288: 551086 files; 128k-records: 1747855 (loss: 44738768385), 512k-records: 551086 (loss: 104571694593)
1048576: 64797 files; 128k-records: 373154 (loss: 4568420348), 512k-records: 129594 (loss: 23602958332)
2097152: 71562 files; 128k-records: 763662 (loss: 1913893573), 512k-records: 229834 (loss: 22318396101)
4194304: 254164 files; 128k-records: 4532115 (loss: 10805865433), 512k-records: 1270941 (loss: 83111603161)
8388608: 47 files; 128k-records: 1880 (loss: 4556033), 512k-records: 496 (loss: 18187521)
16777216: 74 files; 128k-records: 6723 (loss: 5278351), 512k-records: 1710 (loss: 20613775)
33554432: 51 files; 128k-records: 9417 (loss: 3504737), 512k-records: 2374 (loss: 13859425)
67108864: 23 files; 128k-records: 7391 (loss: 1562647), 512k-records: 1859 (loss: 7460887)
2147483648: 1 files; 128k-records: 10480 (loss: 12288), 512k-records: 2620 (loss: 12288)
/storj/nd22
Total amount of files: 4449535 (totalsize 828766393703), 128k-records: 8884182 (loss: 188207860377), 512k-records: 4753191 (loss 234325674649)
4096: 741592 files; 128k-records: 741592 (loss: 705430855), 512k-records: 741592 (loss: 705430855)
8192: 110623 files; 128k-records: 110623 (loss: 248625448), 512k-records: 110623 (loss: 248625448)
16384: 156961 files; 128k-records: 156961 (loss: 638276221), 512k-records: 156961 (loss: 638276221)
32768: 128368 files; 128k-records: 128368 (loss: 1324308015), 512k-records: 128368 (loss: 1324308015)
65536: 139082 files; 128k-records: 139082 (loss: 2750804530), 512k-records: 139082 (loss: 2750804530)
131072: 95255 files; 128k-records: 95255 (loss: 3896618682), 512k-records: 95255 (loss: 3896618682)
262144: 2830544 files; 128k-records: 5661088 (loss: 162113744475), 512k-records: 2830544 (loss: 162113744475)
524288: 147524 files; 128k-records: 456895 (loss: 12283338960), 512k-records: 147524 (loss: 29742260432)
1048576: 18095 files; 128k-records: 102982 (loss: 1269801433), 512k-records: 36190 (loss: 6745727449)
2097152: 24119 files; 128k-records: 250547 (loss: 603198980), 512k-records: 76882 (loss: 8071812612)
4194304: 57317 files; 128k-records: 1026645 (loss: 2369783096), 512k-records: 286612 (loss: 18072601912)
8388608: 22 files; 128k-records: 875 (loss: 2089044), 512k-records: 229 (loss: 7462996)
16777216: 15 files; 128k-records: 1303 (loss: 789277), 512k-records: 331 (loss: 3541789)
33554432: 7 files; 128k-records: 1193 (loss: 402761), 512k-records: 301 (loss: 1844553)
67108864: 4 files; 128k-records: 1502 (loss: 202099), 512k-records: 377 (loss: 988531)
134217728: 6 files; 128k-records: 4296 (loss: 360485), 512k-records: 1076 (loss: 1409061)
1073741824: 1 files; 128k-records: 4975 (loss: 86016), 512k-records: 1244 (loss: 217088)
/storj/nd4
Total amount of files: 13423808 (totalsize 2551182964472), 128k-records: 28148855 (loss: 443987749128), 512k-records: 15355462 (loss 638052034824)
4096: 2345822 files; 128k-records: 2345529 (loss: 2667624342), 512k-records: 2345529 (loss: 2667624342)
8192: 556240 files; 128k-records: 556240 (loss: 1188658165), 512k-records: 556240 (loss: 1188658165)
16384: 1157067 files; 128k-records: 1157067 (loss: 4179085464), 512k-records: 1157067 (loss: 4179085464)
32768: 1042393 files; 128k-records: 1042393 (loss: 12278901008), 512k-records: 1042393 (loss: 12278901008)
65536: 1419181 files; 128k-records: 1419181 (loss: 36284051969), 512k-records: 1419181 (loss: 36284051969)
131072: 302774 files; 128k-records: 302774 (loss: 12155388074), 512k-records: 302774 (loss: 12155388074)
262144: 5661424 files; 128k-records: 11322848 (loss: 320374264550), 512k-records: 5661424 (loss: 320374264550)
524288: 377121 files; 128k-records: 1187233 (loss: 29481569148), 512k-records: 377121 (loss: 71588580220)
1048576: 72721 files; 128k-records: 420937 (loss: 5026784467), 512k-records: 145442 (loss: 26107225299)
2097152: 79326 files; 128k-records: 978349 (loss: 3340031454), 512k-records: 280462 (loss: 22148732382)
4194304: 409553 files; 128k-records: 7393390 (loss: 16994474754), 512k-records: 2062005 (loss: 129012538114)
8388608: 105 files; 128k-records: 3707 (loss: 11446139), 512k-records: 996 (loss: 47753083)
16777216: 39 files; 128k-records: 3529 (loss: 2656531), 512k-records: 895 (loss: 9341203)
33554432: 15 files; 128k-records: 2805 (loss: 651221), 512k-records: 708 (loss: 4190165)
67108864: 25 files; 128k-records: 9049 (loss: 1948426), 512k-records: 2268 (loss: 4963082)
134217728: 1 files; 128k-records: 531 (loss: 106920), 512k-records: 133 (loss: 237992)
536870912: 1 files; 128k-records: 3293 (loss: 106496), 512k-records: 824 (loss: 499712)
/storj/nd6
Total amount of files: 6397986 (totalsize 1055503909533), 128k-records: 11734840 (loss: 176588651875), 512k-records: 6755223 (loss 231772192099)
4096: 1933026 files; 128k-records: 1933023 (loss: 1177728601), 512k-records: 1933023 (loss: 1177728601)
8192: 130287 files; 128k-records: 130287 (loss: 290348591), 512k-records: 130287 (loss: 290348591)
16384: 175876 files; 128k-records: 175876 (loss: 716402612), 512k-records: 175876 (loss: 716402612)
32768: 141933 files; 128k-records: 141933 (loss: 1449199986), 512k-records: 141933 (loss: 1449199986)
65536: 159237 files; 128k-records: 159237 (loss: 3137508594), 512k-records: 159237 (loss: 3137508594)
131072: 122641 files; 128k-records: 122641 (loss: 5128370044), 512k-records: 122641 (loss: 5128370044)
262144: 3438022 files; 128k-records: 6876044 (loss: 144980958473), 512k-records: 3438022 (loss: 144980958473)
524288: 181269 files; 128k-records: 565256 (loss: 14671501556), 512k-records: 181269 (loss: 35619428596)
1048576: 20074 files; 128k-records: 114593 (loss: 1411245696), 512k-records: 40148 (loss: 7440426624)
2097152: 26510 files; 128k-records: 274461 (loss: 627981262), 512k-records: 83938 (loss: 8661515214)
4194304: 69037 files; 128k-records: 1227654 (loss: 2992004776), 512k-records: 345355 (loss: 23146421928)
8388608: 34 files; 128k-records: 1328 (loss: 2668571), 512k-records: 347 (loss: 10532891)
16777216: 17 files; 128k-records: 1412 (loss: 1256035), 512k-records: 362 (loss: 5974627)
33554432: 19 files; 128k-records: 3780 (loss: 1300824), 512k-records: 955 (loss: 6543704)
67108864: 3 files; 128k-records: 882 (loss: 159870), 512k-records: 221 (loss: 422014)
1073741824: 1 files; 128k-records: 6433 (loss: 16384), 512k-records: 1609 (loss: 409600)
/storj/nd9
Total amount of files: 6634934 (totalsize 1247864631628), 128k-records: 13639972 (loss: 144863522484), 512k-records: 7547816 (loss 253475548852)
4096: 2204846 files; 128k-records: 2204832 (loss: 1427453605), 512k-records: 2204832 (loss: 1427453605)
8192: 195260 files; 128k-records: 195260 (loss: 415096776), 512k-records: 195260 (loss: 415096776)
16384: 342005 files; 128k-records: 342005 (loss: 1288130734), 512k-records: 342005 (loss: 1288130734)
32768: 311579 files; 128k-records: 311579 (loss: 3315884317), 512k-records: 311579 (loss: 3315884317)
65536: 324748 files; 128k-records: 324748 (loss: 6434006313), 512k-records: 324748 (loss: 6434006313)
131072: 201166 files; 128k-records: 201166 (loss: 8274273639), 512k-records: 201166 (loss: 8274273639)
262144: 2491939 files; 128k-records: 4983878 (loss: 86781895178), 512k-records: 2491939 (loss: 86781895178)
524288: 292868 files; 128k-records: 918597 (loss: 24819556388), 512k-records: 292868 (loss: 57964388388)
1048576: 37389 files; 128k-records: 215771 (loss: 2596740300), 512k-records: 74778 (loss: 13520411852)
2097152: 34199 files; 128k-records: 378003 (loss: 1243630053), 512k-records: 112479 (loss: 10669410789)
4194304: 198741 files; 128k-records: 3555616 (loss: 8243901974), 512k-records: 993890 (loss: 63286801942)
8388608: 185 files; 128k-records: 6110 (loss: 22384289), 512k-records: 1666 (loss: 94998177)
16777216: 5 files; 128k-records: 498 (loss: 273280), 512k-records: 127 (loss: 1584000)
67108864: 3 files; 128k-records: 1142 (loss: 246486), 512k-records: 287 (loss: 1032918)
134217728: 1 files; 128k-records: 767 (loss: 49152), 512k-records: 192 (loss: 180224)
So, just a little bit below two times as much space used for meta data with 128k recordsize. But about 25-50% more efficiency loss. However, losses seem to be incredible high… I don’t see why.