[Tech Preview] Hashstore backend for storage nodes

Product Discussions STORJLINGS
346

since my storagenode updated to 123.4 i get a ~300 MB/s disk I/O from the node - so for more than 2 days.

i don’t really know whats going on, because the migration was already finished weeks ago.
this is what i see in the logs:

2025-03-16T20:19:43+01:00       INFO    hashstore       compact once started    {"Process": "storagenode", "satellite": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs", "store": "s1", "today": 20163}
2025-03-16T20:19:45+01:00       INFO    hashstore       compaction computed details     {"Process": "storagenode", "satellite": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs", "store": "s1", "nset": 3272011, "nexist": 3272011, "modifications": false, "curr logSlots": 23, "next logSlots": 23, "candidates": [363, 802, 642, 635, 1103, 1034, 1443, 487, 1013, 109, 823, 1117, 401, 1363, 205, 315, 182, 1374, 1273, 81, 1557, 171, 1204, 521, 245, 758, 614, 231, 662, 1138, 125, 199, 151, 599, 950, 1344, 1182, 1291, 795, 947, 1174, 602, 1069, 1282, 1199, 519, 478, 134, 1314, 1011, 461, 982, 890, 906, 1208, 546, 1037, 399, 100, 789, 1540, 189, 296, 913, 501, 776, 1249, 897, 132, 723, 793, 438, 603, 291, 1088, 422, 974, 1378, 1090, 176, 444, 1239, 497, 1296, 22, 1020, 939, 842, 811, 1244, 536, 457, 1285, 1395, 463, 1120, 408, 64, 267, 1340, 1134, 1354, 518, 173, 722, 179, 1162, 1173, 513, 1335, 1350, 1262, 1195, 203, 992, 801, 385, 940, 702, 1023, 568, 370, 671, 153, 917, 1004, 682, 883, 1445, 1328, 650, 117, 1137, 922, 221, 284, 1306, 364, 680, 414, 1332, 828, 258, 1339, 275, 782, 710, 1027, 1353, 532, 226, 654, 142, 1170, 703, 1205, 427, 904, 272, 146, 686, 999, 695, 1067, 105, 888, 1096, 279, 963, 1365, 168, 119, 312, 916, 694, 861, 1434, 247, 1379, 1565, 1411, 159, 92, 1046, 700], "rewrite": [1291], "duration": "1.890047204s"}
2025-03-16T20:21:15+01:00       INFO    hashstore       hashtbl rewritten       {"Process": "storagenode", "satellite": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs", "store": "s1", "total records": 3272011, "total bytes": "1.0 TiB", "rewritten records": 1873, "rewritten bytes": "609.9 MiB", "trashed records": 0, "trashed bytes": "0 B", "restored records": 0, "restored bytes": "0 B", "expired records": 0, "expired bytes": "0 B"}
2025-03-16T20:21:15+01:00       INFO    hashstore       compact once finished   {"Process": "storagenode", "satellite": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs", "store": "s1", "duration": "1m32.009410138s", "completed": false}
2025-03-16T20:21:15+01:00       INFO    hashstore       compact once started    {"Process": "storagenode", "satellite": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs", "store": "s1", "today": 20163}
2025-03-16T20:21:16+01:00       INFO    hashstore       compaction computed details     {"Process": "storagenode", "satellite": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs", "store": "s1", "nset": 3272011, "nexist": 3272011, "modifications": false, "curr logSlots": 23, "next logSlots": 23, "candidates": [790, 121, 831, 1253, 654, 361, 196, 291, 1431, 871, 105, 217, 737, 382, 998, 1302, 540, 1075, 780, 1304, 463, 128, 1051, 1187, 155, 1067, 1113, 487, 802, 1572, 1004, 566, 457, 477, 1230, 788, 793, 728, 1248, 739, 1282, 263, 92, 1280, 1195, 301, 1026, 606, 987, 334, 1132, 974, 1508, 784, 1094, 349, 340, 1180, 897, 176, 321, 1241, 940, 901, 841, 645, 667, 1413, 1017, 1166, 874, 506, 271, 115, 278, 524, 1092, 682, 1239, 150, 1254, 826, 1329, 1395, 289, 902, 132, 1199, 995, 1045, 775, 878, 1342, 1011, 149, 1407, 958, 577, 319, 973, 230, 180, 701, 1041, 1360, 1328, 167, 299, 274, 872, 205, 845, 1020, 855, 652, 389, 865, 554, 622, 896, 883, 1306, 809, 367, 858, 1162, 1256, 228, 1032, 774, 1419, 541, 63, 100, 336, 1408, 407, 411, 1033, 992, 251, 427, 888, 1125, 560, 1371, 1043, 917, 1086, 398, 213, 330, 294, 1295, 803, 1236, 754, 262, 808, 286, 837, 227, 1154, 147, 564, 512, 1170, 703, 1177, 108, 753, 520, 1104], "rewrite": [831], "duration": "1.273032159s"}
2025-03-16T20:22:43+01:00       INFO    hashstore       hashtbl rewritten       {"Process": "storagenode", "satellite": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs", "store": "s1", "total records": 3272011, "total bytes": "1.0 TiB", "rewritten records": 1758, "rewritten bytes": "630.4 MiB", "trashed records": 0, "trashed bytes": "0 B", "restored records": 0, "restored bytes": "0 B", "expired records": 0, "expired bytes": "0 B"}
2025-03-16T20:22:43+01:00       INFO    hashstore       compact once finished   {"Process": "storagenode", "satellite": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs", "store": "s1", "duration": "1m28.096823111s", "completed": false}
2025-03-16T20:22:43+01:00       INFO    hashstore       compact once started    {"Process": "storagenode", "satellite": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs", "store": "s1", "today": 20163}
2025-03-16T20:22:44+01:00       INFO    hashstore       compaction computed details     {"Process": "storagenode", "satellite": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs", "store": "s1", "nset": 3272011, "nexist": 3272011, "modifications": false, "curr logSlots": 23, "next logSlots": 23, "candidates": [1139, 826, 906, 313, 1120, 1169, 156, 1053, 1061, 1408, 1147, 1356, 100, 999, 492, 1300, 703, 233, 265, 1174, 716, 225, 153, 866, 1253, 1560, 697, 1156, 1162, 931, 517, 908, 790, 426, 450, 945, 1428, 823, 185, 1043, 1421, 309, 1378, 1565, 433, 299, 694, 1239, 630, 1392, 958, 1104, 404, 186, 112, 1445, 1000, 1307, 868, 1155, 609, 1099, 252, 732, 736, 596, 776, 1386, 1128, 412, 988, 231, 534, 786, 259, 333, 1251, 634, 987, 482, 1044, 629, 680, 1524, 437, 1294, 701, 828, 1096, 342, 353, 796, 842, 234, 591, 1285, 490, 288, 679, 449, 880, 1411, 943, 134, 385, 203, 1277, 444, 830, 774, 802, 602, 599, 1075, 1319, 245, 748, 384, 1117, 1325, 615, 1192, 1394, 1050, 255, 1133, 1179, 282, 286, 321, 107, 477, 410, 803, 401, 1187, 122, 251, 148, 1240, 1129, 914, 531, 979, 1217, 1092, 1098, 1134, 1365, 571, 1308, 1329, 1350, 849, 1029, 819, 217, 1375, 281, 60, 1434, 133, 407, 326, 740, 1206, 1027, 753, 1045, 254, 179, 166, 486, 442, 420, 1021, 554, 1383, 905, 1124, 598, 758, 1297, 1296, 411, 1052, 837, 25, 147, 370, 977, 883, 28, 244, 1252, 361, 724, 1082, 1222, 119, 1103, 439, 1571, 1545, 346, 1041, 81, 298, 258, 213, 1337, 1004, 171, 127], "rewrite": [1239], "duration": "1.61766216s"}
2025-03-16T20:29:20+01:00       INFO    piecemigrate:chore      enqueued for migration  {"Process": "storagenode", "sat": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs"}
2025-03-16T20:29:20+01:00       INFO    piecemigrate:chore      enqueued for migration  {"Process": "storagenode", "sat": "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE"}
2025-03-16T20:29:20+01:00       INFO    piecemigrate:chore      enqueued for migration  {"Process": "storagenode", "sat": "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6"}
2025-03-16T20:29:23+01:00       INFO    piecemigrate:chore      couldn't migrate        {"Process": "storagenode", "error": "opening the old reader: pieces error: invalid piece file for storage format version 1: too small for header (0 < 512)", "errorVerbose": "opening the old reader: pieces error: invalid piece file for storage format version 1: too small for header (0 < 512)\n\tstorj.io/storj/storagenode/piecemigrate.(*Chore).migrateOne:318\n\tstorj.io/storj/storagenode/piecemigrate.(*Chore).processQueue:260\n\tstorj.io/storj/storagenode/piecemigrate.(*Chore).Run.func2:167\n\tstorj.io/common/errs2.(*Group).Go.func1:23", "sat": "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S", "id": "WEOQ6T5BYKKB4AZDK23NQNTUURDOP7GHY64NLDVO37C2A4PL7RMQ"}
2025-03-16T20:29:24+01:00       INFO    piecemigrate:chore      couldn't migrate        {"Process": "storagenode", "error": "opening the old reader: pieces error: invalid piece file for storage format version 1: too small for header (0 < 512)", "errorVerbose": "opening the old reader: pieces error: invalid piece file for storage format version 1: too small for header (0 < 512)\n\tstorj.io/storj/storagenode/piecemigrate.(*Chore).migrateOne:318\n\tstorj.io/storj/storagenode/piecemigrate.(*Chore).processQueue:260\n\tstorj.io/storj/storagenode/piecemigrate.(*Chore).Run.func2:167\n\tstorj.io/common/errs2.(*Group).Go.func1:23", "sat": "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S", "id": "7FY52IOAKIO4WGT56OPC33TQ5JFLKJSU3IGZYUQPUM2EOONJKPVQ"}
2025-03-16T20:29:24+01:00       INFO    piecemigrate:chore      enqueued for migration  {"Process": "storagenode", "sat": "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S"}
2025-03-16T20:29:24+01:00       INFO    piecemigrate:chore      all enqueued for migration; will sleep before next pooling      {"Process": "storagenode", "active": {"121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6": true, "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S": true, "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs": true, "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE": true}, "interval": "10m0s"}
2025-03-16T20:30:31+01:00       INFO    hashstore       hashtbl rewritten       {"Process": "storagenode", "satellite": "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S", "store": "s1", "total records": 18261660, "total bytes": "3.5 TiB", "rewritten records": 20401, "rewritten bytes": "3.9 GiB", "trashed records": 0, "trashed bytes": "0 B", "restored records": 0, "restored bytes": "0 B", "expired records": 0, "expired bytes": "0 B"}
2025-03-16T20:30:31+01:00       INFO    hashstore       compact once finished   {"Process": "storagenode", "satellite": "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S", "store": "s1", "duration": "15m56.849605772s", "completed": false}
2025-03-16T20:30:31+01:00       INFO    hashstore       compact once started    {"Process": "storagenode", "satellite": "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S", "store": "s1", "today": 20162}
347

Read above. We experience the same thing. The next version will fix the problem.

348

The version you are running is fixing the problem. The root cause is that the previous version didn’t spend enough time on compacting so the version you are running has to catch up first.

3 Likes
349

Selection of candidates for compaction looks a bit strange for me in this version. Previous versions generated the same list of candidates on each step excluding one specific ID that was compacted on previous step and continued compacting until this list become empty.
Now this list looks pretty random/chaotic. In molnart’s logs we see for instance candidate ID 363 on the first step (the first one). Why it disappeared on the second step? It was not compacted. ID 1291 was. Why list on step 2 is almost completely different? Not so much changed on the node in these 1.5 minutes. Are you sure candidate selection works correctly?

350
1 Like
351

sorry was just catching up and saw this… is there a need to change the storj start command parameters?.. or i can just wait for this compact activity to kick in on its own eventually?

352

You do not need to change anything. The compaction should catch up automatically.

353

It looks like the next big step will be comming soon. On the main branch we have now the memtable. Again do not use it. There is still ongoing work.

The problem with memtable and hashtable on SSD is the question how to rebuild the hashtable. We did some benchmark tests and it would take days for a full drive. For the memtable it would have to run that on every reboot. So we need a fast rebuild otherwise we can’t use memtable.

Solution: Write a small hint file side by side with the log file. That way rebuilding the hashtable will be a lot cheaper. So thats the next steps that will get implemented.

In the meantime I have to repeat it one more time. Please do not move your hashtable to SSD or memtable. Let the developer trigger all the landmines first.

8 Likes
354

Meanwhile, nodes that will skip this post.

Thank you for the update and warning!

6 Likes
355

1.So, just to clarify for one noob out there (me)…
-memtable is something that is kept only in RAM?
-hashtable is the equivalent, but kept only on drive?
2.So how big should be the RAM, to keep the memtable in, per TB of stored data?
3.If it dosen’t fit, what happens?
4.Could the SNO choose which one he preffers?
5.Will the repair be implemented for both cases?

1 Like
356

Why memtable should be started from scratch every reboot? Can it write snapshot to disk for example every 5min (and on exit of course) and make checkpoint to logs? In case of unexpected shutdown, node will read logs from some points and restore memtable.

That trick even can be used with hashtable and we won’t kill our SSDs. In that case we also don’t need memtable at all.

357

How is that different from what I have described? A hint file side by side of each log file.

358

how long does it take to finish with this compaction insanity? running for a week already, hogging up my drive pool. is there a way to at least monitor the progress?

359

I wrote above how to do it and how long it takes. Read it!
In short, you let the startup filewalker to finish and give you the no. of pieces on your node. Than… the migration speed is like 1000 pieces/min.
You need info log level to get all these.

360

i am not talking about the migration but the compaction. i have finished migrating my node over a month ago and everything was shiny and dandy until this compaction came that i have no idea why was even necessery and what it it actually doing, except hammering my drive with constant I/O.

361

I talked about that too. It will settle down eventualy in a few days.

362

I think it compacting all the time because of TTL data is is EOL and node cleaning it.

1 Like
363

I have an ideea to avoid this compaction, but maybe my suppositions are wrong.
If the pieces are made of specific sizes, like for ex 128KB, 256KB, 1024KB and so on, the hashstore could put the same size pieces in the same log file, and when a piece is deleted, another new one of the same size will take it’s place. This way, you don’t need to run a compaction anymore, to recover the space after piece deletion. You just fill the wholes with new pieces.

2 Likes
364

so the compaction is not a one off thing that happens once, but this will occur regularly? in that case the benefits of hashstore have diminished very quickly. It maybe replaced random IO with sequential access but in the end the speed impact looks very similar

365

Your situation is strange. I looked logs of one of my nodes (2.1TB data + 250GB trash) and found that the longest compaction run was 3h 40min - probably there was a lot of trash to delete, but in most cases it’s much faster.

Do you have only one node per drive or nodes are fighting for drive I/O?

finished compactions 
2025-03-09T18:42:42Z    INFO    hashstore       finished compaction     {"Process": "storagenode", "satellite": "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE", "store": "s1", "duration": "34.45803ms", "stats": {"NumLogs":2,"LenLogs":"1.2 GiB","NumLogsTTL":0,"LenLogsTTL":"0 B","SetPercent":0.9914324257472669,"TrashPercent":0,"Compacting":false,"Compactions":0,"TableFull":0,"Today":20156,"LastCompact":20156,"LogsRewritten":0,"DataRewritten":"0 B","Table":{"NumSet":1703,"LenSet":"1.2 GiB","AvgSet":748787.0346447446,"NumTrash":0,"LenTrash":"0 B","AvgTrash":0,"NumSlots":16384,"TableSize":"1.0 MiB","Load":0.10394287109375,"Created":20142},"Compaction":{"Elapsed":0,"Remaining":0,"TotalRecords":0,"ProcessedRecords":0}}}
2025-03-10T02:29:10Z    INFO    hashstore       finished compaction     {"Process": "storagenode", "satellite": "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE", "store": "s0", "duration": "12.843098ms", "stats": {"NumLogs":1,"LenLogs":"42.9 MiB","NumLogsTTL":0,"LenLogsTTL":"0 B","SetPercent":1,"TrashPercent":0,"Compacting":false,"Compactions":0,"TableFull":0,"Today":20157,"LastCompact":20157,"LogsRewritten":0,"DataRewritten":"0 B","Table":{"NumSet":76,"LenSet":"42.9 MiB","AvgSet":591626.1052631579,"NumTrash":0,"LenTrash":"0 B","AvgTrash":0,"NumSlots":16384,"TableSize":"1.0 MiB","Load":0.004638671875,"Created":20150},"Compaction":{"Elapsed":0,"Remaining":0,"TotalRecords":0,"ProcessedRecords":0}}}
2025-03-11T07:03:24Z    INFO    hashstore       finished compaction     {"Process": "storagenode", "satellite": "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6", "store": "s1", "duration": "26.704414894s", "stats": {"NumLogs":59,"LenLogs":"49.0 GiB","NumLogsTTL":7,"LenLogsTTL":"0.9 GiB","SetPercent":0.9882455716360055,"TrashPercent":0.14436893532054226,"Compacting":false,"Compactions":0,"TableFull":0,"Today":20158,"LastCompact":20158,"LogsRewritten":5,"DataRewritten":"1.3 GiB","Table":{"NumSet":245861,"LenSet":"48.4 GiB","AvgSet":211543.86727459825,"NumTrash":42656,"LenTrash":"7.1 GiB","AvgTrash":178122.52063015755,"NumSlots":524288,"TableSize":"32.0 MiB","Load":0.46894264221191406,"Created":20158},"Compaction":{"Elapsed":0,"Remaining":0,"TotalRecords":0,"ProcessedRecords":0}}}
2025-03-11T13:00:54Z    INFO    hashstore       finished compaction     {"Process": "storagenode", "satellite": "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S", "store": "s1", "duration": "7m7.224892218s", "stats": {"NumLogs":1147,"LenLogs":"1.1 TiB","NumLogsTTL":53,"LenLogsTTL":"25.7 GiB","SetPercent":0.9519808002664143,"TrashPercent":0.10138588081064104,"Compacting":false,"Compactions":0,"TableFull":0,"Today":20158,"LastCompact":20158,"LogsRewritten":34,"DataRewritten":"10.2 GiB","Table":{"NumSet":5995052,"LenSet":"1.0 TiB","AvgSet":191315.97368830163,"NumTrash":780946,"LenTrash":"113.8 GiB","AvgTrash":156412.86501243364,"NumSlots":16777216,"TableSize":"1.0 GiB","Load":0.3573329448699951,"Created":20158},"Compaction":{"Elapsed":0,"Remaining":0,"TotalRecords":0,"ProcessedRecords":0}}}
2025-03-11T13:01:07Z    INFO    hashstore       finished compaction     {"Process": "storagenode", "satellite": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs", "store": "s1", "duration": "3.116719821s", "stats": {"NumLogs":798,"LenLogs":"0.8 TiB","NumLogsTTL":17,"LenLogsTTL":"6.4 GiB","SetPercent":0.9732349727439179,"TrashPercent":0.5199874506383187,"Compacting":false,"Compactions":0,"TableFull":0,"Today":20158,"LastCompact":20158,"LogsRewritten":3,"DataRewritten":"0 B","Table":{"NumSet":1852730,"LenSet":"0.7 TiB","AvgSet":440298.01319350366,"NumTrash":408587,"LenTrash":"405.9 GiB","AvgTrash":1066717.6248289838,"NumSlots":4194304,"TableSize":"256.0 MiB","Load":0.4417252540588379,"Created":20158},"Compaction":{"Elapsed":0,"Remaining":0,"TotalRecords":0,"ProcessedRecords":0}}}
2025-03-11T23:50:13Z    INFO    hashstore       finished compaction     {"Process": "storagenode", "satellite": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs", "store": "s0", "duration": "19.81522421s", "stats": {"NumLogs":285,"LenLogs":"273.1 GiB","NumLogsTTL":8,"LenLogsTTL":"1.1 GiB","SetPercent":0.973895917581251,"TrashPercent":0.5558159570495057,"Compacting":false,"Compactions":0,"TableFull":0,"Today":20158,"LastCompact":20158,"LogsRewritten":1,"DataRewritten":"0 B","Table":{"NumSet":598511,"LenSet":"266.0 GiB","AvgSet":477193.19348182407,"NumTrash":145540,"LenTrash":"151.8 GiB","AvgTrash":1119959.8973203243,"NumSlots":2097152,"TableSize":"128.0 MiB","Load":0.28539228439331055,"Created":20158},"Compaction":{"Elapsed":0,"Remaining":0,"TotalRecords":0,"ProcessedRecords":0}}}
2025-03-12T00:34:15Z    INFO    hashstore       finished compaction     {"Process": "storagenode", "satellite": "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S", "store": "s0", "duration": "1m29.159075959s", "stats": {"NumLogs":469,"LenLogs":"467.9 GiB","NumLogsTTL":28,"LenLogsTTL":"7.4 GiB","SetPercent":0.9381756194556506,"TrashPercent":0.23643373253551242,"Compacting":false,"Compactions":0,"TableFull":0,"Today":20159,"LastCompact":20159,"LogsRewritten":15,"DataRewritten":"1.9 GiB","Table":{"NumSet":2353111,"LenSet":"439.0 GiB","AvgSet":200322.42951140003,"NumTrash":520394,"LenTrash":"110.6 GiB","AvgTrash":228278.498768241,"NumSlots":8388608,"TableSize":"512.0 MiB","Load":0.2805126905441284,"Created":20159},"Compaction":{"Elapsed":0,"Remaining":0,"TotalRecords":0,"ProcessedRecords":0}}}
2025-03-12T02:29:10Z    INFO    hashstore       finished compaction     {"Process": "storagenode", "satellite": "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE", "store": "s1", "duration": "10.538434ms", "stats": {"NumLogs":2,"LenLogs":"1.2 GiB","NumLogsTTL":0,"LenLogsTTL":"0 B","SetPercent":0.9914782606113428,"TrashPercent":0,"Compacting":false,"Compactions":1,"TableFull":0,"Today":20159,"LastCompact":20159,"LogsRewritten":0,"DataRewritten":"0 B","Table":{"NumSet":1712,"LenSet":"1.2 GiB","AvgSet":748891.5140186916,"NumTrash":0,"LenTrash":"0 B","AvgTrash":0,"NumSlots":16384,"TableSize":"1.0 MiB","Load":0.1044921875,"Created":20142},"Compaction":{"Elapsed":0,"Remaining":0,"TotalRecords":0,"ProcessedRecords":0}}}
2025-03-12T06:06:18Z    INFO    hashstore       finished compaction     {"Process": "storagenode", "satellite": "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6", "store": "s0", "duration": "1m18.777282507s", "stats": {"NumLogs":65,"LenLogs":"56.8 GiB","NumLogsTTL":7,"LenLogsTTL":"0.9 GiB","SetPercent":0.9554339762225073,"TrashPercent":0.31848269665278545,"Compacting":false,"Compactions":0,"TableFull":0,"Today":20159,"LastCompact":20159,"LogsRewritten":8,"DataRewritten":"2.5 GiB","Table":{"NumSet":289207,"LenSet":"54.2 GiB","AvgSet":201339.5707849395,"NumTrash":160933,"LenTrash":"18.1 GiB","AvgTrash":120608.52116098003,"NumSlots":1048576,"TableSize":"64.0 MiB","Load":0.27580928802490234,"Created":20159},"Compaction":{"Elapsed":0,"Remaining":0,"TotalRecords":0,"ProcessedRecords":0}}}
2025-03-12T19:28:18Z    INFO    hashstore       finished compaction     {"Process": "storagenode", "satellite": "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE", "store": "s0", "duration": "13.827601ms", "stats": {"NumLogs":1,"LenLogs":"44.0 MiB","NumLogsTTL":0,"LenLogsTTL":"0 B","SetPercent":1,"TrashPercent":0,"Compacting":false,"Compactions":1,"TableFull":0,"Today":20159,"LastCompact":20159,"LogsRewritten":0,"DataRewritten":"0 B","Table":{"NumSet":77,"LenSet":"44.0 MiB","AvgSet":599818.8051948051,"NumTrash":0,"LenTrash":"0 B","AvgTrash":0,"NumSlots":16384,"TableSize":"1.0 MiB","Load":0.00469970703125,"Created":20150},"Compaction":{"Elapsed":0,"Remaining":0,"TotalRecords":0,"ProcessedRecords":0}}}
2025-03-13T06:17:43Z    INFO    hashstore       finished compaction     {"Process": "storagenode", "satellite": "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S", "store": "s1", "duration": "2m5.280575548s", "stats": {"NumLogs":1179,"LenLogs":"1.1 TiB","NumLogsTTL":55,"LenLogsTTL":"26.9 GiB","SetPercent":0.9516687328421163,"TrashPercent":0.10804021905825723,"Compacting":false,"Compactions":1,"TableFull":0,"Today":20160,"LastCompact":20160,"LogsRewritten":46,"DataRewritten":"12.9 GiB","Table":{"NumSet":6061998,"LenSet":"1.1 TiB","AvgSet":194104.03353580783,"NumTrash":883168,"LenTrash":"124.4 GiB","AvgTrash":151253.93412804813,"NumSlots":16777216,"TableSize":"1.0 GiB","Load":0.3613232374191284,"Created":20160},"Compaction":{"Elapsed":0,"Remaining":0,"TotalRecords":0,"ProcessedRecords":0}}}
2025-03-13T15:59:58Z    INFO    hashstore       finished compaction     {"Process": "storagenode", "satellite": "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6", "store": "s1", "duration": "545.589781ms", "stats": {"NumLogs":66,"LenLogs":"55.0 GiB","NumLogsTTL":8,"LenLogsTTL":"0.9 GiB","SetPercent":0.98952051749481,"TrashPercent":0.16605100170381593,"Compacting":false,"Compactions":1,"TableFull":0,"Today":20160,"LastCompact":20160,"LogsRewritten":7,"DataRewritten":"1.3 GiB","Table":{"NumSet":254140,"LenSet":"54.4 GiB","AvgSet":229846.90542220822,"NumTrash":46632,"LenTrash":"9.1 GiB","AvgTrash":210205.62840967576,"NumSlots":524288,"TableSize":"32.0 MiB","Load":0.48473358154296875,"Created":20160},"Compaction":{"Elapsed":0,"Remaining":0,"TotalRecords":0,"ProcessedRecords":0}}}
2025-03-14T02:13:22Z    INFO    hashstore       finished compaction     {"Process": "storagenode", "satellite": "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6", "store": "s0", "duration": "1.293732874s", "stats": {"NumLogs":68,"LenLogs":"58.6 GiB","NumLogsTTL":6,"LenLogsTTL":"0.9 GiB","SetPercent":0.9568445196278266,"TrashPercent":0.3437337595374011,"Compacting":false,"Compactions":1,"TableFull":0,"Today":20161,"LastCompact":20161,"LogsRewritten":10,"DataRewritten":"2.5 GiB","Table":{"NumSet":291762,"LenSet":"56.1 GiB","AvgSet":206403.87033266842,"NumTrash":165229,"LenTrash":"20.1 GiB","AvgTrash":130930.58366267422,"NumSlots":1048576,"TableSize":"64.0 MiB","Load":0.2782459259033203,"Created":20161},"Compaction":{"Elapsed":0,"Remaining":0,"TotalRecords":0,"ProcessedRecords":0}}}
2025-03-14T13:57:15Z    INFO    hashstore       finished compaction     {"Process": "storagenode", "satellite": "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE", "store": "s1", "duration": "18.56137ms", "stats": {"NumLogs":2,"LenLogs":"1.2 GiB","NumLogsTTL":0,"LenLogsTTL":"0 B","SetPercent":0.9915258403989473,"TrashPercent":0,"Compacting":false,"Compactions":2,"TableFull":0,"Today":20161,"LastCompact":20161,"LogsRewritten":0,"DataRewritten":"0 B","Table":{"NumSet":1721,"LenSet":"1.2 GiB","AvgSet":749193.929110982,"NumTrash":0,"LenTrash":"0 B","AvgTrash":0,"NumSlots":16384,"TableSize":"1.0 MiB","Load":0.10504150390625,"Created":20142},"Compaction":{"Elapsed":0,"Remaining":0,"TotalRecords":0,"ProcessedRecords":0}}}
2025-03-14T16:47:11Z    INFO    hashstore       finished compaction     {"Process": "storagenode", "satellite": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs", "store": "s1", "duration": "2h33m27.210240208s", "stats": {"NumLogs":506,"LenLogs":"495.6 GiB","NumLogsTTL":116,"LenLogsTTL":"105.8 GiB","SetPercent":0.9922527978598807,"TrashPercent":0.2648988848090465,"Compacting":false,"Compactions":1,"TableFull":0,"Today":20161,"LastCompact":20161,"LogsRewritten":688,"DataRewritten":"380.8 GiB","Table":{"NumSet":1647010,"LenSet":"491.8 GiB","AvgSet":320597.84007140214,"NumTrash":165959,"LenTrash":"131.3 GiB","AvgTrash":849403.0511150344,"NumSlots":4194304,"TableSize":"256.0 MiB","Load":0.39267778396606445,"Created":20161},"Compaction":{"Elapsed":0,"Remaining":0,"TotalRecords":0,"ProcessedRecords":0}}}
2025-03-14T19:10:53Z    INFO    hashstore       finished compaction     {"Process": "storagenode", "satellite": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs", "store": "s0", "duration": "49m34.339194128s", "stats": {"NumLogs":183,"LenLogs":"176.8 GiB","NumLogsTTL":52,"LenLogsTTL":"46.4 GiB","SetPercent":0.9791803125976264,"TrashPercent":0.3369358072375038,"Compacting":false,"Compactions":0,"TableFull":0,"Today":20161,"LastCompact":20161,"LogsRewritten":245,"DataRewritten":"143.7 GiB","Table":{"NumSet":549312,"LenSet":"173.1 GiB","AvgSet":338348.72934871254,"NumTrash":97520,"LenTrash":"59.6 GiB","AvgTrash":655804.5689909762,"NumSlots":2097152,"TableSize":"128.0 MiB","Load":0.261932373046875,"Created":20161},"Compaction":{"Elapsed":0,"Remaining":0,"TotalRecords":0,"ProcessedRecords":0}}}
2025-03-14T21:30:55Z    INFO    hashstore       finished compaction     {"Process": "storagenode", "satellite": "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S", "store": "s0", "duration": "44m56.785164158s", "stats": {"NumLogs":499,"LenLogs":"493.8 GiB","NumLogsTTL":56,"LenLogsTTL":"38.6 GiB","SetPercent":0.9583635836501821,"TrashPercent":0.22354178231747737,"Compacting":false,"Compactions":0,"TableFull":0,"Today":20161,"LastCompact":20161,"LogsRewritten":139,"DataRewritten":"127.1 GiB","Table":{"NumSet":2428674,"LenSet":"473.3 GiB","AvgSet":209239.90462943976,"NumTrash":508480,"LenTrash":"110.4 GiB","AvgTrash":233113.9524229075,"NumSlots":8388608,"TableSize":"512.0 MiB","Load":0.2895205020904541,"Created":20161},"Compaction":{"Elapsed":0,"Remaining":0,"TotalRecords":0,"ProcessedRecords":0}}}
2025-03-15T14:58:28Z    INFO    hashstore       finished compaction     {"Process": "storagenode", "satellite": "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S", "store": "s1", "duration": "3h40m55.218503227s", "stats": {"NumLogs":1140,"LenLogs":"1.1 TiB","NumLogsTTL":90,"LenLogsTTL":"67.8 GiB","SetPercent":0.9840535962769547,"TrashPercent":0.14817572341339932,"Compacting":false,"Compactions":0,"TableFull":0,"Today":20162,"LastCompact":20162,"LogsRewritten":550,"DataRewritten":"499.6 GiB","Table":{"NumSet":6041222,"LenSet":"1.1 TiB","AvgSet":196005.71398667357,"NumTrash":982384,"LenTrash":"166.1 GiB","AvgTrash":181497.45943745013,"NumSlots":16777216,"TableSize":"1.0 GiB","Load":0.3600848913192749,"Created":20162},"Compaction":{"Elapsed":0,"Remaining":0,"TotalRecords":0,"ProcessedRecords":0}}}
2025-03-16T04:13:33Z    INFO    hashstore       finished compaction     {"Process": "storagenode", "satellite": "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6", "store": "s1", "duration": "25.230643238s", "stats": {"NumLogs":70,"LenLogs":"58.4 GiB","NumLogsTTL":5,"LenLogsTTL":"219.0 MiB","SetPercent":0.9405279126291443,"TrashPercent":0.12669210093607414,"Compacting":false,"Compactions":0,"TableFull":0,"Today":20163,"LastCompact":20163,"LogsRewritten":7,"DataRewritten":"1.4 GiB","Table":{"NumSet":237950,"LenSet":"55.0 GiB","AvgSet":248045.46676192476,"NumTrash":26828,"LenTrash":"7.4 GiB","AvgTrash":296351.10779782315,"NumSlots":524288,"TableSize":"32.0 MiB","Load":0.4538536071777344,"Created":20163},"Compaction":{"Elapsed":0,"Remaining":0,"TotalRecords":0,"ProcessedRecords":0}}}
2025-03-16T08:21:22Z    INFO    hashstore       finished compaction     {"Process": "storagenode", "satellite": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs", "store": "s1", "duration": "3.334633252s", "stats": {"NumLogs":523,"LenLogs":"509.9 GiB","NumLogsTTL":114,"LenLogsTTL":"105.8 GiB","SetPercent":0.9924699990323311,"TrashPercent":0.2769289505666312,"Compacting":false,"Compactions":0,"TableFull":0,"Today":20163,"LastCompact":20163,"LogsRewritten":4,"DataRewritten":"106.1 KiB","Table":{"NumSet":1686435,"LenSet":"506.1 GiB","AvgSet":322213.1068200079,"NumTrash":209324,"LenTrash":"141.2 GiB","AvgTrash":724343.8157879651,"NumSlots":4194304,"TableSize":"256.0 MiB","Load":0.40207743644714355,"Created":20163},"Compaction":{"Elapsed":0,"Remaining":0,"TotalRecords":0,"ProcessedRecords":0}}}
2025-03-16T18:00:54Z    INFO    hashstore       finished compaction     {"Process": "storagenode", "satellite": "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE", "store": "s1", "duration": "85.748208ms", "stats": {"NumLogs":2,"LenLogs":"1.2 GiB","NumLogsTTL":0,"LenLogsTTL":"0 B","SetPercent":0.9915716592526717,"TrashPercent":0,"Compacting":false,"Compactions":0,"TableFull":0,"Today":20163,"LastCompact":20163,"LogsRewritten":0,"DataRewritten":"0 B","Table":{"NumSet":1734,"LenSet":"1.30 GB","AvgSet":747653.9792387544,"NumTrash":0,"LenTrash":"0 B","AvgTrash":0,"NumSlots":16384,"TableSize":"1.0 MiB","Load":0.1058349609375,"Created":20142},"Compaction":{"Elapsed":0,"Remaining":0,"TotalRecords":0,"ProcessedRecords":0}}}
2025-03-16T20:23:02Z    INFO    hashstore       finished compaction     {"Process": "storagenode", "satellite": "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE", "store": "s0", "duration": "12.907282ms", "stats": {"NumLogs":1,"LenLogs":"44.4 MiB","NumLogsTTL":0,"LenLogsTTL":"0 B","SetPercent":1,"TrashPercent":0,"Compacting":false,"Compactions":0,"TableFull":0,"Today":20163,"LastCompact":20163,"LogsRewritten":0,"DataRewritten":"0 B","Table":{"NumSet":78,"LenSet":"44.4 MiB","AvgSet":596681.8461538461,"NumTrash":0,"LenTrash":"0 B","AvgTrash":0,"NumSlots":16384,"TableSize":"1.0 MiB","Load":0.0047607421875,"Created":20150},"Compaction":{"Elapsed":0,"Remaining":0,"TotalRecords":0,"ProcessedRecords":0}}}
2025-03-17T06:18:05Z    INFO    hashstore       finished compaction     {"Process": "storagenode", "satellite": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs", "store": "s0", "duration": "10m26.33924369s", "stats": {"NumLogs":135,"LenLogs":"129.0 GiB","NumLogsTTL":12,"LenLogsTTL":"6.3 GiB","SetPercent":0.977476256980048,"TrashPercent":0.0605360596612248,"Compacting":false,"Compactions":1,"TableFull":0,"Today":20164,"LastCompact":20164,"LogsRewritten":311,"DataRewritten":"152.8 GiB","Table":{"NumSet":501462,"LenSet":"126.1 GiB","AvgSet":269958.7894915268,"NumTrash":19884,"LenTrash":"7.8 GiB","AvgTrash":421637.93522430095,"NumSlots":1048576,"TableSize":"64.0 MiB","Load":0.47823143005371094,"Created":20164},"Compaction":{"Elapsed":0,"Remaining":0,"TotalRecords":0,"ProcessedRecords":0}}}
2025-03-17T11:55:11Z    INFO    hashstore       finished compaction     {"Process": "storagenode", "satellite": "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S", "store": "s0", "duration": "37m37.510241714s", "stats": {"NumLogs":455,"LenLogs":"447.1 GiB","NumLogsTTL":43,"LenLogsTTL":"23.3 GiB","SetPercent":0.9619675237854887,"TrashPercent":0.08383192003453378,"Compacting":false,"Compactions":1,"TableFull":0,"Today":20164,"LastCompact":20164,"LogsRewritten":315,"DataRewritten":"208.4 GiB","Table":{"NumSet":2202291,"LenSet":"430.1 GiB","AvgSet":209682.92006460545,"NumTrash":180895,"LenTrash":"37.5 GiB","AvgTrash":222464.234567014,"NumSlots":8388608,"TableSize":"512.0 MiB","Load":0.2625335454940796,"Created":20164},"Compaction":{"Elapsed":0,"Remaining":0,"TotalRecords":0,"ProcessedRecords":0}}}
2025-03-17T17:37:00Z    INFO    hashstore       finished compaction     {"Process": "storagenode", "satellite": "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S", "store": "s1", "duration": "1h5m23.886708469s", "stats": {"NumLogs":1056,"LenLogs":"1.0 TiB","NumLogsTTL":49,"LenLogsTTL":"33.2 GiB","SetPercent":0.9844274126567788,"TrashPercent":0.11403466419219797,"Compacting":false,"Compactions":1,"TableFull":0,"Today":20164,"LastCompact":20164,"LogsRewritten":773,"DataRewritten":"627.2 GiB","Table":{"NumSet":5442000,"LenSet":"1.0 TiB","AvgSet":202432.92298419698,"NumTrash":493285,"LenTrash":"118.8 GiB","AvgTrash":258699.12710096597,"NumSlots":16777216,"TableSize":"1.0 GiB","Load":0.3243684768676758,"Created":20164},"Compaction":{"Elapsed":0,"Remaining":0,"TotalRecords":0,"ProcessedRecords":0}}}
2025-03-17T23:00:59Z    INFO    hashstore       finished compaction     {"Process": "storagenode", "satellite": "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6", "store": "s0", "duration": "4m41.254715654s", "stats": {"NumLogs":62,"LenLogs":"52.6 GiB","NumLogsTTL":7,"LenLogsTTL":"1.3 GiB","SetPercent":0.8478543478837633,"TrashPercent":0.185078292569971,"Compacting":false,"Compactions":0,"TableFull":0,"Today":20164,"LastCompact":20164,"LogsRewritten":19,"DataRewritten":"7.4 GiB","Table":{"NumSet":155403,"LenSet":"44.6 GiB","AvgSet":308399.7350887692,"NumTrash":28250,"LenTrash":"9.7 GiB","AvgTrash":370330.2388672566,"NumSlots":524288,"TableSize":"32.0 MiB","Load":0.29640769958496094,"Created":20164},"Compaction":{"Elapsed":0,"Remaining":0,"TotalRecords":0,"ProcessedRecords":0}}}
2025-03-18T09:28:44Z    INFO    hashstore       finished compaction     {"Process": "storagenode", "satellite": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs", "store": "s1", "duration": "5m32.049257518s", "stats": {"NumLogs":427,"LenLogs":"416.1 GiB","NumLogsTTL":26,"LenLogsTTL":"15.7 GiB","SetPercent":0.9789808308774428,"TrashPercent":0.10638853875398453,"Compacting":false,"Compactions":1,"TableFull":0,"Today":20165,"LastCompact":20165,"LogsRewritten":123,"DataRewritten":"16.9 GiB","Table":{"NumSet":1602995,"LenSet":"407.3 GiB","AvgSet":272845.9445375687,"NumTrash":122902,"LenTrash":"44.3 GiB","AvgTrash":386733.12837870826,"NumSlots":4194304,"TableSize":"256.0 MiB","Load":0.3821837902069092,"Created":20165},"Compaction":{"Elapsed":0,"Remaining":0,"TotalRecords":0,"ProcessedRecords":0}}}
2025-03-18T14:39:43Z    INFO    hashstore       finished compaction     {"Process": "storagenode", "satellite": "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6", "store": "s1", "duration": "33.221621719s", "stats": {"NumLogs":68,"LenLogs":"58.7 GiB","NumLogsTTL":5,"LenLogsTTL":"349.8 MiB","SetPercent":0.949545658523835,"TrashPercent":0.21592057944433496,"Compacting":false,"Compactions":1,"TableFull":0,"Today":20165,"LastCompact":20165,"LogsRewritten":11,"DataRewritten":"2.8 GiB","Table":{"NumSet":243342,"LenSet":"55.8 GiB","AvgSet":246008.55224334475,"NumTrash":35060,"LenTrash":"12.7 GiB","AvgTrash":388269.74557900743,"NumSlots":524288,"TableSize":"32.0 MiB","Load":0.4641380310058594,"Created":20165},"Compaction":{"Elapsed":0,"Remaining":0,"TotalRecords":0,"ProcessedRecords":0}}}
2025-03-18T18:21:21Z    INFO    hashstore       finished compaction     {"Process": "storagenode", "satellite": "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE", "store": "s1", "duration": "18.777444ms", "stats": {"NumLogs":2,"LenLogs":"1.2 GiB","NumLogsTTL":0,"LenLogsTTL":"0 B","SetPercent":0.9916686585912271,"TrashPercent":0,"Compacting":false,"Compactions":0,"TableFull":0,"Today":20165,"LastCompact":20165,"LogsRewritten":0,"DataRewritten":"0 B","Table":{"NumSet":1750,"LenSet":"1.2 GiB","AvgSet":749516.7268571429,"NumTrash":0,"LenTrash":"0 B","AvgTrash":0,"NumSlots":16384,"TableSize":"1.0 MiB","Load":0.1068115234375,"Created":20142},"Compaction":{"Elapsed":0,"Remaining":0,"TotalRecords":0,"ProcessedRecords":0}}}
2025-03-18T21:50:15Z    INFO    hashstore       finished compaction     {"Process": "storagenode", "satellite": "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE", "store": "s0", "duration": "21.865381ms", "stats": {"NumLogs":1,"LenLogs":"44.4 MiB","NumLogsTTL":0,"LenLogsTTL":"0 B","SetPercent":1,"TrashPercent":0,"Compacting":false,"Compactions":0,"TableFull":0,"Today":20165,"LastCompact":20165,"LogsRewritten":0,"DataRewritten":"0 B","Table":{"NumSet":78,"LenSet":"44.4 MiB","AvgSet":596681.8461538461,"NumTrash":0,"LenTrash":"0 B","AvgTrash":0,"NumSlots":16384,"TableSize":"1.0 MiB","Load":0.0047607421875,"Created":20150},"Compaction":{"Elapsed":0,"Remaining":0,"TotalRecords":0,"ProcessedRecords":0}}}
2025-03-19T06:03:52Z    INFO    hashstore       finished compaction     {"Process": "storagenode", "satellite": "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S", "store": "s1", "duration": "21m6.813864161s", "stats": {"NumLogs":1029,"LenLogs":"1.0 TiB","NumLogsTTL":28,"LenLogsTTL":"14.5 GiB","SetPercent":0.9860338552638493,"TrashPercent":0.09259167730859334,"Compacting":false,"Compactions":0,"TableFull":0,"Today":20166,"LastCompact":20166,"LogsRewritten":74,"DataRewritten":"37.0 GiB","Table":{"NumSet":5239094,"LenSet":"1.0 TiB","AvgSet":205593.5572310785,"NumTrash":278239,"LenTrash":"94.2 GiB","AvgTrash":363519.5838972969,"NumSlots":16777216,"TableSize":"1.0 GiB","Load":0.31227433681488037,"Created":20166},"Compaction":{"Elapsed":0,"Remaining":0,"TotalRecords":0,"ProcessedRecords":0}}}
2025-03-19T10:02:42Z    INFO    hashstore       finished compaction     {"Process": "storagenode", "satellite": "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6", "store": "s0", "duration": "1m34.579402571s", "stats": {"NumLogs":54,"LenLogs":"50.2 GiB","NumLogsTTL":5,"LenLogsTTL":"1.6 GiB","SetPercent":0.8915286394919524,"TrashPercent":0.2010531153068113,"Compacting":false,"Compactions":0,"TableFull":0,"Today":20166,"LastCompact":20166,"LogsRewritten":11,"DataRewritten":"4.77 GB","Table":{"NumSet":157034,"LenSet":"44.8 GiB","AvgSet":306201.53332399356,"NumTrash":29329,"LenTrash":"10.1 GiB","AvgTrash":369725.4163455965,"NumSlots":524288,"TableSize":"32.0 MiB","Load":0.2995185852050781,"Created":20166},"Compaction":{"Elapsed":0,"Remaining":0,"TotalRecords":0,"ProcessedRecords":0}}}
2025-03-19T18:46:38Z    INFO    hashstore       finished compaction     {"Process": "storagenode", "satellite": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs", "store": "s0", "duration": "57.542701843s", "stats": {"NumLogs":140,"LenLogs":"130.2 GiB","NumLogsTTL":9,"LenLogsTTL":"1.0 GiB","SetPercent":0.9883789064663699,"TrashPercent":0.022027641970846654,"Compacting":false,"Compactions":0,"TableFull":0,"Today":20166,"LastCompact":20166,"LogsRewritten":11,"DataRewritten":"2.1 GiB","Table":{"NumSet":516893,"LenSet":"128.7 GiB","AvgSet":267387.420143047,"NumTrash":11976,"LenTrash":"2.9 GiB","AvgTrash":257202.02004008016,"NumSlots":1048576,"TableSize":"64.0 MiB","Load":0.4929475784301758,"Created":20166},"Compaction":{"Elapsed":0,"Remaining":0,"TotalRecords":0,"ProcessedRecords":0}}}
2025-03-20T09:10:12Z    INFO    hashstore       finished compaction     {"Process": "storagenode", "satellite": "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE", "store": "s1", "duration": "60.229695ms", "stats": {"NumLogs":2,"LenLogs":"1.2 GiB","NumLogsTTL":0,"LenLogsTTL":"0 B","SetPercent":0.9917256368777997,"TrashPercent":0,"Compacting":false,"Compactions":1,"TableFull":0,"Today":20167,"LastCompact":20167,"LogsRewritten":0,"DataRewritten":"0 B","Table":{"NumSet":1768,"LenSet":"1.2 GiB","AvgSet":747037.5384615385,"NumTrash":0,"LenTrash":"0 B","AvgTrash":0,"NumSlots":16384,"TableSize":"1.0 MiB","Load":0.10791015625,"Created":20142},"Compaction":{"Elapsed":0,"Remaining":0,"TotalRecords":0,"ProcessedRecords":0}}}
2025-03-21T00:06:30Z    INFO    hashstore       finished compaction     {"Process": "storagenode", "satellite": "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S", "store": "s0", "duration": "22m17.35549623s", "stats": {"NumLogs":501,"LenLogs":"489.5 GiB","NumLogsTTL":41,"LenLogsTTL":"20.5 GiB","SetPercent":0.9776326527260366,"TrashPercent":0.06993258926964308,"Compacting":false,"Compactions":0,"TableFull":0,"Today":20168,"LastCompact":20167,"LogsRewritten":83,"DataRewritten":"49.9 GiB","Table":{"NumSet":2311640,"LenSet":"478.5 GiB","AvgSet":222282.83318769358,"NumTrash":155133,"LenTrash":"34.2 GiB","AvgTrash":236933.15810304706,"NumSlots":8388608,"TableSize":"512.0 MiB","Load":0.27556896209716797,"Created":20167},"Compaction":{"Elapsed":0,"Remaining":0,"TotalRecords":0,"ProcessedRecords":0}}}
2025-03-21T01:48:44Z    INFO    hashstore       finished compaction     {"Process": "storagenode", "satellite": "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6", "store": "s1", "duration": "1m9.549506316s", "stats": {"NumLogs":65,"LenLogs":"59.2 GiB","NumLogsTTL":6,"LenLogsTTL":"223.3 MiB","SetPercent":0.9103636253639041,"TrashPercent":0.1638555968164577,"Compacting":false,"Compactions":0,"TableFull":0,"Today":20168,"LastCompact":20168,"LogsRewritten":11,"DataRewritten":"4.9 GiB","Table":{"NumSet":261030,"LenSet":"53.9 GiB","AvgSet":221550.4611117496,"NumTrash":27064,"LenTrash":"9.7 GiB","AvgTrash":384607.26219331956,"NumSlots":524288,"TableSize":"32.0 MiB","Load":0.4978752136230469,"Created":20168},"Compaction":{"Elapsed":0,"Remaining":0,"TotalRecords":0,"ProcessedRecords":0}}}
2025-03-21T04:40:43Z    INFO    hashstore       finished compaction     {"Process": "storagenode", "satellite": "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE", "store": "s1", "duration": "11.543543ms", "stats": {"NumLogs":2,"LenLogs":"1.2 GiB","NumLogsTTL":0,"LenLogsTTL":"0 B","SetPercent":0.9917263382413162,"TrashPercent":0,"Compacting":false,"Compactions":0,"TableFull":0,"Today":20168,"LastCompact":20168,"LogsRewritten":0,"DataRewritten":"0 B","Table":{"NumSet":1772,"LenSet":"1.2 GiB","AvgSet":745414.934537246,"NumTrash":0,"LenTrash":"0 B","AvgTrash":0,"NumSlots":16384,"TableSize":"1.0 MiB","Load":0.108154296875,"Created":20142},"Compaction":{"Elapsed":0,"Remaining":0,"TotalRecords":0,"ProcessedRecords":0}}}
2025-03-21T06:13:46Z    INFO    hashstore       finished compaction     {"Process": "storagenode", "satellite": "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S", "store": "s1", "duration": "21m31.419894324s", "stats": {"NumLogs":1022,"LenLogs":"1.0 TiB","NumLogsTTL":29,"LenLogsTTL":"10.8 GiB","SetPercent":0.9873348988457331,"TrashPercent":0.096168718668957,"Compacting":false,"Compactions":0,"TableFull":0,"Today":20168,"LastCompact":20168,"LogsRewritten":61,"DataRewritten":"37.2 GiB","Table":{"NumSet":5174383,"LenSet":"1.0 TiB","AvgSet":206185.0558093593,"NumTrash":230550,"LenTrash":"96.8 GiB","AvgTrash":450733.6386206896,"NumSlots":16777216,"TableSize":"1.0 GiB","Load":0.30841726064682007,"Created":20168},"Compaction":{"Elapsed":0,"Remaining":0,"TotalRecords":0,"ProcessedRecords":0}}}
2025-03-21T17:17:53Z    INFO    hashstore       finished compaction     {"Process": "storagenode", "satellite": "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6", "store": "s0", "duration": "1m23.7158304s", "stats": {"NumLogs":53,"LenLogs":"47.6 GiB","NumLogsTTL":5,"LenLogsTTL":"342.7 MiB","SetPercent":0.8661878928124593,"TrashPercent":0.14440633811046907,"Compacting":false,"Compactions":0,"TableFull":0,"Today":20168,"LastCompact":20168,"LogsRewritten":8,"DataRewritten":"4.0 GiB","Table":{"NumSet":149620,"LenSet":"41.2 GiB","AvgSet":295982.09950541373,"NumTrash":12092,"LenTrash":"6.9 GiB","AvgTrash":610563.8742970559,"NumSlots":524288,"TableSize":"32.0 MiB","Load":0.28537750244140625,"Created":20168},"Compaction":{"Elapsed":0,"Remaining":0,"TotalRecords":0,"ProcessedRecords":0}}}
2025-03-21T20:12:05Z    INFO    hashstore       finished compaction     {"Process": "storagenode", "satellite": "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE", "store": "s0", "duration": "29.303753ms", "stats": {"NumLogs":1,"LenLogs":"49.1 MiB","NumLogsTTL":0,"LenLogsTTL":"0 B","SetPercent":0.9580159452935044,"TrashPercent":0,"Compacting":false,"Compactions":0,"TableFull":0,"Today":20168,"LastCompact":20168,"LogsRewritten":0,"DataRewritten":"0 B","Table":{"NumSet":81,"LenSet":"47.1 MiB","AvgSet":609252.3456790124,"NumTrash":0,"LenTrash":"0 B","AvgTrash":0,"NumSlots":16384,"TableSize":"1.0 MiB","Load":0.00494384765625,"Created":20150},"Compaction":{"Elapsed":0,"Remaining":0,"TotalRecords":0,"ProcessedRecords":0}}}

Also, may be this fix will change your situation: https://github.com/storj/storj/commit/82674e6

1 Like