How do you solve slow file deletion on ext4?

root@STORJ7:~# docker logs storagenode 2> /dev/null | grep -i walk
2024-07-25T17:42:25Z    INFO    lazyfilewalker.trash-cleanup-filewalker starting subprocess                            {"Process": "storagenode", "satelliteID": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs"}
2024-07-25T17:42:25Z    INFO    lazyfilewalker.trash-cleanup-filewalker subprocess started                             {"Process": "storagenode", "satelliteID": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs"}
2024-07-25T17:42:25Z    INFO    lazyfilewalker.trash-cleanup-filewalker.subprocess                                     trash-filewalker started {"Process": "storagenode", "satelliteID": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs", "Process": "storagenode", "dateBefore": "2024-07-18T17:42:25Z"}
2024-07-25T17:42:25Z    INFO    lazyfilewalker.trash-cleanup-filewalker.subprocess                                     Database started {"Process": "storagenode", "satelliteID": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs", "Process": "storagenode"}
2024-07-25T17:42:25Z    INFO    lazyfilewalker.trash-cleanup-filewalker.subprocess                                     trash-filewalker completed       {"Process": "storagenode", "satelliteID": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs", "Process": "storagenode", "bytesDeleted": 0, "numKeysDeleted": 0}
2024-07-25T17:42:25Z    INFO    pieces  used-space-filewalker started   {"Process": "storagenode", "Satellite ID": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs"}
2024-07-25T17:42:25Z    INFO    lazyfilewalker.used-space-filewalker    starting subprocess                            {"Process": "storagenode", "satelliteID": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs"}
2024-07-25T17:42:25Z    INFO    lazyfilewalker.used-space-filewalker    subprocess started                             {"Process": "storagenode", "satelliteID": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs"}
2024-07-25T17:42:25Z    INFO    lazyfilewalker.trash-cleanup-filewalker subprocess finished successfully               {"Process": "storagenode", "satelliteID": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs"}
2024-07-25T17:42:25Z    INFO    lazyfilewalker.trash-cleanup-filewalker starting subprocess                            {"Process": "storagenode", "satelliteID": "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE"}
2024-07-25T17:42:25Z    INFO    lazyfilewalker.trash-cleanup-filewalker subprocess started                             {"Process": "storagenode", "satelliteID": "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE"}
2024-07-25T17:42:25Z    INFO    lazyfilewalker.trash-cleanup-filewalker.subprocess                                     trash-filewalker started {"Process": "storagenode", "satelliteID": "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE", "Process": "storagenode", "dateBefore": "2024-07-18T17:42:25Z"}
2024-07-25T17:42:25Z    INFO    lazyfilewalker.used-space-filewalker.subprocess                                        Database started {"Process": "storagenode", "satelliteID": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs", "Process": "storagenode"}
2024-07-25T17:42:25Z    INFO    lazyfilewalker.trash-cleanup-filewalker.subprocess                                     Database started {"Process": "storagenode", "satelliteID": "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE", "Process": "storagenode"}
2024-07-25T17:42:25Z    INFO    lazyfilewalker.trash-cleanup-filewalker.subprocess                                     trash-filewalker completed       {"Process": "storagenode", "satelliteID": "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE", "Process": "storagenode", "bytesDeleted": 0, "numKeysDeleted": 0}
2024-07-25T17:42:25Z    INFO    lazyfilewalker.trash-cleanup-filewalker subprocess finished successfully               {"Process": "storagenode", "satelliteID": "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE"}
2024-07-25T17:42:25Z    INFO    lazyfilewalker.trash-cleanup-filewalker starting subprocess                            {"Process": "storagenode", "satelliteID": "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6"}
2024-07-25T17:42:25Z    INFO    lazyfilewalker.trash-cleanup-filewalker subprocess started                             {"Process": "storagenode", "satelliteID": "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6"}
2024-07-25T17:42:25Z    INFO    lazyfilewalker.trash-cleanup-filewalker.subprocess                                     trash-filewalker started {"Process": "storagenode", "satelliteID": "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6", "Process": "storagenode", "dateBefore": "2024-07-18T17:42:25Z"}
2024-07-25T17:42:25Z    INFO    lazyfilewalker.trash-cleanup-filewalker.subprocess                                     Database started {"Process": "storagenode", "satelliteID": "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6", "Process": "storagenode"}
2024-07-25T17:42:25Z    INFO    lazyfilewalker.trash-cleanup-filewalker.subprocess                                     trash-filewalker completed       {"Process": "storagenode", "satelliteID": "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6", "Process": "storagenode", "bytesDeleted": 0, "numKeysDeleted": 0}
2024-07-25T17:42:25Z    INFO    lazyfilewalker.trash-cleanup-filewalker subprocess finished successfully               {"Process": "storagenode", "satelliteID": "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6"}
2024-07-25T17:42:25Z    INFO    lazyfilewalker.trash-cleanup-filewalker starting subprocess                            {"Process": "storagenode", "satelliteID": "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S"}
2024-07-25T17:42:25Z    INFO    lazyfilewalker.trash-cleanup-filewalker subprocess started                             {"Process": "storagenode", "satelliteID": "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S"}
2024-07-25T17:42:25Z    INFO    lazyfilewalker.trash-cleanup-filewalker.subprocess                                     trash-filewalker started {"Process": "storagenode", "satelliteID": "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S", "Process": "storagenode", "dateBefore": "2024-07-18T17:42:25Z"}
2024-07-25T17:42:25Z    INFO    lazyfilewalker.trash-cleanup-filewalker.subprocess                                     Database started {"Process": "storagenode", "satelliteID": "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S", "Process": "storagenode"}
2024-07-25T17:42:25Z    INFO    lazyfilewalker.trash-cleanup-filewalker.subprocess                                     trash-filewalker completed       {"Process": "storagenode", "satelliteID": "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S", "Process": "storagenode", "bytesDeleted": 0, "numKeysDeleted": 0}
2024-07-25T17:42:25Z    INFO    lazyfilewalker.trash-cleanup-filewalker subprocess finished successfully               {"Process": "storagenode", "satelliteID": "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S"}
2024-07-25T17:45:02Z    INFO    lazyfilewalker.used-space-filewalker    subprocess finished successfully               {"Process": "storagenode", "satelliteID": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs"}
2024-07-25T17:45:02Z    INFO    pieces  used-space-filewalker completed {"Process": "storagenode", "Satellite ID": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs", "Lazy File Walker": true, "Total Pieces Size": 67104322048, "Total Pieces Content Size": 66981463552}
2024-07-25T17:45:02Z    INFO    pieces  used-space-filewalker started   {"Process": "storagenode", "Satellite ID": "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S"}
2024-07-25T17:45:02Z    INFO    lazyfilewalker.used-space-filewalker    starting subprocess                            {"Process": "storagenode", "satelliteID": "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S"}
2024-07-25T17:45:02Z    INFO    lazyfilewalker.used-space-filewalker    subprocess started                             {"Process": "storagenode", "satelliteID": "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S"}
2024-07-25T17:45:02Z    INFO    lazyfilewalker.used-space-filewalker.subprocess                                        Database started {"Process": "storagenode", "satelliteID": "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S", "Process": "storagenode"}
2024-07-25T17:48:33Z    INFO    lazyfilewalker.used-space-filewalker    subprocess finished successfully               {"Process": "storagenode", "satelliteID": "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S"}
2024-07-25T17:48:33Z    INFO    pieces  used-space-filewalker completed {"Process": "storagenode", "Satellite ID": "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S", "Lazy File Walker": true, "Total Pieces Size": 324812015368, "Total Pieces Content Size": 324030394632}
2024-07-25T17:48:33Z    INFO    pieces  used-space-filewalker started   {"Process": "storagenode", "Satellite ID": "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE"}
2024-07-25T17:48:33Z    INFO    lazyfilewalker.used-space-filewalker    starting subprocess                            {"Process": "storagenode", "satelliteID": "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE"}
2024-07-25T17:48:33Z    INFO    lazyfilewalker.used-space-filewalker    subprocess started                             {"Process": "storagenode", "satelliteID": "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE"}
2024-07-25T17:48:33Z    INFO    lazyfilewalker.used-space-filewalker.subprocess                                        Database started {"Process": "storagenode", "satelliteID": "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE", "Process": "storagenode"}
2024-07-25T18:03:16Z    INFO    lazyfilewalker.used-space-filewalker    subprocess finished successfully               {"Process": "storagenode", "satelliteID": "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE"}
2024-07-25T18:03:16Z    INFO    pieces  used-space-filewalker completed {"Process": "storagenode", "Satellite ID": "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE", "Lazy File Walker": true, "Total Pieces Size": 394558090752, "Total Pieces Content Size": 393343318016}
2024-07-25T18:03:16Z    INFO    pieces  used-space-filewalker started   {"Process": "storagenode", "Satellite ID": "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6"}
2024-07-25T18:03:16Z    INFO    lazyfilewalker.used-space-filewalker    starting subprocess                            {"Process": "storagenode", "satelliteID": "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6"}
2024-07-25T18:03:16Z    INFO    lazyfilewalker.used-space-filewalker    subprocess started                             {"Process": "storagenode", "satelliteID": "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6"}
2024-07-25T18:03:16Z    INFO    lazyfilewalker.used-space-filewalker.subprocess                                        Database started {"Process": "storagenode", "satelliteID": "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6", "Process": "storagenode"}
2024-07-25T18:03:50Z    INFO    lazyfilewalker.used-space-filewalker    subprocess finished successfully               {"Process": "storagenode", "satelliteID": "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6"}
2024-07-25T18:03:50Z    INFO    pieces  used-space-filewalker completed {"Process": "storagenode", "Satellite ID": "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6", "Lazy File Walker": true, "Total Pieces Size": 8268178176, "Total Pieces Content Size": 8256300288}
2024-07-26T01:28:32Z    INFO    lazyfilewalker.gc-filewalker    starting subprocess                                    {"Process": "storagenode", "satelliteID": "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6"}
2024-07-26T01:28:32Z    INFO    lazyfilewalker.gc-filewalker    subprocess started                                     {"Process": "storagenode", "satelliteID": "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6"}
2024-07-26T01:28:32Z    INFO    lazyfilewalker.gc-filewalker.subprocess Database started                               {"Process": "storagenode", "satelliteID": "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6", "Process": "storagenode"}
2024-07-26T01:28:32Z    INFO    lazyfilewalker.gc-filewalker.subprocess gc-filewalker started                          {"Process": "storagenode", "satelliteID": "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6", "createdBefore": "2024-07-21T17:59:57Z", "bloomFilterSize": 12864, "Process": "storagenode"}
2024-07-26T01:28:36Z    INFO    lazyfilewalker.gc-filewalker.subprocess gc-filewalker completed                        {"Process": "storagenode", "satelliteID": "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6", "piecesSkippedCount": 0, "Process": "storagenode", "piecesCount": 23199, "trashPiecesCount": 1442, "piecesTrashed": 1442}
2024-07-26T01:28:36Z    INFO    lazyfilewalker.gc-filewalker    subprocess finished successfully                       {"Process": "storagenode", "satelliteID": "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6"}
2024-07-26T17:42:25Z    INFO    lazyfilewalker.trash-cleanup-filewalker starting subprocess                            {"Process": "storagenode", "satelliteID": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs"}
2024-07-26T17:42:25Z    INFO    lazyfilewalker.trash-cleanup-filewalker subprocess started                             {"Process": "storagenode", "satelliteID": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs"}
2024-07-26T17:42:25Z    INFO    lazyfilewalker.trash-cleanup-filewalker.subprocess                                     trash-filewalker started {"Process": "storagenode", "satelliteID": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs", "Process": "storagenode", "dateBefore": "2024-07-19T17:42:25Z"}
2024-07-26T17:42:26Z    INFO    lazyfilewalker.trash-cleanup-filewalker.subprocess                                     Database started {"Process": "storagenode", "satelliteID": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs", "Process": "storagenode"}
2024-07-26T17:46:55Z    INFO    lazyfilewalker.trash-cleanup-filewalker.subprocess                                     trash-filewalker completed       {"Process": "storagenode", "satelliteID": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs", "numKeysDeleted": 28124, "Process": "storagenode", "bytesDeleted": 26861542400}
2024-07-26T17:46:55Z    INFO    lazyfilewalker.trash-cleanup-filewalker subprocess finished successfully               {"Process": "storagenode", "satelliteID": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs"}
2024-07-26T17:46:55Z    INFO    lazyfilewalker.trash-cleanup-filewalker starting subprocess                            {"Process": "storagenode", "satelliteID": "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE"}
2024-07-26T17:46:55Z    INFO    lazyfilewalker.trash-cleanup-filewalker subprocess started                             {"Process": "storagenode", "satelliteID": "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE"}
2024-07-26T17:46:55Z    INFO    lazyfilewalker.trash-cleanup-filewalker.subprocess                                     trash-filewalker started {"Process": "storagenode", "satelliteID": "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE", "Process": "storagenode", "dateBefore": "2024-07-19T17:46:55Z"}
2024-07-26T17:46:56Z    INFO    lazyfilewalker.trash-cleanup-filewalker.subprocess                                     Database started {"Process": "storagenode", "satelliteID": "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE", "Process": "storagenode"}
2024-07-26T17:46:56Z    INFO    lazyfilewalker.trash-cleanup-filewalker.subprocess                                     trash-filewalker completed       {"Process": "storagenode", "satelliteID": "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE", "Process": "storagenode", "bytesDeleted": 0, "numKeysDeleted": 0}
2024-07-26T17:46:56Z    INFO    lazyfilewalker.trash-cleanup-filewalker subprocess finished successfully               {"Process": "storagenode", "satelliteID": "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE"}
2024-07-26T17:46:56Z    INFO    lazyfilewalker.trash-cleanup-filewalker starting subprocess                            {"Process": "storagenode", "satelliteID": "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6"}
2024-07-26T17:46:56Z    INFO    lazyfilewalker.trash-cleanup-filewalker subprocess started                             {"Process": "storagenode", "satelliteID": "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6"}
2024-07-26T17:46:56Z    INFO    lazyfilewalker.trash-cleanup-filewalker.subprocess                                     trash-filewalker started {"Process": "storagenode", "satelliteID": "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6", "Process": "storagenode", "dateBefore": "2024-07-19T17:46:56Z"}
2024-07-26T17:46:56Z    INFO    lazyfilewalker.trash-cleanup-filewalker.subprocess                                     Database started {"Process": "storagenode", "satelliteID": "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6", "Process": "storagenode"}
2024-07-26T17:46:56Z    INFO    lazyfilewalker.trash-cleanup-filewalker.subprocess                                     trash-filewalker completed       {"Process": "storagenode", "satelliteID": "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6", "bytesDeleted": 0, "numKeysDeleted": 0, "Process": "storagenode"}
2024-07-26T17:46:56Z    INFO    lazyfilewalker.trash-cleanup-filewalker subprocess finished successfully               {"Process": "storagenode", "satelliteID": "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6"}
2024-07-26T17:46:56Z    INFO    lazyfilewalker.trash-cleanup-filewalker starting subprocess                            {"Process": "storagenode", "satelliteID": "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S"}
2024-07-26T17:46:56Z    INFO    lazyfilewalker.trash-cleanup-filewalker subprocess started                             {"Process": "storagenode", "satelliteID": "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S"}
2024-07-26T17:46:56Z    INFO    lazyfilewalker.trash-cleanup-filewalker.subprocess                                     trash-filewalker started {"Process": "storagenode", "satelliteID": "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S", "Process": "storagenode", "dateBefore": "2024-07-19T17:46:56Z"}
2024-07-26T17:46:56Z    INFO    lazyfilewalker.trash-cleanup-filewalker.subprocess                                     Database started {"Process": "storagenode", "satelliteID": "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S", "Process": "storagenode"}
2024-07-26T17:46:56Z    INFO    lazyfilewalker.trash-cleanup-filewalker.subprocess                                     trash-filewalker completed       {"Process": "storagenode", "satelliteID": "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S", "Process": "storagenode", "bytesDeleted": 0, "numKeysDeleted": 0}
2024-07-26T17:46:56Z    INFO    lazyfilewalker.trash-cleanup-filewalker subprocess finished successfully               {"Process": "storagenode", "satelliteID": "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S"}

For a 1TB node. All walkers within some minutes…

1 Like