=== RUN   TestZs3serverWriteThroughFallback
    10_write_through_fallback_test.go:56: ZS3_UPSTREAM_ENDPOINT unset — write-through requires an upstream to observe
--- SKIP: TestZs3serverWriteThroughFallback (0.00s)
=== RUN   TestZs3serverCacheEvictionPolicy
=== RUN   TestZs3serverCacheEvictionPolicy/xattr_invariants
    11_cache_eviction_policy_test.go:67: Test case [xattr invariants] scheduled at [04-25-2026 01:56:12] 
    system_test_framework.go:148: Test case [xattr invariants] start at [04-25-2026 01:56:12] 
    11_cache_eviction_policy_test.go:67: Test case [xattr invariants] exit at [04-25-2026 01:56:12]
=== RUN   TestZs3serverCacheEvictionPolicy/usage_&_LRU_order
    11_cache_eviction_policy_test.go:120: Test case [usage & LRU order] scheduled at [04-25-2026 01:56:12] 
    system_test_framework.go:148: Test case [usage & LRU order] start at [04-25-2026 01:56:12] 
    11_cache_eviction_policy_test.go:179: eviction simulated: 7 of 12 files survive (target 7)
    11_cache_eviction_policy_test.go:120: Test case [usage & LRU order] exit at [04-25-2026 01:56:12]
=== RUN   TestZs3serverCacheEvictionPolicy/tracker_stats_endpoint
    11_cache_eviction_policy_test.go:184: Test case [tracker stats endpoint] scheduled at [04-25-2026 01:56:12] 
    system_test_framework.go:148: Test case [tracker stats endpoint] start at [04-25-2026 01:56:12] 
    11_cache_eviction_policy_test.go:184: Test case [tracker stats endpoint] exit at [04-25-2026 01:56:12]
--- PASS: TestZs3serverCacheEvictionPolicy (0.69s)
    --- PASS: TestZs3serverCacheEvictionPolicy/xattr_invariants (0.01s)
    --- PASS: TestZs3serverCacheEvictionPolicy/usage_&_LRU_order (0.02s)
    --- PASS: TestZs3serverCacheEvictionPolicy/tracker_stats_endpoint (0.67s)
=== RUN   TestZs3serverCrossProtocolWrite
=== RUN   TestZs3serverCrossProtocolWrite/S3_PUT_→_NFS_read
    12_cross_protocol_write_test.go:63: Test case [S3 PUT → NFS read] scheduled at [04-25-2026 01:56:12] 
    system_test_framework.go:148: Test case [S3 PUT → NFS read] start at [04-25-2026 01:56:12] 
    12_cross_protocol_write_test.go:86: NFS did not see S3-written key s3-to-nfs-1777074972937998671.bin within 10s
    12_cross_protocol_write_test.go:63: Test case [S3 PUT → NFS read] exit at [04-25-2026 01:56:23]
=== RUN   TestZs3serverCrossProtocolWrite/NFS_write_→_S3_GET
    12_cross_protocol_write_test.go:97: Test case [NFS write → S3 GET] scheduled at [04-25-2026 01:56:23] 
    system_test_framework.go:148: Test case [NFS write → S3 GET] start at [04-25-2026 01:56:23] 
    12_cross_protocol_write_test.go:122: S3 did not see NFS-written key nfs-to-s3-1777074983570108567.bin within 10s
    12_cross_protocol_write_test.go:97: Test case [NFS write → S3 GET] exit at [04-25-2026 01:56:33]
--- FAIL: TestZs3serverCrossProtocolWrite (20.72s)
    --- FAIL: TestZs3serverCrossProtocolWrite/S3_PUT_→_NFS_read (10.63s)
    --- FAIL: TestZs3serverCrossProtocolWrite/NFS_write_→_S3_GET (10.03s)
=== RUN   TestZs3serverWritePressure
=== RUN   TestZs3serverWritePressure/concurrent_S3+NFS_same-key_pressure
    13_write_pressure_test.go:77: Test case [concurrent S3+NFS same-key pressure] scheduled at [04-25-2026 01:56:33] 
    system_test_framework.go:148: Test case [concurrent S3+NFS same-key pressure] start at [04-25-2026 01:56:33] 
    13_write_pressure_test.go:157: ops: 103/200 complete, errors=0, writers s3=4 nfs=4
    13_write_pressure_test.go:190: final NFS read hotkey_006.bin: open /mnt/zus_nfs/zs3-write-pressure/hotkey_006.bin: no such file or directory
    13_write_pressure_test.go:77: Test case [concurrent S3+NFS same-key pressure] exit at [04-25-2026 01:56:34]
--- FAIL: TestZs3serverWritePressure (1.14s)
    --- FAIL: TestZs3serverWritePressure/concurrent_S3+NFS_same-key_pressure (1.11s)
=== RUN   TestZs3serverListCoverage
=== RUN   TestZs3serverListCoverage/pagination_(1500_objects)
    14_list_coverage_test.go:51: Test case [pagination (1500 objects)] scheduled at [04-25-2026 01:56:34] 
    system_test_framework.go:148: Test case [pagination (1500 objects)] start at [04-25-2026 01:56:34] 
    14_list_coverage_test.go:51: Test case [pagination (1500 objects)] timed out after [10m0s]
    14_list_coverage_test.go:51: Test case [pagination (1500 objects)] exit at [04-25-2026 02:06:34]
=== RUN   TestZs3serverListCoverage/prefix_filter
    14_list_coverage_test.go:101: Test case [prefix filter] scheduled at [04-25-2026 02:06:34] 
    system_test_framework.go:148: Test case [prefix filter] start at [04-25-2026 02:06:34] 
    14_list_coverage_test.go:137: prefix (ambiguous) expected ≥4, got 0
    14_list_coverage_test.go:101: Test case [prefix filter] exit at [04-25-2026 02:06:39]
=== RUN   TestZs3serverListCoverage/cross-protocol_LIST_S3→NFS
    14_list_coverage_test.go:142: Test case [cross-protocol LIST S3→NFS] scheduled at [04-25-2026 02:06:39] 
    system_test_framework.go:148: Test case [cross-protocol LIST S3→NFS] start at [04-25-2026 02:06:39] 
    14_list_coverage_test.go:145: ZS3_NFS_MOUNT unset, skipping cross-protocol LIST
    14_list_coverage_test.go:142: Test case [cross-protocol LIST S3→NFS] exit at [04-25-2026 02:06:39]
=== RUN   TestZs3serverListCoverage/post-delete_LIST
    14_list_coverage_test.go:187: Test case [post-delete LIST] scheduled at [04-25-2026 02:06:39] 
    system_test_framework.go:148: Test case [post-delete LIST] start at [04-25-2026 02:06:39] 
    14_list_coverage_test.go:216: post-delete LIST OK: no ghosts
    14_list_coverage_test.go:187: Test case [post-delete LIST] exit at [04-25-2026 02:06:42]
=== RUN   TestZs3serverListCoverage/delimiter_hierarchy
    14_list_coverage_test.go:219: Test case [delimiter hierarchy] scheduled at [04-25-2026 02:06:42] 
    system_test_framework.go:148: Test case [delimiter hierarchy] start at [04-25-2026 02:06:42] 
    14_list_coverage_test.go:258: delimiter hierarchy OK
    14_list_coverage_test.go:219: Test case [delimiter hierarchy] exit at [04-25-2026 02:06:45]
=== RUN   TestZs3serverListCoverage/empty_bucket_LIST
    14_list_coverage_test.go:261: Test case [empty bucket LIST] scheduled at [04-25-2026 02:06:45] 
    system_test_framework.go:148: Test case [empty bucket LIST] start at [04-25-2026 02:06:45] 
    14_list_coverage_test.go:273: empty LIST OK
    14_list_coverage_test.go:261: Test case [empty bucket LIST] exit at [04-25-2026 02:06:45]
--- FAIL: TestZs3serverListCoverage (610.51s)
    --- FAIL: TestZs3serverListCoverage/pagination_(1500_objects) (600.00s)
    --- FAIL: TestZs3serverListCoverage/prefix_filter (4.58s)
    --- SKIP: TestZs3serverListCoverage/cross-protocol_LIST_S3→NFS (0.00s)
    --- PASS: TestZs3serverListCoverage/post-delete_LIST (2.77s)
    --- PASS: TestZs3serverListCoverage/delimiter_hierarchy (3.06s)
    --- PASS: TestZs3serverListCoverage/empty_bucket_LIST (0.06s)
=== RUN   TestZs3serverPorcupineCopyMoveRename
=== RUN   TestZs3serverPorcupineCopyMoveRename/s3/copy
    15_porcupine_copy_move_rename_test.go:76: Test case [s3/copy] scheduled at [04-25-2026 02:06:45] 
    system_test_framework.go:148: Test case [s3/copy] start at [04-25-2026 02:06:45] 
    15_porcupine_copy_move_rename_test.go:416: op=copy collected 400 ops in 61.962s (errors=30) throughput=6.5 ops/s
    15_porcupine_copy_move_rename_test.go:424: RESULT op=copy: NON-LINEARIZABLE
    15_porcupine_copy_move_rename_test.go:76: Test case [s3/copy] exit at [04-25-2026 02:07:47]
=== RUN   TestZs3serverPorcupineCopyMoveRename/s3/move
    15_porcupine_copy_move_rename_test.go:82: Test case [s3/move] scheduled at [04-25-2026 02:07:47] 
    system_test_framework.go:148: Test case [s3/move] start at [04-25-2026 02:07:47] 
    15_porcupine_copy_move_rename_test.go:416: op=move collected 400 ops in 64.746s (errors=25) throughput=6.2 ops/s
    15_porcupine_copy_move_rename_test.go:424: RESULT op=move: NON-LINEARIZABLE
    15_porcupine_copy_move_rename_test.go:82: Test case [s3/move] exit at [04-25-2026 02:08:52]
=== RUN   TestZs3serverPorcupineCopyMoveRename/s3/rename
    15_porcupine_copy_move_rename_test.go:88: Test case [s3/rename] scheduled at [04-25-2026 02:08:52] 
    system_test_framework.go:148: Test case [s3/rename] start at [04-25-2026 02:08:52] 
    15_porcupine_copy_move_rename_test.go:93: rename backend: logical (CopyObject+DeleteObject) — no native S3 RENAME on zs3server
    15_porcupine_copy_move_rename_test.go:416: op=rename collected 400 ops in 59.999s (errors=16) throughput=6.7 ops/s
    15_porcupine_copy_move_rename_test.go:424: RESULT op=rename: NON-LINEARIZABLE
    15_porcupine_copy_move_rename_test.go:88: Test case [s3/rename] exit at [04-25-2026 02:09:52]
=== NAME  TestZs3serverPorcupineCopyMoveRename
    15_porcupine_copy_move_rename_test.go:101: ZS3_NFS_MOUNT unset, skipping fs/* subtests
--- FAIL: TestZs3serverPorcupineCopyMoveRename (186.78s)
    --- FAIL: TestZs3serverPorcupineCopyMoveRename/s3/copy (61.97s)
    --- FAIL: TestZs3serverPorcupineCopyMoveRename/s3/move (64.76s)
    --- FAIL: TestZs3serverPorcupineCopyMoveRename/s3/rename (60.01s)
=== RUN   TestZs3serverDirectZusNFSVisibility
=== RUN   TestZs3serverDirectZusNFSVisibility/direct-upload_via_rclone-zus_→_NFS_readdir_surfaces
    16_direct_zus_nfs_visibility_test.go:90: Test case [direct-upload via rclone-zus → NFS readdir surfaces] scheduled at [04-25-2026 02:09:52] 
    system_test_framework.go:148: Test case [direct-upload via rclone-zus → NFS readdir surfaces] start at [04-25-2026 02:09:52] 
    16_direct_zus_nfs_visibility_test.go:102: rclone-zus upload direct-1777075792030481552-a.bin: exit status 1: 2026/04/25 02:09:52 CRITICAL: Failed to create file system for destination "automation1k:zs3-direct-nfs/": allocation_fetch_error: Error fetching the allocation.internal_error: can't get allocation: error retrieving allocation: c7da617ff56df930beb58ba384d79de4e87a4688e029b12c3908e60b58981761, error: record not found
    16_direct_zus_nfs_visibility_test.go:90: Test case [direct-upload via rclone-zus → NFS readdir surfaces] exit at [04-25-2026 02:09:52]
--- FAIL: TestZs3serverDirectZusNFSVisibility (0.20s)
    --- FAIL: TestZs3serverDirectZusNFSVisibility/direct-upload_via_rclone-zus_→_NFS_readdir_surfaces (0.20s)
=== RUN   TestZs3ServerEnterpriseBasic
=== RUN   TestZs3ServerEnterpriseBasic/S3_PUT/GET_round-trip_via_enterprise_blobbers
    17_enterprise_blobber_test.go:63: Test case [S3 PUT/GET round-trip via enterprise blobbers] scheduled at [04-25-2026 02:09:52] 
    system_test_framework.go:148: Test case [S3 PUT/GET round-trip via enterprise blobbers] start at [04-25-2026 02:09:52] 
    17_enterprise_blobber_test.go:95: enterprise blobber S3 PUT→GET round-trip verified
    17_enterprise_blobber_test.go:63: Test case [S3 PUT/GET round-trip via enterprise blobbers] exit at [04-25-2026 02:09:52]
=== RUN   TestZs3ServerEnterpriseBasic/S3_multipart_PUT_large_file_via_enterprise_blobbers
    17_enterprise_blobber_test.go:98: Test case [S3 multipart PUT large file via enterprise blobbers] scheduled at [04-25-2026 02:09:52] 
    system_test_framework.go:148: Test case [S3 multipart PUT large file via enterprise blobbers] start at [04-25-2026 02:09:52] 
    17_enterprise_blobber_test.go:123: enterprise blobber warp PUT throughput:
        Report: PUT. Concurrency: 4. Ran: 27s
        * Average: 3.74 MiB/s, 3.74 obj/s
        * Reqs: Avg: 1071.9ms, 50%: 1088.2ms, 90%: 1110.3ms, 99%: 1112.4ms, Fastest: 578.6ms, Slowest: 1131.0ms, StdDev: 61.5ms
        Throughput, split into 27 x 1s:
        * Fastest: 5.6MiB/s, 5.56 obj/s
        * 50% Median: 3.7MiB/s, 3.67 obj/s
        * Slowest: 3.6MiB/s, 3.55 obj/s
    17_enterprise_blobber_test.go:98: Test case [S3 multipart PUT large file via enterprise blobbers] exit at [04-25-2026 02:10:26]
=== RUN   TestZs3ServerEnterpriseBasic/S3_DELETE_via_enterprise_blobbers
    17_enterprise_blobber_test.go:126: Test case [S3 DELETE via enterprise blobbers] scheduled at [04-25-2026 02:10:26] 
    system_test_framework.go:148: Test case [S3 DELETE via enterprise blobbers] start at [04-25-2026 02:10:26] 
    17_enterprise_blobber_test.go:149: enterprise blobber S3 DELETE verified
    17_enterprise_blobber_test.go:126: Test case [S3 DELETE via enterprise blobbers] exit at [04-25-2026 02:10:27]
--- PASS: TestZs3ServerEnterpriseBasic (35.42s)
    --- PASS: TestZs3ServerEnterpriseBasic/S3_PUT/GET_round-trip_via_enterprise_blobbers (0.71s)
    --- PASS: TestZs3ServerEnterpriseBasic/S3_multipart_PUT_large_file_via_enterprise_blobbers (33.97s)
    --- PASS: TestZs3ServerEnterpriseBasic/S3_DELETE_via_enterprise_blobbers (0.72s)
=== RUN   TestZs3serverMixedWarpTests
=== RUN   TestZs3serverMixedWarpTests/Warp_Mixed_Benchmark
    1_mixed_test.go:36: Test case [Warp Mixed Benchmark] scheduled at [04-25-2026 02:10:27] 
    system_test_framework.go:148: Test case [Warp Mixed Benchmark] start at [04-25-2026 02:10:27] 
    1_mixed_test.go:98: Bucket created successfully
    1_mixed_test.go:36: Test case [Warp Mixed Benchmark] exit at [04-25-2026 02:11:05]
--- PASS: TestZs3serverMixedWarpTests (37.57s)
    --- PASS: TestZs3serverMixedWarpTests/Warp_Mixed_Benchmark (37.57s)
=== RUN   TestZs3serverPutWarpTests
    2_put_test.go:99: Bucket created successfully
--- PASS: TestZs3serverPutWarpTests (34.74s)
=== RUN   TestZs3serverFanoutTests
    3_fanout_test.go:106: Bucket created successfully
--- PASS: TestZs3serverFanoutTests (33.44s)
=== RUN   TestZs3serverListTests
    4_listing_purge_test.go:102: Bucket created successfully
=== RUN   TestZs3serverListTests/Warp_List_Benchmark
    4_listing_purge_test.go:105: Test case [Warp List Benchmark] scheduled at [04-25-2026 02:12:13] 
    system_test_framework.go:148: Test case [Warp List Benchmark] start at [04-25-2026 02:12:13] 
    4_listing_purge_test.go:105: Test case [Warp List Benchmark] exit at [04-25-2026 02:12:52]
=== RUN   TestZs3serverListTests/Warp_List_Benchmark_Concurrent
    4_listing_purge_test.go:133: Test case [Warp List Benchmark Concurrent] scheduled at [04-25-2026 02:12:52] 
    system_test_framework.go:148: Test case [Warp List Benchmark Concurrent] start at [04-25-2026 02:12:52] 
    4_listing_purge_test.go:133: Test case [Warp List Benchmark Concurrent] exit at [04-25-2026 02:13:25]
--- PASS: TestZs3serverListTests (72.33s)
    --- PASS: TestZs3serverListTests/Warp_List_Benchmark (38.71s)
    --- PASS: TestZs3serverListTests/Warp_List_Benchmark_Concurrent (33.17s)
=== RUN   TestZs3serverDualAccess
=== RUN   TestZs3serverDualAccess/S3_PUT_then_NFS_READ_small_file
    5_dual_access_test.go:79: Test case [S3_PUT_then_NFS_READ_small_file] scheduled at [04-25-2026 02:13:27] 
    system_test_framework.go:148: Test case [S3_PUT_then_NFS_READ_small_file] start at [04-25-2026 02:13:27] 
=== NAME  TestZs3serverDualAccess
    5_dual_access_test.go:92: NFS read failed: open /mnt/zus_nfs/dual-test-1777076005/s3-to-nfs.txt: no such file or directory
=== NAME  TestZs3serverDualAccess/S3_PUT_then_NFS_READ_small_file
    5_dual_access_test.go:79: Test case [S3_PUT_then_NFS_READ_small_file] exit at [04-25-2026 02:13:30]
=== RUN   TestZs3serverDualAccess/NFS_WRITE_then_S3_READ_small_file
    5_dual_access_test.go:101: Test case [NFS_WRITE_then_S3_READ_small_file] scheduled at [04-25-2026 02:13:30] 
    system_test_framework.go:148: Test case [NFS_WRITE_then_S3_READ_small_file] start at [04-25-2026 02:13:30] 
=== NAME  TestZs3serverDualAccess
    5_dual_access_test.go:105: NFS write failed: open /mnt/zus_nfs/dual-test-1777076005/nfs-to-s3.txt: no such file or directory
=== NAME  TestZs3serverDualAccess/NFS_WRITE_then_S3_READ_small_file
    5_dual_access_test.go:101: Test case [NFS_WRITE_then_S3_READ_small_file] exit at [04-25-2026 02:13:30]
=== RUN   TestZs3serverDualAccess/S3_PUT_100KB_then_NFS_READ_checksum
    5_dual_access_test.go:121: Test case [S3_PUT_100KB_then_NFS_READ_checksum] scheduled at [04-25-2026 02:13:30] 
    system_test_framework.go:148: Test case [S3_PUT_100KB_then_NFS_READ_checksum] start at [04-25-2026 02:13:30] 
=== NAME  TestZs3serverDualAccess
    5_dual_access_test.go:132: open /mnt/zus_nfs/dual-test-1777076005/medium-s3.bin for MD5: open /mnt/zus_nfs/dual-test-1777076005/medium-s3.bin: no such file or directory
=== NAME  TestZs3serverDualAccess/S3_PUT_100KB_then_NFS_READ_checksum
    5_dual_access_test.go:121: Test case [S3_PUT_100KB_then_NFS_READ_checksum] exit at [04-25-2026 02:13:33]
=== RUN   TestZs3serverDualAccess/NFS_WRITE_100KB_then_S3_READ_checksum
    5_dual_access_test.go:140: Test case [NFS_WRITE_100KB_then_S3_READ_checksum] scheduled at [04-25-2026 02:13:33] 
    system_test_framework.go:148: Test case [NFS_WRITE_100KB_then_S3_READ_checksum] start at [04-25-2026 02:13:33] 
=== NAME  TestZs3serverDualAccess
    5_dual_access_test.go:150: NFS write 100KB failed: open /mnt/zus_nfs/dual-test-1777076005/medium-nfs.bin: no such file or directory
=== NAME  TestZs3serverDualAccess/NFS_WRITE_100KB_then_S3_READ_checksum
    5_dual_access_test.go:140: Test case [NFS_WRITE_100KB_then_S3_READ_checksum] exit at [04-25-2026 02:13:33]
=== RUN   TestZs3serverDualAccess/S3_LIST_sees_NFS_written_files
    5_dual_access_test.go:168: Test case [S3_LIST_sees_NFS_written_files] scheduled at [04-25-2026 02:13:33] 
    system_test_framework.go:148: Test case [S3_LIST_sees_NFS_written_files] start at [04-25-2026 02:13:33] 
=== NAME  TestZs3serverDualAccess
    5_dual_access_test.go:175: S3 LIST does not see NFS-written file nfs-to-s3.txt. Listing:
        [2026-04-25 02:13:31 CEST] 100KiB STANDARD medium-s3.bin
        [2026-04-25 02:13:28 CEST]    29B STANDARD s3-to-nfs.txt
=== NAME  TestZs3serverDualAccess/S3_LIST_sees_NFS_written_files
    5_dual_access_test.go:168: Test case [S3_LIST_sees_NFS_written_files] exit at [04-25-2026 02:13:33]
=== RUN   TestZs3serverDualAccess/NFS_LIST_sees_S3_written_files
    5_dual_access_test.go:184: Test case [NFS_LIST_sees_S3_written_files] scheduled at [04-25-2026 02:13:33] 
    system_test_framework.go:148: Test case [NFS_LIST_sees_S3_written_files] start at [04-25-2026 02:13:33] 
=== NAME  TestZs3serverDualAccess
    5_dual_access_test.go:187: NFS readdir failed: open /mnt/zus_nfs/dual-test-1777076005: no such file or directory
=== NAME  TestZs3serverDualAccess/NFS_LIST_sees_S3_written_files
    5_dual_access_test.go:184: Test case [NFS_LIST_sees_S3_written_files] exit at [04-25-2026 02:13:33]
=== RUN   TestZs3serverDualAccess/S3_DELETE_then_NFS_verify_gone
    5_dual_access_test.go:204: Test case [S3_DELETE_then_NFS_verify_gone] scheduled at [04-25-2026 02:13:33] 
    system_test_framework.go:148: Test case [S3_DELETE_then_NFS_verify_gone] start at [04-25-2026 02:13:33] 
    5_dual_access_test.go:214: PASS: S3 DELETE -> NFS file gone
    5_dual_access_test.go:204: Test case [S3_DELETE_then_NFS_verify_gone] exit at [04-25-2026 02:13:35]
=== RUN   TestZs3serverDualAccess/NFS_DELETE_then_S3_verify_gone
    5_dual_access_test.go:218: Test case [NFS_DELETE_then_S3_verify_gone] scheduled at [04-25-2026 02:13:35] 
    system_test_framework.go:148: Test case [NFS_DELETE_then_S3_verify_gone] start at [04-25-2026 02:13:35] 
=== NAME  TestZs3serverDualAccess
    5_dual_access_test.go:220: NFS DELETE failed: remove /mnt/zus_nfs/dual-test-1777076005/nfs-to-s3.txt: no such file or directory
=== NAME  TestZs3serverDualAccess/NFS_DELETE_then_S3_verify_gone
    5_dual_access_test.go:218: Test case [NFS_DELETE_then_S3_verify_gone] exit at [04-25-2026 02:13:35]
=== RUN   TestZs3serverDualAccess/S3_overwrite_then_NFS_sees_new_content
    5_dual_access_test.go:233: Test case [S3_overwrite_then_NFS_sees_new_content] scheduled at [04-25-2026 02:13:35] 
    system_test_framework.go:148: Test case [S3_overwrite_then_NFS_sees_new_content] start at [04-25-2026 02:13:35] 
=== NAME  TestZs3serverDualAccess
    5_dual_access_test.go:246: NFS read overwrite.txt failed: open /mnt/zus_nfs/dual-test-1777076005/overwrite.txt: no such file or directory
=== NAME  TestZs3serverDualAccess/S3_overwrite_then_NFS_sees_new_content
    5_dual_access_test.go:233: Test case [S3_overwrite_then_NFS_sees_new_content] exit at [04-25-2026 02:13:40]
=== RUN   TestZs3serverDualAccess/NFS_overwrite_then_S3_sees_new_content
    5_dual_access_test.go:255: Test case [NFS_overwrite_then_S3_sees_new_content] scheduled at [04-25-2026 02:13:40] 
    system_test_framework.go:148: Test case [NFS_overwrite_then_S3_sees_new_content] start at [04-25-2026 02:13:40] 
    utils.go:133: Command failed on final attempt [1/1] due to error [exit status 1]. Command String: [] Output: [mc: <ERROR> Unable to read from `zs3dual/dual-test-1777076005/nfs-overwrite.txt`. Object does not exist.]
    utils.go:136: The verbose output for the command is:
    utils.go:140: mc: <ERROR> Unable to read from `zs3dual/dual-test-1777076005/nfs-overwrite.txt`. Object does not exist.
=== NAME  TestZs3serverDualAccess
    5_dual_access_test.go:264: S3 GET nfs-overwrite.txt failed: exit status 1
=== NAME  TestZs3serverDualAccess/NFS_overwrite_then_S3_sees_new_content
    5_dual_access_test.go:255: Test case [NFS_overwrite_then_S3_sees_new_content] exit at [04-25-2026 02:13:44]
--- FAIL: TestZs3serverDualAccess (18.81s)
    --- PASS: TestZs3serverDualAccess/S3_PUT_then_NFS_READ_small_file (2.59s)
    --- PASS: TestZs3serverDualAccess/NFS_WRITE_then_S3_READ_small_file (0.00s)
    --- PASS: TestZs3serverDualAccess/S3_PUT_100KB_then_NFS_READ_checksum (2.60s)
    --- PASS: TestZs3serverDualAccess/NFS_WRITE_100KB_then_S3_READ_checksum (0.00s)
    --- PASS: TestZs3serverDualAccess/S3_LIST_sees_NFS_written_files (0.04s)
    --- PASS: TestZs3serverDualAccess/NFS_LIST_sees_S3_written_files (0.00s)
    --- PASS: TestZs3serverDualAccess/S3_DELETE_then_NFS_verify_gone (2.07s)
    --- PASS: TestZs3serverDualAccess/NFS_DELETE_then_S3_verify_gone (0.00s)
    --- PASS: TestZs3serverDualAccess/S3_overwrite_then_NFS_sees_new_content (5.17s)
    --- PASS: TestZs3serverDualAccess/NFS_overwrite_then_S3_sees_new_content (4.08s)
=== RUN   TestZs3serverFallbackS3
    6_fallback_s3_test.go:62: ZS3_ALLOC_ID env var not set
--- SKIP: TestZs3serverFallbackS3 (0.00s)
=== RUN   TestZs3serverMLPerfThroughput
=== RUN   TestZs3serverMLPerfThroughput/MLPerf_resnet
    7_mlperf_throughput_test.go:70: Test case [MLPerf resnet] scheduled at [04-25-2026 02:13:44] 
    system_test_framework.go:148: Test case [MLPerf resnet] start at [04-25-2026 02:13:44] 
    7_mlperf_throughput_test.go:75: generating 5000 objects
    7_mlperf_throughput_test.go:77: PUT: 14.0 obj/s 2.4 MB/s p50=1134.3ms p95=1179.1ms err=0
    7_mlperf_throughput_test.go:91: GET w=1: 64.1 obj/s 10.9 MB/s p50=14.2ms p95=21.3ms err=0  AU=34.6%
    7_mlperf_throughput_test.go:91: GET w=4: 128.9 obj/s 21.9 MB/s p50=29.2ms p95=40.6ms err=0  AU=51.5%
    7_mlperf_throughput_test.go:91: GET w=8: 165.6 obj/s 28.1 MB/s p50=45.8ms p95=63.4ms err=0  AU=57.7%
    7_mlperf_throughput_test.go:91: GET w=16: 205.0 obj/s 34.8 MB/s p50=74.6ms p95=104.2ms err=0  AU=62.8%
    7_mlperf_throughput_test.go:70: Test case [MLPerf resnet] exit at [04-25-2026 02:22:56]
=== RUN   TestZs3serverMLPerfThroughput/MLPerf_bert
    7_mlperf_throughput_test.go:70: Test case [MLPerf bert] scheduled at [04-25-2026 02:22:56] 
    system_test_framework.go:148: Test case [MLPerf bert] start at [04-25-2026 02:22:56] 
    7_mlperf_throughput_test.go:75: generating 30 objects
    7_mlperf_throughput_test.go:77: PUT: 5.4 obj/s 158.1 MB/s p50=2815.8ms p95=2873.0ms err=0
    7_mlperf_throughput_test.go:91: GET w=1: 7.2 obj/s 209.5 MB/s p50=46.0ms p95=179.4ms err=0  AU=40.3%
    7_mlperf_throughput_test.go:91: GET w=4: 9.1 obj/s 267.4 MB/s p50=149.0ms p95=465.6ms err=0  AU=46.3%
    7_mlperf_throughput_test.go:91: GET w=8: 19.8 obj/s 579.4 MB/s p50=188.1ms p95=242.3ms err=0  AU=65.1%
    7_mlperf_throughput_test.go:91: GET w=16: 22.0 obj/s 644.4 MB/s p50=328.9ms p95=548.7ms err=0  AU=67.5%
    7_mlperf_throughput_test.go:70: Test case [MLPerf bert] exit at [04-25-2026 02:23:15]
=== RUN   TestZs3serverMLPerfThroughput/MLPerf_cosmoflow
    7_mlperf_throughput_test.go:70: Test case [MLPerf cosmoflow] scheduled at [04-25-2026 02:23:15] 
    system_test_framework.go:148: Test case [MLPerf cosmoflow] start at [04-25-2026 02:23:15] 
    7_mlperf_throughput_test.go:75: generating 20 objects
    7_mlperf_throughput_test.go:77: PUT: 0.9 obj/s 108.9 MB/s p50=15921.4ms p95=15924.4ms err=0
    7_mlperf_throughput_test.go:91: GET w=1: 1.9 obj/s 243.5 MB/s p50=48.7ms p95=367.1ms err=0  AU=7.9%
    7_mlperf_throughput_test.go:91: GET w=4: 4.6 obj/s 592.3 MB/s p50=181.5ms p95=458.6ms err=0  AU=17.2%
    7_mlperf_throughput_test.go:91: GET w=8: 4.8 obj/s 619.7 MB/s p50=255.4ms p95=668.3ms err=0  AU=17.9%
    7_mlperf_throughput_test.go:91: GET w=16: 5.6 obj/s 718.2 MB/s p50=405.4ms p95=769.9ms err=0  AU=20.2%
    7_mlperf_throughput_test.go:70: Test case [MLPerf cosmoflow] exit at [04-25-2026 02:24:10]
=== NAME  TestZs3serverMLPerfThroughput
    7_mlperf_throughput_test.go:109: MLPerf results written to /tmp/zs3_mlperf_1777076650.json
--- PASS: TestZs3serverMLPerfThroughput (626.05s)
    --- PASS: TestZs3serverMLPerfThroughput/MLPerf_resnet (552.31s)
    --- PASS: TestZs3serverMLPerfThroughput/MLPerf_bert (19.05s)
    --- PASS: TestZs3serverMLPerfThroughput/MLPerf_cosmoflow (54.66s)
=== RUN   TestZs3serverPorcupineLinearizability
=== RUN   TestZs3serverPorcupineLinearizability/s3/zs3server
    8_porcupine_linearizability_test.go:60: Test case [s3/zs3server] scheduled at [04-25-2026 02:24:10] 
    system_test_framework.go:148: Test case [s3/zs3server] start at [04-25-2026 02:24:10] 
    8_porcupine_linearizability_test.go:285: Collected 4000 ops in 301.688s (errors=0) throughput=13.3 ops/s
    8_porcupine_linearizability_test.go:295: RESULT: UNKNOWN (check timeout 1m0s) — tolerated
    8_porcupine_linearizability_test.go:60: Test case [s3/zs3server] exit at [04-25-2026 02:30:13]
--- PASS: TestZs3serverPorcupineLinearizability (362.51s)
    --- PASS: TestZs3serverPorcupineLinearizability/s3/zs3server (362.51s)
=== RUN   TestZs3serverLlama3Checkpoint
=== RUN   TestZs3serverLlama3Checkpoint/Llama3_ckpt_4_×_1.50_GB_rotate=false
    9_llama3_checkpoint_test.go:55: Test case [Llama3 ckpt 4 × 1.50 GB rotate=false] scheduled at [04-25-2026 02:30:13] 
    system_test_framework.go:148: Test case [Llama3 ckpt 4 × 1.50 GB rotate=false] start at [04-25-2026 02:30:13] 
    9_llama3_checkpoint_test.go:75: ckpt_00: write 140.5 MB/s (10.93s)
    9_llama3_checkpoint_test.go:75: ckpt_01: write 113.6 MB/s (13.52s)
    9_llama3_checkpoint_test.go:75: ckpt_02: write 85.5 MB/s (17.96s)
    9_llama3_checkpoint_test.go:75: ckpt_03: write 102.1 MB/s (15.05s)
    9_llama3_checkpoint_test.go:87: READ w=1: 14.5 MB/s p50=63.0ms p95=288.1ms
    9_llama3_checkpoint_test.go:87: READ w=2: 65.7 MB/s p50=60.7ms p95=73.8ms
    9_llama3_checkpoint_test.go:87: READ w=4: 63.0 MB/s p50=148.7ms p95=167.6ms
    9_llama3_checkpoint_test.go:97: Llama3 peak write: 140.5 MB/s
    9_llama3_checkpoint_test.go:55: Test case [Llama3 ckpt 4 × 1.50 GB rotate=false] exit at [04-25-2026 02:41:31]
--- PASS: TestZs3serverLlama3Checkpoint (678.07s)
    --- PASS: TestZs3serverLlama3Checkpoint/Llama3_ckpt_4_×_1.50_GB_rotate=false (678.04s)
FAIL
FAIL	github.com/0chain/system_test/tests/cli_tests/zs3server_tests	2721.352s
FAIL