{ "test_id": "TEST 3.5 / SAIL-169", "name": "sam3-cldice-loss", "created_at": "2026-09-07T14:11:42.884982+00:00", "status": "blocked_gpu_access_cpu_diagnostics_complete", "scope": "conditional / measurement only", "product_question": "Does a centerline-aware loss help SailScan finish faint stripes?", "bottom_line": "Unknown: clDice has not been shown to reduce truncated faint-stripe ends. The improvement is unmeasured, not zero. Any benefit is gated on SAM3 staying in production; that decision is undecided.", "product_verdict": { "cuts_truncated_ends": null, "delta_percentage_points": null, "relative_reduction_percent": null, "sam3_production_decision": "undecided" }, "holdout": { "id_file": "server/inference/test-holdout-ids-canonical.json", "freeze_date": "2026-08-25", "count": 332, "sha256": "a49d9139d6eac54951c4c3e257770855192a75f8c941799ef4141c49a4c11243" }, "execution": { "gpu_training_ran": false, "sam3_inference_ran": false, "paired_holdout_photos_evaluated": 0, "trained_candidate_checkpoints": 0, "pod_created": false, "weights_promoted": false, "deployed": false, "gpu_access_checks": [ { "endpoint": "GET https://rest.runpod.io/v1/pods", "credential_source": "inherited environment", "http_status": 403 }, { "endpoint": "GET https://rest.runpod.io/v1/pods", "credential_source": "~/.config/keys/keys.env (memory-documented source)", "http_status": 403 } ], "not_run": [ "Meta SAM3 integration on GPU", "baseline-matched retraining with clDice", "candidate inference on all 332", "paired faint-end truncation comparison", "full product pipeline geometry/metric regression evaluation" ] }, "baseline_inventory_only": { "path": "server/inference/runs/v27-eval/frozen300-sam3ft-all-lines.json", "sha256": "d5c3c396470a1d9940dd54e4bafc51b9d97a3d2e24bb0de6adf764e20dcf2db9", "cached_holdout_photos": 300, "missing_holdout_photos": 32, "missing_ids": [ "0236f5a7-9533-4c32-8304-0682421de226", "050d4686-1302-489a-add5-a8eae2972140", "052f1898-16db-400b-9365-54468834a80d", "0f6e6446-8fb8-455d-bfbe-de26f09eefd2", "113271a7-6855-4665-a2ad-e0536328066c", "13f2a53e-ccbb-473f-b5a6-8a49e84f7426", "1471afa4-7389-4ea5-9e61-50d01b8f921a", "28fff923-fad0-4a45-a71e-b8616c1081ed", "2f9ff5ea-dcca-4b96-8e2c-124a2d8e1403", "3393ed66-2f9e-43b1-91cf-a96d30b1904d", "3a951997-0e8f-4988-a03b-629c32423f0d", "4529715b-88c6-462c-b31e-875dc150c9f8", "564c75e1-f65c-4a8b-ae98-ec19a0fc951d", "56e036ce-5c31-4aa6-8140-9c255f0e1060", "5e214a18-26bc-452b-9c87-5e783eaf0188", "618f2d8b-6489-48fc-b168-ec657df98db3", "658ba7e5-61b1-47c1-a292-d130b6469e52", "6bd96ae9-cf40-4164-aeed-eca33fb39d31", "6cb1b894-a716-4a8b-b43d-3eec1868df7e", "73065657-0305-4c2a-878c-cdd061343c31", "788356aa-7d0d-4523-a062-b6727b213788", "903b76d3-792f-4f39-ae12-83c1058606db", "9ff8f32e-3267-4ce1-837b-834b7e7250ef", "a633a49b-144d-4bda-bf46-ace03a838b73", "ab5eabe9-8d95-439f-af29-c8239f9cfa58", "b89aa759-62ff-4fd0-85fb-9483fa7651e0", "c6dd23db-fa58-4572-9142-cdd293dc0097", "d075a28b-1ca0-4751-a8c5-94f70cbd1b29", "d0ee29d7-6549-476b-a7e1-865bb284b9b5", "d8807b94-6b3d-45a8-8e58-6e6233506880", "f234ddf1-6c71-4739-a668-a910242f1225", "faea1f97-5d33-4e44-b3b0-d46e90b0164a" ], "reference_stripes": 981, "raw_prediction_lines": 2292, "faint_end_annotation_present": false, "limitation": "Archived cache inventory only; no checkpoint hash/confidence provenance attached. Not treated as a verified current-baseline quality score. Raw line count is not recall or precision." }, "cpu_diagnostics": { "device": "cpu", "torch": "2.14.0", "python": "3.14.7", "synthetic_masks_only": true, "mask_shape": [ 1, 1, 32, 64 ], "skeleton_iterations": 10, "tests_passed": 7, "test_output": ".......\n----------------------------------------------------------------------\nRan 7 tests in 0.188s\n\nOK", "cases": [ { "case": "complete", "missing_foreground_pixels": 0, "soft_cldice_loss": 0.0, "skeleton_recall_loss": 0.0, "unweighted_foreground_dice_loss": 0.0 }, { "case": "50_missing_edge_pixels", "missing_foreground_pixels": 50, "soft_cldice_loss": 0.0, "skeleton_recall_loss": 0.0, "unweighted_foreground_dice_loss": 0.11627906560897827 }, { "case": "50_missing_end_pixels", "missing_foreground_pixels": 50, "soft_cldice_loss": 0.09999996423721313, "skeleton_recall_loss": 0.1818181872367859, "unweighted_foreground_dice_loss": 0.11627906560897827 }, { "case": "100_missing_two_end_pixels", "missing_foreground_pixels": 100, "soft_cldice_loss": 0.2222222089767456, "skeleton_recall_loss": 0.363636314868927, "unweighted_foreground_dice_loss": 0.2631579041481018 }, { "case": "entire_stripe_missing", "missing_foreground_pixels": 240, "soft_cldice_loss": 0.9999999403953552, "skeleton_recall_loss": 1.0, "unweighted_foreground_dice_loss": 1.0 } ], "interpretation": "Same-area end removal incurs more centerline loss than edge removal. This checks the mechanism and gradients; it does not measure real image performance." }, "implementation": { "isolated_entry_point": "server/inference/sam3-finetune/cldice_train.py", "objective": "unchanged baseline focal/Dice/box/presence losses + 10 * soft-clDice", "adapter_status": "CPU loss and config generation checked; actual Meta SAM3 adapter execution unrun", "default_lambda": 10.0, "lambda_selection": "provisional, must be fixed using non-holdout dev data before frozen evaluation", "frozen_holdout_guard": "required --holdout-ids, exact canonical bytes, reject frozen IDs and derivative crop names in COCO pool" }, "required_gpu_protocol": [ "Use the actual current Meta SAM3 checkpoint/recipe and record its SHA256 and upstream commit; never substitute base SAM3 or SAM2.", "Provision a RunPod GPU. Use a dedicated per-run venv; pin ultralytics==8.4.84 even if unused by Meta training. Record all dependency versions.", "Build training masks with the canonical exclusion file, then run cldice_train.py with --holdout-ids. Dereference dataset symlinks before creating a tarball.", "Use disjoint non-holdout development data to choose loss weight/iterations. Match data, initialization, seed, updates, and augmentations across loss-off/loss-on training arms; compare to the current checkpoint too.", "Confirm mask loss attachment, checkpoint reload, and gradients on GPU before the complete run. Never train locally or auto-promote.", "Stage all 332 photos including North images; use ImageOps.exif_transpose and the same Meta inference settings for both arms (concept sail draft stripe, confidence 0.25, resolution 1008).", "Blindly annotate faint endpoints independently of either prediction. Existing cache has no such annotations; do not infer faintness from model misses.", "Before evaluation, lock one-to-one stripe matching and an end-coverage rule on development data. Proposed endpoint event: more than 5% of reference arc length uncovered from that end, with proximity tolerance 1% of image diagonal. These are proposed, not validated thresholds.", "Count wholly missed stripes as two failed ends; report these separately from matched-stripe truncation. Report denominators, absolute percentage-point delta and photo-cluster bootstrap 95% CI.", "Report raw-mask endpoint behavior and product-pipeline results separately; apply the same production geometry rules to both arms, and check stripe count, point error, twist/angles, with camber/draft non-regression. No deployment." ], "sources": [ "https://github.com/jocpae/clDice", "https://openaccess.thecvf.com/content/CVPR2021/papers/Shit_clDice_-_A_Novel_Topology-Preserving_Loss_Function_for_Tubular_Structure_CVPR_2021_paper.pdf", "https://github.com/facebookresearch/sam3/blob/main/sam3/train/loss/loss_fns.py" ], "code_sha256": { "cldice_loss.py": "1b87140a7a217ba53c0d0582ec85827fc4c7b4960cc8c31b88a6bae5f62527ce", "cldice_sam3_adapter.py": "4e7f4e19e15fcd782788d2fbd5fe6220758bb2f7d1ad97c981a6b114d4b8dac5", "cldice_train.py": "8904228c5e2b4b5ca8bf52df9f2671242ddd5b15b40ef40c68f1b5dd8da6c8b7", "test_cldice_loss.py": "70ff4c4db6dff344e980f48664cafbe4172e79e9a4f73d9c2fdd13dba2984c1f" } }