What is the purpose of the "lib" folder in this repository?

[RelayZ]

Thank you for sharing this impressive work and for making it publicly available! I’ve been exploring this repository, which I understand is a SAM2-based object tracking framework. While going through the code, I came across a folder named lib located in the root directory. I was wondering if you could kindly clarify the primary purpose of the lib folder, particularly in relation to ARTrack.

Additionally, I am interested in evaluating the framework on GOT-10k, TrackingNet, NFS, OTB, etc. Would it be possible for you to provide the evaluation code or any relevant guidelines for performing evaluations on these datasets?

[LouisFinner]

Same question here 🙌 I'm looking for evaluation code to reproduce the results presented in the paper. It seems that the 'lib' is used to generate the tracking evaluation plot shown in the paper, but I can't locate the entry function. I kindly hope the authors could clarify more about this 😊

[yangchris11]

@RelayZ, @LouisFinner Please use the script for your reference, this is a (very messy) script that we used to evaluate and plot the results need for paper. We also cross-verify the raw result files from other works to ensure our re-implementation is correct.

Due to limited bandwidth on our side, please adapt the script for your usage on other dataset. One other good source to look in to will be the evaluation package by VastTrack.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os.path as osp
import sys

import matplotlib as mpl
import matplotlib.pyplot as plt


def add_path(path):
    if path not in sys.path:
        sys.path.insert(0, path)

this_dir = osp.dirname(__file__)

prj_path = osp.join(this_dir, '..')
add_path(prj_path)

from lib.test.analysis.plot_results import (plot_results, print_results, 
                                             print_per_sequence_results, 
                                             print_per_attribute_results, 
                                             plot_per_attribute_results)
from lib.test.evaluation import get_dataset, trackerlist


mpl.rcParams.update(mpl.rcParamsDefault)
plt.rcParams['figure.figsize'] = [8, 8]


class VOTEval():
    def __init__(self):
        # you'll need to modify the local.py in vot-toolkit/lib/test/evaluation to run this script
        
        self.vot_results_path = {
            "DiMP":{
                # https://arxiv.org/pdf/1904.07220
                "folder_name": "DiMP",
                "DiMP":{
                    "LaSOT": "DiMP50",
                    "LaSOT_ext": "DiMP50"
                }
            },
            # add the raw results from other papers or repos here
            "OSTrack":{
                "folder_name": "OSTrack",
                "OSTrack-B":{
                    "LaSOT": "vitb_384_mae_ce_32x4_ep300/lasot",
                    "LaSOT_ext": "vitb_384_mae_ce_32x4_ep300/lasot_ext"
                }
            },
            "ROMTrack":{
                # Robust Object Modeling for Visual Tracking (https://arxiv.org/abs/2308.05140
                "folder_name": "ROMTrack",
                "ROMTrack-B":{
                    "LaSOT": "ROMTrack-Base/ROMTrack-256/LaSOT_LaSOText/baseline",
                    "LaSOT_ext": "ROMTrack-Base/ROMTrack-256/LaSOT_LaSOText/baseline"
                }
            },
            "TaMOs":{
                # Beyond SOT: Tracking Multiple Generic Objects at Once (https://arxiv.org/abs/2212.11920)
                "folder_name": "TaMOs",
                "TaMOs-B":{
                    "LaSOT": "tamos_swin_base"
                }
            },
            "EVPTrack":{
                "folder_name": "EVPTrack",
                "EVPTrack-B":{
                    "LaSOT": "EVPTrack-full-384",
                    "LaSOT_ext": "EVPTrack-full-384"
                }
            },
            "ODTrack":{
                "folder_name": "ODTrack",
                "ODTrack-B":{
                    "LaSOT": "ODTrack-B/lasot",
                    "LaSOT_ext": "ODTrack-B/lasot_extension_subset"
                },
                "ODTrack-L":{
                    "LaSOT": "ODTrack-L/lasot",
                    "LaSOT_ext": "ODTrack-L/lasot_extension_subset"
                }
            },
            "HIPTrack":{
                "folder_name": "HIPTrack",
                "HIPTrack-B":{
                    "LaSOT": "hiptrack_lasot+lasotext+otb2015+uav123",
                    "LaSOT_ext": "hiptrack_lasot+lasotext+otb2015+uav123"
                }
            },
            "AQATrack":{
                # https://arxiv.org/abs/2403.10574
                "folder_name": "AQATrack",
                "AQATrack-B":{
                    "LaSOT": "full-256/lasot",
                    "LaSOT_ext": "full-256/lasot_extension_subset"
                },
                "AQATrack-L":{
                    "LaSOT": "full-384/lasot",
                    "LaSOT_ext": "full-384/lasot_extension_subset"
                }
            },
            "LoRAT":{
                # https://arxiv.org/abs/2403.05231
                "folder_name": "LoRAT",
                "LoRAT-B":{
                    "LaSOT": "base/DINOv2-B-224",
                    "LaSOT_ext": "base/DINOv2-B-224_001"
                },
                "LoRAT-L":{
                    "LaSOT": "large/DINOv2-L-224",
                    "LaSOT_ext": "large/DINOv2-L-224_001"
                }
            },
            "SAM2.1":{
                "folder_name": "sam2",
                "SAM2.1-T":{
                    "LaSOT": "sam2_tiny",
                    "LaSOT_ext": "sam2_tiny"
                },
                "SAM2.1-S":{
                    "LaSOT": "sam2_small",
                    "LaSOT_ext": "sam2_small"
                },
                "SAM2.1-B":{
                    "LaSOT": "sam2_base_plus",
                    "LaSOT_ext": "sam2_base_plus"
                },
                "SAM2.1-L":{
                    "LaSOT": "sam2_large",
                    "LaSOT_ext": "sam2_large"
                }
            },
            "SAMURAI":{
                "folder_name": "samurai",
                "SAMURAI-T":{
                    "LaSOT": "samurai_tiny",    
                    "LaSOT_ext": "samurai_tiny"
                },
                "SAMURAI-S":{
                    "LaSOT": "samurai_small",
                    "LaSOT_ext": "samurai_small"
                },
                "SAMURAI-B":{
                    "LaSOT": "samurai_base_plus",
                    "LaSOT_ext": "samurai_base_plus"
                },
                "SAMURAI-L":{
                    "LaSOT": "samurai_large",
                    "LaSOT_ext": "samurai_large"
                }
            }
        }

    def get_tracker_folder(self, tracker_name, tracker_variant, dataset_name):
        assert tracker_name in self.vot_results_path, f"Tracker {tracker_name} results are not available."
        assert tracker_variant  in self.vot_results_path[tracker_name], f"Tracker variant {tracker_variant} is not available for tracker {tracker_name}."
        assert dataset_name in self.vot_results_path[tracker_name][tracker_variant], f"Dataset {dataset_name} is not available for tracker {tracker_name} variant {tracker_variant}."

        return self.vot_results_path[tracker_name]["folder_name"], self.vot_results_path[tracker_name][tracker_variant][dataset_name]

    def evaluate(self, dataset_name="LaSOT"):
        trackers = []

        full_tracker_to_evaluate = {
            "LaSOT": {
                # "DiMP": ["DiMP"],
                # "ROMTrack": ["ROMTrack-B"],
                # "TaMOs": ["TaMOs-B"],
                # "EVPTrack": ["EVPTrack-B"],
                # "ODTrack": ["ODTrack-B", "ODTrack-L"],
                # "HIPTrack": ["HIPTrack-B"],
                # "AQATrack": ["AQATrack-B", "AQATrack-L"],
                # "LoRAT": ["LoRAT-B", "LoRAT-L"],
                "SAM2.1": ["SAM2.1-T", "SAM2.1-S", "SAM2.1-B", "SAM2.1-L"],
                "SAMURAI": ["SAMURAI-T", "SAMURAI-S", "SAMURAI-B", "SAMURAI-L"],
                # "DiMP": ["DiMP"],
                # "ROMTrack": ["ROMTrack-B"],
                # "TaMOs": ["TaMOs-B"],
                # "EVPTrack": ["EVPTrack-B"],
                # "ODTrack": ["ODTrack-L"],
                # "HIPTrack": ["HIPTrack-B"],
                # "AQATrack": ["AQATrack-L"],
                # "LoRAT": ["LoRAT-L"],
                # "SAM2.1": ["SAM2.1-L", "SAM2.1-B"],
                # "SAMURAI": ["SAMURAI-L", "SAMURAI-B"],
            },
            "LaSOT_ext": {
                # "ROMTrack": ["ROMTrack-B"],
                # "EVPTrack": ["EVPTrack-B"],
                # "ODTrack": ["ODTrack-B", "ODTrack-L"],
                # "HIPTrack": ["HIPTrack-B"],
                # "AQATrack": ["AQATrack-B", "AQATrack-L"],
                # "LoRAT": ["LoRAT-B", "LoRAT-L"],
                "SAM2.1": ["SAM2.1-T", "SAM2.1-S", "SAM2.1-B", "SAM2.1-L"],
                "SAMURAI": ["SAMURAI-T", "SAMURAI-S", "SAMURAI-B", "SAMURAI-L"],
                # "OSTrack": ["OSTrack-B"],
                # "EVPTrack": ["EVPTrack-B"],
                # "ODTrack": ["ODTrack-L"],
                # "HIPTrack": ["HIPTrack-B"],
                # "AQATrack": ["AQATrack-L"],
                # "LoRAT": ["LoRAT-L"],
                # "SAM2.1": ["SAM2.1-L", "SAM2.1-B"],
                # "SAMURAI": ["SAMURAI-L", "SAMURAI-B"],
            }
        }
        tracker_to_evaluate = full_tracker_to_evaluate[dataset_name]

        for tracker_name, tracker_variants in tracker_to_evaluate.items():
            for tracker_variant in tracker_variants:
                tracker_folder, tracker_sub_folder = self.get_tracker_folder(tracker_name, tracker_variant, dataset_name)
                trackers.extend(trackerlist(tracker_folder, tracker_sub_folder, None, None, tracker_variant))

        if dataset_name == "LaSOT":
            dataset = get_dataset('lasot')
        elif dataset_name == "LaSOT_ext":
            dataset = get_dataset('lasot_extension_subset')
        dataset_name = "LaSOT$_{ext}$"

        print_results(trackers, dataset, dataset_name, merge_results=True, plot_types=('success', 'prec', 'norm_prec'))

            
def main():
    vot_eval = VOTEval()

    vot_eval.evaluate(dataset_name="LaSOT")
    vot_eval.evaluate(dataset_name="LaSOT_ext")

if __name__ == '__main__':
    main()

[yangchris11]

@RelayZ We are reporting the VOT result on the testing split for GOT-10k and TrackingNet as both our them held their private leaderboard. Please refer to their official website and submit your result.