boringKey commited on
Commit
99da2fc
·
verified ·
1 Parent(s): 70af8f8

Delete DFA-MoE

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. DFA-MoE/MTIL_datasets/__init__.py +0 -4
  2. DFA-MoE/MTIL_datasets/caltech101.py +0 -92
  3. DFA-MoE/MTIL_datasets/cifar10.py +0 -94
  4. DFA-MoE/MTIL_datasets/cifar100.py +0 -89
  5. DFA-MoE/MTIL_datasets/clevr_count.py +0 -118
  6. DFA-MoE/MTIL_datasets/country211.py +0 -386
  7. DFA-MoE/MTIL_datasets/dtd.py +0 -95
  8. DFA-MoE/MTIL_datasets/eurosat.py +0 -75
  9. DFA-MoE/MTIL_datasets/fer2013.py +0 -170
  10. DFA-MoE/MTIL_datasets/fgvc_aircraft.py +0 -73
  11. DFA-MoE/MTIL_datasets/food101.py +0 -52
  12. DFA-MoE/MTIL_datasets/gtsrb.py +0 -202
  13. DFA-MoE/MTIL_datasets/hatefulmemes.py +0 -117
  14. DFA-MoE/MTIL_datasets/imagenet_r.py +0 -231
  15. DFA-MoE/MTIL_datasets/kitti_distance.py +0 -228
  16. DFA-MoE/MTIL_datasets/mnist.py +0 -132
  17. DFA-MoE/MTIL_datasets/oxford_flowers.py +0 -89
  18. DFA-MoE/MTIL_datasets/oxford_pets.py +0 -176
  19. DFA-MoE/MTIL_datasets/pcam.py +0 -198
  20. DFA-MoE/MTIL_datasets/resisc.py +0 -199
  21. DFA-MoE/MTIL_datasets/sst2.py +0 -122
  22. DFA-MoE/MTIL_datasets/stanford_cars.py +0 -83
  23. DFA-MoE/MTIL_datasets/stl10.py +0 -143
  24. DFA-MoE/MTIL_datasets/sun397.py +0 -81
  25. DFA-MoE/MTIL_datasets/ucf101.py +0 -360
  26. DFA-MoE/MTIL_datasets/utils.py +0 -299
  27. DFA-MoE/MTIL_datasets/voc2007.py +0 -173
  28. DFA-MoE/README.md +0 -227
  29. DFA-MoE/calculate_SCR.py +0 -266
  30. DFA-MoE/calculate_sim.py +0 -456
  31. DFA-MoE/class_orders/cifar100.yaml +0 -1
  32. DFA-MoE/class_orders/imagenet100.yaml +0 -11
  33. DFA-MoE/class_orders/imagenet1000.yaml +0 -48
  34. DFA-MoE/class_orders/tinyimagenet.yaml +0 -17
  35. DFA-MoE/clip/README.md +0 -1
  36. DFA-MoE/clip/__init__.py +0 -1
  37. DFA-MoE/clip/adapter.py +0 -69
  38. DFA-MoE/clip/bpe_simple_vocab_16e6.txt.gz +0 -3
  39. DFA-MoE/clip/clip.py +0 -310
  40. DFA-MoE/clip/model.py +0 -713
  41. DFA-MoE/clip/tokenizer.py +0 -140
  42. DFA-MoE/configs/class/aircraft.yaml +0 -76
  43. DFA-MoE/configs/class/caltech.yaml +0 -79
  44. DFA-MoE/configs/class/car.yaml +0 -79
  45. DFA-MoE/configs/class/cifar10.yaml +0 -79
  46. DFA-MoE/configs/class/cifar100.yaml +0 -79
  47. DFA-MoE/configs/class/clevr.yaml +0 -79
  48. DFA-MoE/configs/class/conuntry.yaml +0 -79
  49. DFA-MoE/configs/class/dtd.yaml +0 -79
  50. DFA-MoE/configs/class/eurosat.yaml +0 -79
DFA-MoE/MTIL_datasets/__init__.py DELETED
@@ -1,4 +0,0 @@
1
- # This dataset is originally proposed by Zangwei Zheng et al. (https://arxiv.org/abs/2303.06628)
2
- # Code here is based on CoOp and its following works. (https://github.com/KaiyangZhou/CoOp)
3
- # Modified by Longxiang Tang (lloong.x@gmail.com) to release the dependence of Dassl lib.
4
- # To prepare data, please refer to https://github.com/muzairkhattak/PromptSRC/blob/main/docs/DATASETS.md
 
 
 
 
 
DFA-MoE/MTIL_datasets/caltech101.py DELETED
@@ -1,92 +0,0 @@
1
- import os
2
- import pickle
3
-
4
- from .utils import *
5
-
6
- from .oxford_pets import OxfordPets
7
- from .dtd import DescribableTextures as DTD
8
-
9
- IGNORED = ["BACKGROUND_Google", "Faces_easy"]
10
- NEW_CNAMES = {
11
- "airplanes": "airplane",
12
- "Faces": "face",
13
- "Leopards": "leopard",
14
- "Motorbikes": "motorbike",
15
- }
16
-
17
-
18
- class Caltech101(DatasetBase):
19
-
20
- dataset_dir = "caltech-101"
21
-
22
- def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'):
23
- root = os.path.abspath(os.path.expanduser(root))
24
- self.dataset_dir = os.path.join(root, self.dataset_dir)
25
- self.image_dir = os.path.join(self.dataset_dir, "101_ObjectCategories")
26
- self.split_path = os.path.join(self.dataset_dir, "split_zhou_Caltech101.json")
27
- self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot")
28
- mkdir_if_missing(self.split_fewshot_dir)
29
-
30
- if os.path.exists(self.split_path):
31
- train, val, test = OxfordPets.read_split(self.split_path, self.image_dir)
32
- else:
33
- train, val, test = DTD.read_and_split_data(self.image_dir, ignored=IGNORED, new_cnames=NEW_CNAMES)
34
- OxfordPets.save_split(train, val, test, self.split_path, self.image_dir)
35
-
36
- if num_shots >= 1:
37
- preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
38
-
39
- if os.path.exists(preprocessed):
40
- print(f"Loading preprocessed few-shot data from {preprocessed}")
41
- with open(preprocessed, "rb") as file:
42
- data = pickle.load(file)
43
- train, val = data["train"], data["val"]
44
- else:
45
- train = self.generate_fewshot_dataset(train, num_shots=num_shots)
46
- val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
47
- data = {"train": train, "val": val}
48
- print(f"Saving preprocessed few-shot data to {preprocessed}")
49
- with open(preprocessed, "wb") as file:
50
- pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)
51
-
52
- subsample = subsample_classes
53
- train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample)
54
-
55
- self.templates = [
56
- lambda c: f"a photo of a {c}.",
57
- lambda c: f"a painting of a {c}.",
58
- lambda c: f"a plastic {c}.",
59
- lambda c: f"a sculpture of a {c}.",
60
- lambda c: f"a sketch of a {c}.",
61
- lambda c: f"a tattoo of a {c}.",
62
- lambda c: f"a toy {c}.",
63
- lambda c: f"a rendition of a {c}.",
64
- lambda c: f"a embroidered {c}.",
65
- lambda c: f"a cartoon {c}.",
66
- lambda c: f"a {c} in a video game.",
67
- lambda c: f"a plushie {c}.",
68
- lambda c: f"a origami {c}.",
69
- lambda c: f"art of a {c}.",
70
- lambda c: f"graffiti of a {c}.",
71
- lambda c: f"a drawing of a {c}.",
72
- lambda c: f"a doodle of a {c}.",
73
- lambda c: f"a photo of the {c}.",
74
- lambda c: f"a painting of the {c}.",
75
- lambda c: f"the plastic {c}.",
76
- lambda c: f"a sculpture of the {c}.",
77
- lambda c: f"a sketch of the {c}.",
78
- lambda c: f"a tattoo of the {c}.",
79
- lambda c: f"the toy {c}.",
80
- lambda c: f"a rendition of the {c}.",
81
- lambda c: f"the embroidered {c}.",
82
- lambda c: f"the cartoon {c}.",
83
- lambda c: f"the {c} in a video game.",
84
- lambda c: f"the plushie {c}.",
85
- lambda c: f"the origami {c}.",
86
- lambda c: f"art of the {c}.",
87
- lambda c: f"graffiti of the {c}.",
88
- lambda c: f"a drawing of the {c}.",
89
- lambda c: f"a doodle of the {c}.",
90
- ]
91
-
92
- super().__init__(train_x=train, val=val, test=test)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/MTIL_datasets/cifar10.py DELETED
@@ -1,94 +0,0 @@
1
- import os
2
- from typing import List
3
-
4
- from .utils import * # Datum, DatasetBase
5
- from .oxford_pets import OxfordPets
6
-
7
- try:
8
- from torchvision.datasets import CIFAR10 as TorchCIFAR10
9
- except Exception as e:
10
- TorchCIFAR10 = None
11
- print(f"Warning: torchvision not available for CIFAR10: {e}")
12
-
13
-
14
- CIFAR10_CLASSES: List[str] = [
15
- 'airplane',
16
- 'automobile',
17
- 'bird',
18
- 'cat',
19
- 'deer',
20
- 'dog',
21
- 'frog',
22
- 'horse',
23
- 'ship',
24
- 'truck',
25
- ]
26
-
27
- CIFAR10_TEMPLATES: List[str] = [
28
- 'a photo of a {}.',
29
- 'a blurry photo of a {}.',
30
- 'a black and white photo of a {}.',
31
- 'a low contrast photo of a {}.',
32
- 'a high contrast photo of a {}.',
33
- 'a bad photo of a {}.',
34
- 'a good photo of a {}.',
35
- 'a photo of a small {}.',
36
- 'a photo of a big {}.',
37
- 'a photo of the {}.',
38
- 'a blurry photo of the {}.',
39
- 'a black and white photo of the {}.',
40
- 'a low contrast photo of the {}.',
41
- 'a high contrast photo of the {}.',
42
- 'a bad photo of the {}.',
43
- 'a good photo of the {}.',
44
- 'a photo of the small {}.',
45
- 'a photo of the big {}.',
46
- ]
47
-
48
-
49
- class CIFAR10(DatasetBase):
50
-
51
- dataset_dir = "cifar10"
52
-
53
- def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'):
54
- if TorchCIFAR10 is None:
55
- raise ImportError("torchvision is required for CIFAR10 dataset. Please install torchvision.")
56
-
57
- root = os.path.abspath(os.path.expanduser(root))
58
- self.dataset_dir = os.path.join(root, self.dataset_dir)
59
-
60
- # Use torchvision to download/load data to dataset_dir
61
- train_ds = TorchCIFAR10(root=self.dataset_dir, train=True, download=True)
62
- test_ds = TorchCIFAR10(root=self.dataset_dir, train=False, download=True)
63
-
64
- # Build Datum lists
65
- trainval = []
66
- # train_ds.data: numpy array HWC, train_ds.targets: list[int]
67
- for idx in range(len(train_ds.data)):
68
- img = Image.fromarray(train_ds.data[idx])
69
- label = int(train_ds.targets[idx])
70
- classname = CIFAR10_CLASSES[label]
71
- trainval.append(Datum(impath=img, label=label, classname=classname))
72
-
73
- test = []
74
- for idx in range(len(test_ds.data)):
75
- img = Image.fromarray(test_ds.data[idx])
76
- label = int(test_ds.targets[idx])
77
- classname = CIFAR10_CLASSES[label]
78
- test.append(Datum(impath=img, label=label, classname=classname))
79
-
80
- # Split train/val
81
- train, val = OxfordPets.split_trainval(trainval)
82
-
83
- # Few-shot sampling if requested
84
- if num_shots >= 1:
85
- train = self.generate_fewshot_dataset(train, num_shots=num_shots)
86
- val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
87
-
88
- # Optional class subsampling (base/new)
89
- train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample_classes)
90
-
91
- # Templates
92
- self.templates = CIFAR10_TEMPLATES
93
-
94
- super().__init__(train_x=train, val=val, test=test)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/MTIL_datasets/cifar100.py DELETED
@@ -1,89 +0,0 @@
1
- import os
2
- import pickle
3
- import numpy as np
4
-
5
- from .utils import *
6
-
7
- from .oxford_pets import OxfordPets
8
- from .dtd import DescribableTextures as DTD
9
-
10
-
11
- class CIFAR100(DatasetBase):
12
-
13
- dataset_dir = "cifar100"
14
-
15
- def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'):
16
- root = os.path.abspath(os.path.expanduser(root))
17
- self.dataset_dir = os.path.join(root, self.dataset_dir)
18
-
19
- file_path = os.path.join(root, self.dataset_dir, 'train')
20
- with open(file_path, "rb") as f:
21
- entry = pickle.load(f, encoding="latin1")
22
- trainval_data = entry["data"]
23
- if "labels" in entry:
24
- trainval_targets = entry["labels"]
25
- else:
26
- trainval_targets = entry["fine_labels"]
27
- trainval_data = trainval_data.reshape(-1, 3, 32, 32)
28
- trainval_data = trainval_data.transpose((0, 2, 3, 1))
29
-
30
- file_path = os.path.join(root, self.dataset_dir, 'test')
31
- with open(file_path, "rb") as f:
32
- entry = pickle.load(f, encoding="latin1")
33
- test_data = entry["data"]
34
- if "labels" in entry:
35
- test_targets = entry["labels"]
36
- else:
37
- test_targets = entry["fine_labels"]
38
- test_data = test_data.reshape(-1, 3, 32, 32)
39
- test_data = test_data.transpose((0, 2, 3, 1))
40
-
41
- path = os.path.join(self.dataset_dir, "meta")
42
- with open(path, "rb") as infile:
43
- data = pickle.load(infile, encoding="latin1")
44
- classes = data["fine_label_names"]
45
- classes = [s.replace("_", " ") for s in classes]
46
-
47
- trainval = []
48
- for idx in range(trainval_data.shape[0]):
49
- item = Datum(impath=Image.fromarray(trainval_data[idx]),
50
- label=int(trainval_targets[idx]), classname=classes[trainval_targets[idx]])
51
- trainval.append(item)
52
-
53
- test = []
54
- for idx in range(test_data.shape[0]):
55
- item = Datum(impath=Image.fromarray(test_data[idx]),
56
- label=int(test_targets[idx]), classname=classes[test_targets[idx]])
57
- test.append(item)
58
-
59
- train, val = OxfordPets.split_trainval(trainval)
60
-
61
- if num_shots >= 1:
62
- train = self.generate_fewshot_dataset(train, num_shots=num_shots)
63
- val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
64
-
65
- subsample = subsample_classes
66
- train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample)
67
-
68
- self.templates = [
69
- lambda c : f'a bad photo of a {c}.',
70
- lambda c : f'a blurry photo of a {c}.',
71
- lambda c : f'a black and white photo of a {c}.',
72
- lambda c : f'a low contrast photo of a {c}.',
73
- lambda c : f'a high contrast photo of a {c}.',
74
- lambda c : f'a photo of a {c}.',
75
- lambda c : f'a good photo of a {c}.',
76
- lambda c : f'a photo of a small {c}.',
77
- lambda c : f'a photo of a big {c}.',
78
- lambda c : f'a photo of the {c}.',
79
- lambda c : f'a blurry photo of the {c}.',
80
- lambda c : f'a black and white photo of the {c}.',
81
- lambda c : f'a low contrast photo of the {c}.',
82
- lambda c : f'a high contrast photo of the {c}.',
83
- lambda c : f'a bad photo of the {c}.',
84
- lambda c : f'a good photo of the {c}.',
85
- lambda c : f'a photo of the small {c}.',
86
- lambda c : f'a photo of the big {c}.',
87
- ]
88
-
89
- super().__init__(train_x=train, val=val, test=test)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/MTIL_datasets/clevr_count.py DELETED
@@ -1,118 +0,0 @@
1
- import os
2
- import pickle
3
- from typing import List
4
-
5
- from .utils import Datum, DatasetBase, mkdir_if_missing
6
- from .oxford_pets import OxfordPets
7
- from .utils import read_json, write_json
8
-
9
- # Class list and templates as specified
10
- CLEVR_COUNT_CLASSES: List[str] = [
11
- '10', '3', '4', '5', '6', '7', '8', '9'
12
- ]
13
-
14
- CLEVR_COUNT_TEMPLATES: List[str] = [
15
- 'a photo of {} objects.',
16
- ]
17
-
18
-
19
- class CLEVRCount(DatasetBase):
20
-
21
- dataset_dir = 'clevr'
22
-
23
- def __init__(self, root, num_shots: int = 0, seed: int = 1, subsample_classes: str = 'all'):
24
- # Root and directories
25
- root = os.path.abspath(os.path.expanduser(root))
26
- self.dataset_dir = os.path.join(root, self.dataset_dir)
27
- self.images_dir = os.path.join(self.dataset_dir, 'images')
28
- self.scenes_dir = os.path.join(self.dataset_dir, 'scenes')
29
- self.split_path = os.path.join(self.dataset_dir, 'split_custom_CLEVRCount.json')
30
- self.split_fewshot_dir = os.path.join(self.dataset_dir, 'split_fewshot')
31
- mkdir_if_missing(self.split_fewshot_dir)
32
-
33
- # Required files
34
- train_scenes = os.path.join(self.scenes_dir, 'CLEVR_train_scenes.json')
35
- val_scenes = os.path.join(self.scenes_dir, 'CLEVR_val_scenes.json')
36
- if not os.path.isfile(train_scenes) or not os.path.isfile(val_scenes):
37
- raise FileNotFoundError(
38
- f"CLEVRCount expects scenes JSON at {train_scenes} and {val_scenes}"
39
- )
40
-
41
- # Load or build split
42
- if os.path.exists(self.split_path):
43
- train, val, test = OxfordPets.read_split(self.split_path, self.dataset_dir)
44
- else:
45
- trainval = self._read_scenes(train_scenes, split='train')
46
- test = self._read_scenes(val_scenes, split='val')
47
- train, val = OxfordPets.split_trainval(trainval)
48
- OxfordPets.save_split(train, val, test, self.split_path, self.dataset_dir)
49
-
50
- # Few-shot
51
- if num_shots >= 1:
52
- preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
53
- if os.path.exists(preprocessed):
54
- print(f"Loading preprocessed few-shot data from {preprocessed}")
55
- with open(preprocessed, 'rb') as f:
56
- data = pickle.load(f)
57
- train, val = data['train'], data['val']
58
- else:
59
- train = self.generate_fewshot_dataset(train, num_shots=num_shots)
60
- val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
61
- data = {'train': train, 'val': val}
62
- print(f"Saving preprocessed few-shot data to {preprocessed}")
63
- with open(preprocessed, 'wb') as f:
64
- pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL)
65
-
66
- # Optional class subsampling (base/new)
67
- train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample_classes)
68
-
69
- # Templates
70
- self.templates = CLEVR_COUNT_TEMPLATES
71
-
72
- # Debug stats
73
- try:
74
- def _hist(items: List[Datum]):
75
- from collections import Counter
76
- cnt = Counter([it.label for it in items])
77
- out = {i: int(cnt.get(i, 0)) for i in range(len(CLEVR_COUNT_CLASSES))}
78
- return out
79
- except Exception as e:
80
- print(f"CLEVRCount stats printing failed: {e}")
81
-
82
- super().__init__(train_x=train, val=val, test=test)
83
- # Ensure class metadata is stable and matches the provided list
84
- self._classnames = CLEVR_COUNT_CLASSES
85
- self._lab2cname = {i: c for i, c in enumerate(CLEVR_COUNT_CLASSES)}
86
- self._num_classes = len(CLEVR_COUNT_CLASSES)
87
-
88
- def _read_scenes(self, json_path: str, split: str) -> List[Datum]:
89
- """Read CLEVR scenes JSON and construct a list of Datum entries.
90
- Only images whose object count appears in CLEVR_COUNT_CLASSES are kept.
91
- """
92
- obj = read_json(json_path)
93
- scenes = obj.get('scenes', [])
94
- # Map count -> class index
95
- lab2idx = {int(c): i for i, c in enumerate(CLEVR_COUNT_CLASSES)}
96
- items: List[Datum] = []
97
- for sc in scenes:
98
- # Some files use key 'split', some typos list 'spit'; be robust
99
- image_filename = sc.get('image_filename', None)
100
- if not image_filename:
101
- continue
102
- n_objects = sc.get('objects', [])
103
- try:
104
- num = int(len(n_objects))
105
- except Exception:
106
- continue
107
- if num not in lab2idx:
108
- # skip counts not in the configured class list
109
- continue
110
- label_i = lab2idx[num]
111
- # Build absolute path to the image based on split
112
- if split not in ['train', 'val', 'test']:
113
- split_dir = 'train'
114
- else:
115
- split_dir = split
116
- impath = os.path.join(self.dataset_dir, 'images', split_dir, image_filename)
117
- items.append(Datum(impath=impath, label=label_i, classname=str(num)))
118
- return items
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/MTIL_datasets/country211.py DELETED
@@ -1,386 +0,0 @@
1
- import os
2
- import json
3
- import pickle
4
- import re
5
-
6
- from .utils import *
7
- from .oxford_pets import OxfordPets
8
- from .dtd import DescribableTextures as DTD
9
-
10
-
11
- class Country211(DatasetBase):
12
- """
13
- Country211 dataset loader for MTIL.
14
- Expected structure:
15
- <root>/country211/
16
- ├─ train/<ISO2>/*.jpg
17
- ├─ valid/<ISO2>/*.jpg
18
- └─ test/<ISO2>/*.jpg
19
-
20
- Where <ISO2> are ISO-3166 alpha-2 country codes (e.g., AD, US, CN).
21
- Class names are mapped from ISO2 codes to provided human-readable names.
22
- Optionally, a JSON mapping file can override defaults:
23
- <root>/country211/iso2_to_name.json => {"AD": "Andorra", ...}
24
- """
25
-
26
- dataset_dir = "country211"
27
-
28
- def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'):
29
- root = os.path.abspath(os.path.expanduser(root))
30
- self.dataset_dir = os.path.join(root, self.dataset_dir)
31
- self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot")
32
- mkdir_if_missing(self.split_fewshot_dir)
33
-
34
- # Prefer explicit split directories if they exist
35
- train_dir = os.path.join(self.dataset_dir, "train")
36
- valid_dir = os.path.join(self.dataset_dir, "valid")
37
- test_dir = os.path.join(self.dataset_dir, "test")
38
-
39
- use_folder_splits = os.path.isdir(train_dir) and os.path.isdir(valid_dir) and os.path.isdir(test_dir)
40
-
41
- if use_folder_splits:
42
- # Build ISO2 -> name mapping (allow override via json)
43
- iso_map = self._load_iso2_to_name()
44
- # Create unified code set across splits to keep label mapping stable
45
- all_codes = set()
46
- for d in [train_dir, valid_dir, test_dir]:
47
- if os.path.isdir(d):
48
- for code in listdir_nohidden(d):
49
- code_path = os.path.join(d, code)
50
- if os.path.isdir(code_path):
51
- all_codes.add(code)
52
- # Validate code format and mapping coverage early
53
- override_path = os.path.join(self.dataset_dir, 'iso2_to_name.json')
54
- pat = re.compile(r'^[A-Z]{2}$')
55
- invalid_codes = sorted([c for c in all_codes if not (pat.match(c) or c == 'XK')])
56
- if invalid_codes:
57
- raise ValueError(
58
- "Country211: Found invalid ISO2 code folder names: {}. "
59
- "Codes must be two uppercase letters (e.g., 'US', 'CN') or 'XK'. "
60
- "Please rename these folders accordingly.".format(invalid_codes)
61
- )
62
- unknown_codes = sorted([c for c in all_codes if c not in iso_map])
63
- if unknown_codes:
64
- raise ValueError(
65
- "Country211: ISO2 codes missing from mapping: {}. "
66
- "Add them to {} as a JSON dict, e.g., {\"XX\": \"Country Name\"}.".format(
67
- unknown_codes, override_path
68
- )
69
- )
70
- # Sort by human name (fallback to code) for stable labels
71
- def _code_to_name(c):
72
- return iso_map.get(c, c)
73
- sorted_codes = sorted(list(all_codes), key=lambda c: _code_to_name(c))
74
- code_to_label = {c: i for i, c in enumerate(sorted_codes)}
75
-
76
- train = self._read_split_dir(train_dir, code_to_label, iso_map)
77
- val = self._read_split_dir(valid_dir, code_to_label, iso_map)
78
- test = self._read_split_dir(test_dir, code_to_label, iso_map)
79
- else:
80
- # Fallbacks: JSON split or naive folder split
81
- image_dir = os.path.join(self.dataset_dir, "images")
82
- self.image_dir = image_dir if os.path.isdir(image_dir) else self.dataset_dir
83
- split_path = os.path.join(self.dataset_dir, "split_zhou_Country211.json")
84
- if os.path.exists(split_path):
85
- train, val, test = OxfordPets.read_split(split_path, self.image_dir)
86
- else:
87
- train, val, test = DTD.read_and_split_data(self.image_dir)
88
- OxfordPets.save_split(train, val, test, split_path, self.image_dir)
89
-
90
- if num_shots >= 1:
91
- preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
92
- if os.path.exists(preprocessed):
93
- print(f"Loading preprocessed few-shot data from {preprocessed}")
94
- with open(preprocessed, "rb") as file:
95
- data = pickle.load(file)
96
- train, val = data["train"], data["val"]
97
- else:
98
- train = self.generate_fewshot_dataset(train, num_shots=num_shots)
99
- val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
100
- data = {"train": train, "val": val}
101
- print(f"Saving preprocessed few-shot data to {preprocessed}")
102
- with open(preprocessed, "wb") as file:
103
- pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)
104
-
105
- subsample = subsample_classes
106
- train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample)
107
-
108
- # Templates provided by user
109
- self.templates = [
110
- lambda c: f'a photo i took in {c}.',
111
- lambda c: f'a photo i took while visiting {c}.',
112
- lambda c: f'a photo from my home country of {c}.',
113
- lambda c: f'a photo from my visit to {c}.',
114
- lambda c: f'a photo showing the country of {c}.',
115
- ]
116
-
117
- super().__init__(train_x=train, val=val, test=test)
118
-
119
- def _read_split_dir(self, split_dir, code_to_label, iso_map):
120
- items = []
121
- if not os.path.isdir(split_dir):
122
- return items
123
- codes = listdir_nohidden(split_dir)
124
- for code in codes:
125
- class_dir = os.path.join(split_dir, code)
126
- if not os.path.isdir(class_dir):
127
- continue
128
- label = code_to_label.get(code)
129
- if label is None:
130
- raise ValueError(
131
- f"Country211: Inconsistent label mapping for code '{code}' in split '{split_dir}'. "
132
- f"This indicates an internal mismatch between discovered codes and label map."
133
- )
134
- try:
135
- cname = iso_map[code]
136
- except KeyError:
137
- raise ValueError(
138
- f"Country211: Missing ISO mapping for code '{code}'. "
139
- f"Please add it to '{os.path.join(self.dataset_dir, 'iso2_to_name.json')}'."
140
- )
141
- for fname in listdir_nohidden(class_dir):
142
- impath = os.path.join(class_dir, fname)
143
- items.append(Datum(impath=impath, label=label, classname=cname))
144
- return items
145
-
146
- def _load_iso2_to_name(self):
147
- """Load ISO2->country name mapping, allow local JSON override.
148
- Fallback to built-in mapping; unknown codes map to themselves at usage time.
149
- """
150
- override_path = os.path.join(self.dataset_dir, 'iso2_to_name.json')
151
- if os.path.exists(override_path):
152
- try:
153
- data = read_json(override_path)
154
- if not isinstance(data, dict):
155
- raise ValueError(
156
- f"Country211: Expected a JSON object in '{override_path}', got {type(data)}"
157
- )
158
- # Validate keys as ISO2 (two uppercase letters) or 'XK', and values are non-empty strings
159
- pat = re.compile(r'^[A-Z]{2}$')
160
- bad_keys = [k for k in data.keys() if not (isinstance(k, str) and (pat.match(k) or k == 'XK'))]
161
- bad_vals = [k for k, v in data.items() if not (isinstance(v, str) and v.strip())]
162
- if bad_keys or bad_vals:
163
- raise ValueError(
164
- (
165
- "Country211: Invalid iso2_to_name.json entries. "
166
- f"Bad keys (must be ISO2 like 'US'): {bad_keys}. "
167
- f"Bad values (must be non-empty strings) for keys: {bad_vals}."
168
- )
169
- )
170
- return data
171
- except Exception as e:
172
- raise ValueError(f"Country211: Failed to load '{override_path}': {e}")
173
- # Built-in mapping aligned with provided classes
174
- return {
175
- 'AD': 'Andorra',
176
- 'AE': 'United Arab Emirates',
177
- 'AF': 'Afghanistan',
178
- 'AG': 'Antigua and Barbuda',
179
- 'AI': 'Anguilla',
180
- 'AL': 'Albania',
181
- 'AM': 'Armenia',
182
- 'AO': 'Angola',
183
- 'AQ': 'Antarctica',
184
- 'AR': 'Argentina',
185
- 'AT': 'Austria',
186
- 'AU': 'Australia',
187
- 'AW': 'Aruba',
188
- 'AX': 'Aland Islands',
189
- 'AZ': 'Azerbaijan',
190
- 'BA': 'Bosnia and Herzegovina',
191
- 'BB': 'Barbados',
192
- 'BD': 'Bangladesh',
193
- 'BE': 'Belgium',
194
- 'BF': 'Burkina Faso',
195
- 'BG': 'Bulgaria',
196
- 'BH': 'Bahrain',
197
- 'BJ': 'Benin',
198
- 'BM': 'Bermuda',
199
- 'BN': 'Brunei Darussalam',
200
- 'BO': 'Bolivia',
201
- 'BQ': 'Bonaire, Saint Eustatius and Saba',
202
- 'BR': 'Brazil',
203
- 'BS': 'Bahamas',
204
- 'BT': 'Bhutan',
205
- 'BW': 'Botswana',
206
- 'BY': 'Belarus',
207
- 'BZ': 'Belize',
208
- 'CA': 'Canada',
209
- 'CD': 'DR Congo',
210
- 'CF': 'Central African Republic',
211
- 'CH': 'Switzerland',
212
- 'CI': "Cote d'Ivoire",
213
- 'CK': 'Cook Islands',
214
- 'CL': 'Chile',
215
- 'CM': 'Cameroon',
216
- 'CN': 'China',
217
- 'CO': 'Colombia',
218
- 'CR': 'Costa Rica',
219
- 'CU': 'Cuba',
220
- 'CV': 'Cabo Verde',
221
- 'CW': 'Curacao',
222
- 'CY': 'Cyprus',
223
- 'CZ': 'Czech Republic',
224
- 'DE': 'Germany',
225
- 'DK': 'Denmark',
226
- 'DM': 'Dominica',
227
- 'DO': 'Dominican Republic',
228
- 'DZ': 'Algeria',
229
- 'EC': 'Ecuador',
230
- 'EE': 'Estonia',
231
- 'EG': 'Egypt',
232
- 'ES': 'Spain',
233
- 'ET': 'Ethiopia',
234
- 'FI': 'Finland',
235
- 'FJ': 'Fiji',
236
- 'FK': 'Falkland Islands',
237
- 'FO': 'Faeroe Islands',
238
- 'FR': 'France',
239
- 'GA': 'Gabon',
240
- 'GB': 'United Kingdom',
241
- 'GD': 'Grenada',
242
- 'GE': 'Georgia',
243
- 'GF': 'French Guiana',
244
- 'GG': 'Guernsey',
245
- 'GH': 'Ghana',
246
- 'GI': 'Gibraltar',
247
- 'GL': 'Greenland',
248
- 'GM': 'Gambia',
249
- 'GP': 'Guadeloupe',
250
- 'GR': 'Greece',
251
- 'GS': 'South Georgia and South Sandwich Is.',
252
- 'GT': 'Guatemala',
253
- 'GU': 'Guam',
254
- 'GY': 'Guyana',
255
- 'HK': 'Hong Kong',
256
- 'HN': 'Honduras',
257
- 'HR': 'Croatia',
258
- 'HT': 'Haiti',
259
- 'HU': 'Hungary',
260
- 'ID': 'Indonesia',
261
- 'IE': 'Ireland',
262
- 'IL': 'Israel',
263
- 'IM': 'Isle of Man',
264
- 'IN': 'India',
265
- 'IQ': 'Iraq',
266
- 'IR': 'Iran',
267
- 'IS': 'Iceland',
268
- 'IT': 'Italy',
269
- 'JE': 'Jersey',
270
- 'JM': 'Jamaica',
271
- 'JO': 'Jordan',
272
- 'JP': 'Japan',
273
- 'KE': 'Kenya',
274
- 'KG': 'Kyrgyz Republic',
275
- 'KH': 'Cambodia',
276
- 'KN': 'St. Kitts and Nevis',
277
- 'KP': 'North Korea',
278
- 'KR': 'South Korea',
279
- 'KW': 'Kuwait',
280
- 'KY': 'Cayman Islands',
281
- 'KZ': 'Kazakhstan',
282
- 'LA': 'Laos',
283
- 'LB': 'Lebanon',
284
- 'LC': 'St. Lucia',
285
- 'LI': 'Liechtenstein',
286
- 'LK': 'Sri Lanka',
287
- 'LR': 'Liberia',
288
- 'LT': 'Lithuania',
289
- 'LU': 'Luxembourg',
290
- 'LV': 'Latvia',
291
- 'LY': 'Libya',
292
- 'MA': 'Morocco',
293
- 'MC': 'Monaco',
294
- 'MD': 'Moldova',
295
- 'ME': 'Montenegro',
296
- 'MF': 'Saint-Martin',
297
- 'MG': 'Madagascar',
298
- 'MK': 'Macedonia',
299
- 'ML': 'Mali',
300
- 'MM': 'Myanmar',
301
- 'MN': 'Mongolia',
302
- 'MO': 'Macau',
303
- 'MQ': 'Martinique',
304
- 'MR': 'Mauritania',
305
- 'MT': 'Malta',
306
- 'MU': 'Mauritius',
307
- 'MV': 'Maldives',
308
- 'MW': 'Malawi',
309
- 'MX': 'Mexico',
310
- 'MY': 'Malaysia',
311
- 'MZ': 'Mozambique',
312
- 'NA': 'Namibia',
313
- 'NC': 'New Caledonia',
314
- 'NG': 'Nigeria',
315
- 'NI': 'Nicaragua',
316
- 'NL': 'Netherlands',
317
- 'NO': 'Norway',
318
- 'NP': 'Nepal',
319
- 'NZ': 'New Zealand',
320
- 'OM': 'Oman',
321
- 'PA': 'Panama',
322
- 'PE': 'Peru',
323
- 'PF': 'French Polynesia',
324
- 'PG': 'Papua New Guinea',
325
- 'PH': 'Philippines',
326
- 'PK': 'Pakistan',
327
- 'PL': 'Poland',
328
- 'PR': 'Puerto Rico',
329
- 'PS': 'Palestine',
330
- 'PT': 'Portugal',
331
- 'PW': 'Palau',
332
- 'PY': 'Paraguay',
333
- 'QA': 'Qatar',
334
- 'RE': 'Reunion',
335
- 'RO': 'Romania',
336
- 'RS': 'Serbia',
337
- 'RU': 'Russia',
338
- 'RW': 'Rwanda',
339
- 'SA': 'Saudi Arabia',
340
- 'SB': 'Solomon Islands',
341
- 'SC': 'Seychelles',
342
- 'SD': 'Sudan',
343
- 'SE': 'Sweden',
344
- 'SG': 'Singapore',
345
- 'SH': 'St. Helena',
346
- 'SI': 'Slovenia',
347
- 'SJ': 'Svalbard and Jan Mayen Islands',
348
- 'SK': 'Slovakia',
349
- 'SL': 'Sierra Leone',
350
- 'SM': 'San Marino',
351
- 'SN': 'Senegal',
352
- 'SO': 'Somalia',
353
- 'SS': 'South Sudan',
354
- 'SV': 'El Salvador',
355
- 'SX': 'Sint Maarten',
356
- 'SY': 'Syria',
357
- 'SZ': 'Eswatini',
358
- 'TG': 'Togo',
359
- 'TH': 'Thailand',
360
- 'TJ': 'Tajikistan',
361
- 'TL': 'Timor-Leste',
362
- 'TM': 'Turkmenistan',
363
- 'TN': 'Tunisia',
364
- 'TO': 'Tonga',
365
- 'TR': 'Turkey',
366
- 'TT': 'Trinidad and Tobago',
367
- 'TW': 'Taiwan',
368
- 'TZ': 'Tanzania',
369
- 'UA': 'Ukraine',
370
- 'UG': 'Uganda',
371
- 'US': 'United States',
372
- 'UY': 'Uruguay',
373
- 'UZ': 'Uzbekistan',
374
- 'VA': 'Vatican',
375
- 'VE': 'Venezuela',
376
- 'VG': 'British Virgin Islands',
377
- 'VI': 'United States Virgin Islands',
378
- 'VN': 'Vietnam',
379
- 'VU': 'Vanuatu',
380
- 'WS': 'Samoa',
381
- 'XK': 'Kosovo',
382
- 'YE': 'Yemen',
383
- 'ZA': 'South Africa',
384
- 'ZM': 'Zambia',
385
- 'ZW': 'Zimbabwe',
386
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/MTIL_datasets/dtd.py DELETED
@@ -1,95 +0,0 @@
1
- import os
2
- import pickle
3
- import random
4
-
5
- from .utils import *
6
-
7
- from .oxford_pets import OxfordPets
8
-
9
-
10
- class DescribableTextures(DatasetBase):
11
-
12
- dataset_dir = "dtd"
13
-
14
- def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'):
15
- root = os.path.abspath(os.path.expanduser(root))
16
- self.dataset_dir = os.path.join(root, self.dataset_dir)
17
- self.image_dir = os.path.join(self.dataset_dir, "images")
18
- self.split_path = os.path.join(self.dataset_dir, "split_zhou_DescribableTextures.json")
19
- self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot")
20
- mkdir_if_missing(self.split_fewshot_dir)
21
-
22
- if os.path.exists(self.split_path):
23
- train, val, test = OxfordPets.read_split(self.split_path, self.image_dir)
24
- else:
25
- train, val, test = self.read_and_split_data(self.image_dir)
26
- OxfordPets.save_split(train, val, test, self.split_path, self.image_dir)
27
-
28
- if num_shots >= 1:
29
- preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
30
-
31
- if os.path.exists(preprocessed):
32
- print(f"Loading preprocessed few-shot data from {preprocessed}")
33
- with open(preprocessed, "rb") as file:
34
- data = pickle.load(file)
35
- train, val = data["train"], data["val"]
36
- else:
37
- train = self.generate_fewshot_dataset(train, num_shots=num_shots)
38
- val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
39
- data = {"train": train, "val": val}
40
- print(f"Saving preprocessed few-shot data to {preprocessed}")
41
- with open(preprocessed, "wb") as file:
42
- pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)
43
-
44
- subsample = subsample_classes
45
- train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample)
46
-
47
- self.templates = [
48
- lambda c: f'a photo of a {c} texture.',
49
- lambda c: f'a photo of a {c} pattern.',
50
- lambda c: f'a photo of a {c} thing.',
51
- lambda c: f'a photo of a {c} object.',
52
- lambda c: f'a photo of the {c} texture.',
53
- lambda c: f'a photo of the {c} pattern.',
54
- lambda c: f'a photo of the {c} thing.',
55
- lambda c: f'a photo of the {c} object.',
56
- ]
57
-
58
- super().__init__(train_x=train, val=val, test=test)
59
-
60
- @staticmethod
61
- def read_and_split_data(image_dir, p_trn=0.5, p_val=0.2, ignored=[], new_cnames=None):
62
- categories = listdir_nohidden(image_dir)
63
- categories = [c for c in categories if c not in ignored]
64
- categories.sort()
65
-
66
- p_tst = 1 - p_trn - p_val
67
- print(f"Splitting into {p_trn:.0%} train, {p_val:.0%} val, and {p_tst:.0%} test")
68
-
69
- def _collate(ims, y, c):
70
- items = []
71
- for im in ims:
72
- item = Datum(impath=im, label=y, classname=c)
73
- items.append(item)
74
- return items
75
-
76
- train, val, test = [], [], []
77
- for label, category in enumerate(categories):
78
- category_dir = os.path.join(image_dir, category)
79
- images = listdir_nohidden(category_dir)
80
- images = [os.path.join(category_dir, im) for im in images]
81
- random.shuffle(images)
82
- n_total = len(images)
83
- n_train = round(n_total * p_trn)
84
- n_val = round(n_total * p_val)
85
- n_test = n_total - n_train - n_val
86
- assert n_train > 0 and n_val > 0 and n_test > 0
87
-
88
- if new_cnames is not None and category in new_cnames:
89
- category = new_cnames[category]
90
-
91
- train.extend(_collate(images[:n_train], label, category))
92
- val.extend(_collate(images[n_train : n_train + n_val], label, category))
93
- test.extend(_collate(images[n_train + n_val :], label, category))
94
-
95
- return train, val, test
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/MTIL_datasets/eurosat.py DELETED
@@ -1,75 +0,0 @@
1
- import os
2
- import pickle
3
-
4
- from .utils import *
5
-
6
- from .oxford_pets import OxfordPets
7
- from .dtd import DescribableTextures as DTD
8
-
9
- NEW_CNAMES = {
10
- "AnnualCrop": "Annual Crop Land",
11
- "Forest": "Forest",
12
- "HerbaceousVegetation": "Herbaceous Vegetation Land",
13
- "Highway": "Highway or Road",
14
- "Industrial": "Industrial Buildings",
15
- "Pasture": "Pasture Land",
16
- "PermanentCrop": "Permanent Crop Land",
17
- "Residential": "Residential Buildings",
18
- "River": "River",
19
- "SeaLake": "Sea or Lake",
20
- }
21
-
22
-
23
- class EuroSAT(DatasetBase):
24
-
25
- dataset_dir = "eurosat"
26
-
27
- def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'):
28
- root = os.path.abspath(os.path.expanduser(root))
29
- self.dataset_dir = os.path.join(root, self.dataset_dir)
30
- self.image_dir = os.path.join(self.dataset_dir, "2750")
31
- self.split_path = os.path.join(self.dataset_dir, "split_zhou_EuroSAT.json")
32
- self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot")
33
- mkdir_if_missing(self.split_fewshot_dir)
34
-
35
- if os.path.exists(self.split_path):
36
- train, val, test = OxfordPets.read_split(self.split_path, self.image_dir)
37
- else:
38
- train, val, test = DTD.read_and_split_data(self.image_dir, new_cnames=NEW_CNAMES)
39
- OxfordPets.save_split(train, val, test, self.split_path, self.image_dir)
40
-
41
- if num_shots >= 1:
42
- preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
43
-
44
- if os.path.exists(preprocessed):
45
- print(f"Loading preprocessed few-shot data from {preprocessed}")
46
- with open(preprocessed, "rb") as file:
47
- data = pickle.load(file)
48
- train, val = data["train"], data["val"]
49
- else:
50
- train = self.generate_fewshot_dataset(train, num_shots=num_shots)
51
- val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
52
- data = {"train": train, "val": val}
53
- print(f"Saving preprocessed few-shot data to {preprocessed}")
54
- with open(preprocessed, "wb") as file:
55
- pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)
56
-
57
- subsample = subsample_classes
58
- train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample)
59
-
60
- self.templates = [
61
- lambda c: f"a centered satellite photo of {c}.",
62
- lambda c: f"a centered satellite photo of a {c}.",
63
- lambda c: f"a centered satellite photo of the {c}.",
64
- ]
65
-
66
- super().__init__(train_x=train, val=val, test=test)
67
-
68
- def update_classname(self, dataset_old):
69
- dataset_new = []
70
- for item_old in dataset_old:
71
- cname_old = item_old.classname
72
- cname_new = NEW_CLASSNAMES[cname_old]
73
- item_new = Datum(impath=item_old.impath, label=item_old.label, classname=cname_new)
74
- dataset_new.append(item_new)
75
- return dataset_new
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/MTIL_datasets/fer2013.py DELETED
@@ -1,170 +0,0 @@
1
- import os
2
- import pickle
3
- import random
4
- import warnings
5
- from collections import defaultdict
6
-
7
- from .utils import *
8
- from .oxford_pets import OxfordPets
9
-
10
- # Synonym sets per class (label index is the list index)
11
- FER2013_CLASSES_SYNONYMS = [
12
- ['angry'],
13
- ['disgusted', 'disgust'],
14
- ['fearful', 'fear'],
15
- ['happy', 'smiling'],
16
- ['sad', 'depressed'],
17
- ['surprised', 'surprise', 'shocked', 'spooked'],
18
- ['neutral', 'bored'],
19
- ]
20
-
21
- # Canonical class names are the first synonym in each list
22
- FER2013_CANONICAL = [syns[0] for syns in FER2013_CLASSES_SYNONYMS]
23
-
24
- # Prompt templates
25
- FER2013_TEMPLATES = [
26
- 'a photo of a {} looking face.',
27
- 'a photo of a face showing the emotion: {}.',
28
- 'a photo of a face looking {}.',
29
- 'a face that looks {}.',
30
- 'they look {}.',
31
- 'look at how {} they are.',
32
- ]
33
-
34
- FER2013_DEBUG = os.environ.get("FER2013_DEBUG", "0") not in ("0", "false", "False", "")
35
-
36
- def _dbg(msg: str):
37
- if FER2013_DEBUG:
38
- print(f"[FER2013][DEBUG] {msg}")
39
-
40
-
41
- def _norm(s: str) -> str:
42
- s = s.lower().strip()
43
- for ch in [" ", "_", "-", "."]:
44
- s = s.replace(ch, "")
45
- return s
46
-
47
-
48
- def _build_syn_map():
49
- m = {}
50
- for y, syns in enumerate(FER2013_CLASSES_SYNONYMS):
51
- for s in syns:
52
- m[_norm(s)] = y
53
- # add common canonical variants for safety
54
- aliases = {
55
- 'disgust': 1,
56
- 'fear': 2,
57
- 'surprise': 5,
58
- }
59
- for k, v in aliases.items():
60
- m[_norm(k)] = v
61
- return m
62
-
63
-
64
- class FER2013(DatasetBase):
65
-
66
- dataset_dir = "fer2013"
67
-
68
- def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'):
69
- root = os.path.abspath(os.path.expanduser(root))
70
- self.dataset_dir = os.path.join(root, self.dataset_dir)
71
- self.split_path = os.path.join(self.dataset_dir, "split_custom_FER2013.json")
72
- self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot")
73
- mkdir_if_missing(self.split_fewshot_dir)
74
-
75
- train_dir = os.path.join(self.dataset_dir, "train")
76
- test_dir = os.path.join(self.dataset_dir, "test")
77
- if not os.path.isdir(train_dir) or not os.path.isdir(test_dir):
78
- raise ValueError(
79
- f"FER2013: expected train/test folders under '{self.dataset_dir}'. Got train={os.path.isdir(train_dir)}, test={os.path.isdir(test_dir)}"
80
- )
81
-
82
- # try cache
83
- if os.path.exists(self.split_path):
84
- try:
85
- train, val, test = OxfordPets.read_split(self.split_path, self.dataset_dir)
86
- except Exception as e:
87
- warnings.warn(f"FER2013: failed to read cached split; rebuilding. Error: {e}")
88
- train, val, test = self._build_split(train_dir, test_dir)
89
- try:
90
- OxfordPets.save_split(train, val, test, self.split_path, self.dataset_dir)
91
- except Exception as e2:
92
- warnings.warn(f"FER2013: failed to save split: {e2}")
93
- else:
94
- train, val, test = self._build_split(train_dir, test_dir)
95
- try:
96
- OxfordPets.save_split(train, val, test, self.split_path, self.dataset_dir)
97
- except Exception as e:
98
- warnings.warn(f"FER2013: failed to save split: {e}")
99
-
100
- if num_shots >= 1:
101
- preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
102
- if os.path.exists(preprocessed):
103
- print(f"Loading preprocessed few-shot data from {preprocessed}")
104
- with open(preprocessed, "rb") as file:
105
- data = pickle.load(file)
106
- train, val = data["train"], data["val"]
107
- else:
108
- train = self.generate_fewshot_dataset(train, num_shots=num_shots)
109
- val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
110
- data = {"train": train, "val": val}
111
- print(f"Saving preprocessed few-shot data to {preprocessed}")
112
- with open(preprocessed, "wb") as file:
113
- pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)
114
-
115
- train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample_classes)
116
- self.templates = FER2013_TEMPLATES
117
- super().__init__(train_x=train, val=val, test=test)
118
-
119
- def _build_split(self, train_dir, test_dir, p_val=0.2):
120
- syn_map = _build_syn_map()
121
- # read train per class
122
- tr_items_by_label = defaultdict(list)
123
- class_dirs = listdir_nohidden(train_dir, sort=True)
124
- if not class_dirs:
125
- warnings.warn(f"FER2013: no class folders found in {train_dir}")
126
- for cls in class_dirs:
127
- full = os.path.join(train_dir, cls)
128
- if not os.path.isdir(full):
129
- continue
130
- key = _norm(cls)
131
- y = syn_map.get(key)
132
- if y is None:
133
- warnings.warn(f"FER2013: unexpected class folder in train: '{cls}'")
134
- continue
135
- cname = FER2013_CANONICAL[y]
136
- for fname in listdir_nohidden(full):
137
- impath = os.path.join(full, fname)
138
- tr_items_by_label[y].append(Datum(impath=impath, label=y, classname=cname))
139
-
140
- # stratified split train->(train,val)
141
- train, val = [], []
142
- for y, items in tr_items_by_label.items():
143
- random.shuffle(items)
144
- n_val = max(1, round(len(items) * p_val)) if len(items) > 1 else 0
145
- val.extend(items[:n_val])
146
- train.extend(items[n_val:])
147
-
148
- # read test
149
- test = []
150
- class_dirs = listdir_nohidden(test_dir, sort=True)
151
- if not class_dirs:
152
- warnings.warn(f"FER2013: no class folders found in {test_dir}")
153
- for cls in class_dirs:
154
- full = os.path.join(test_dir, cls)
155
- if not os.path.isdir(full):
156
- continue
157
- key = _norm(cls)
158
- y = syn_map.get(key)
159
- if y is None:
160
- warnings.warn(f"FER2013: unexpected class folder in test: '{cls}'")
161
- continue
162
- cname = FER2013_CANONICAL[y]
163
- for fname in listdir_nohidden(full):
164
- impath = os.path.join(full, fname)
165
- test.append(Datum(impath=impath, label=y, classname=cname))
166
-
167
- # basic sanity
168
- if not train or not val or not test:
169
- warnings.warn(f"FER2013: split sizes train={len(train)} val={len(val)} test={len(test)}")
170
- return train, val, test
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/MTIL_datasets/fgvc_aircraft.py DELETED
@@ -1,73 +0,0 @@
1
- import os
2
- import pickle
3
-
4
- from .utils import *
5
-
6
- from .oxford_pets import OxfordPets
7
- import torchvision.datasets
8
-
9
-
10
- class FGVCAircraft(DatasetBase):
11
-
12
- dataset_dir = "fgvc_aircraft"
13
-
14
- def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'):
15
- root = os.path.abspath(os.path.expanduser(root))
16
- self.dataset_dir = os.path.join(root, self.dataset_dir)
17
- self.image_dir = os.path.join(self.dataset_dir, "images")
18
- self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot")
19
- mkdir_if_missing(self.split_fewshot_dir)
20
-
21
- classnames = []
22
- with open(os.path.join(self.dataset_dir, "variants.txt"), "r") as f:
23
- lines = f.readlines()
24
- for line in lines:
25
- classnames.append(line.strip())
26
- cname2lab = {c: i for i, c in enumerate(classnames)}
27
-
28
- train = self.read_data(cname2lab, "images_variant_train.txt")
29
- val = self.read_data(cname2lab, "images_variant_val.txt")
30
- test = self.read_data(cname2lab, "images_variant_test.txt")
31
-
32
- if num_shots >= 1:
33
- preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
34
-
35
- if os.path.exists(preprocessed):
36
- print(f"Loading preprocessed few-shot data from {preprocessed}")
37
- with open(preprocessed, "rb") as file:
38
- data = pickle.load(file)
39
- train, val = data["train"], data["val"]
40
- else:
41
- train = self.generate_fewshot_dataset(train, num_shots=num_shots)
42
- val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
43
- data = {"train": train, "val": val}
44
- print(f"Saving preprocessed few-shot data to {preprocessed}")
45
- with open(preprocessed, "wb") as file:
46
- pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)
47
-
48
- subsample = subsample_classes
49
- train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample)
50
-
51
- self.templates = [
52
- lambda c: f"a photo of a {c}, a type of aircraft.",
53
- lambda c: f"a photo of the {c}, a type of aircraft.",
54
- ]
55
-
56
- super().__init__(train_x=train, val=val, test=test)
57
-
58
- def read_data(self, cname2lab, split_file):
59
- filepath = os.path.join(self.dataset_dir, split_file)
60
- items = []
61
-
62
- with open(filepath, "r") as f:
63
- lines = f.readlines()
64
- for line in lines:
65
- line = line.strip().split(" ")
66
- imname = line[0] + ".jpg"
67
- classname = " ".join(line[1:])
68
- impath = os.path.join(self.image_dir, imname)
69
- label = cname2lab[classname]
70
- item = Datum(impath=impath, label=label, classname=classname)
71
- items.append(item)
72
-
73
- return items
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/MTIL_datasets/food101.py DELETED
@@ -1,52 +0,0 @@
1
- import os
2
- import pickle
3
-
4
- from .utils import *
5
-
6
- from .oxford_pets import OxfordPets
7
- from .dtd import DescribableTextures as DTD
8
-
9
-
10
-
11
- class Food101(DatasetBase):
12
-
13
- dataset_dir = "food-101"
14
-
15
- def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'):
16
- root = os.path.abspath(os.path.expanduser(root))
17
- self.dataset_dir = os.path.join(root, self.dataset_dir)
18
- self.image_dir = os.path.join(self.dataset_dir, "images")
19
- self.split_path = os.path.join(self.dataset_dir, "split_zhou_Food101.json")
20
- self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot")
21
- mkdir_if_missing(self.split_fewshot_dir)
22
-
23
- if os.path.exists(self.split_path):
24
- train, val, test = OxfordPets.read_split(self.split_path, self.image_dir)
25
- else:
26
- train, val, test = DTD.read_and_split_data(self.image_dir)
27
- OxfordPets.save_split(train, val, test, self.split_path, self.image_dir)
28
-
29
- if num_shots >= 1:
30
- preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
31
-
32
- if os.path.exists(preprocessed):
33
- print(f"Loading preprocessed few-shot data from {preprocessed}")
34
- with open(preprocessed, "rb") as file:
35
- data = pickle.load(file)
36
- train, val = data["train"], data["val"]
37
- else:
38
- train = self.generate_fewshot_dataset(train, num_shots=num_shots)
39
- val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
40
- data = {"train": train, "val": val}
41
- print(f"Saving preprocessed few-shot data to {preprocessed}")
42
- with open(preprocessed, "wb") as file:
43
- pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)
44
-
45
- subsample = subsample_classes
46
- train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample)
47
-
48
- self.templates = [
49
- lambda c: f"a photo of a {c}, a type of food.",
50
- ]
51
-
52
- super().__init__(train_x=train, val=val, test=test)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/MTIL_datasets/gtsrb.py DELETED
@@ -1,202 +0,0 @@
1
- import os
2
- import pickle
3
- import random
4
- import warnings
5
-
6
- from .utils import *
7
- from .oxford_pets import OxfordPets
8
-
9
- # 43 classes for GTSRB with human-readable names
10
- classes = [
11
- 'red and white circle 20 kph speed limit',
12
- 'red and white circle 30 kph speed limit',
13
- 'red and white circle 50 kph speed limit',
14
- 'red and white circle 60 kph speed limit',
15
- 'red and white circle 70 kph speed limit',
16
- 'red and white circle 80 kph speed limit',
17
- 'end / de-restriction of 80 kph speed limit',
18
- 'red and white circle 100 kph speed limit',
19
- 'red and white circle 120 kph speed limit',
20
- 'red and white circle red car and black car no passing',
21
- 'red and white circle red truck and black car no passing',
22
- 'red and white triangle road intersection warning',
23
- 'white and yellow diamond priority road',
24
- 'red and white upside down triangle yield right-of-way',
25
- 'stop',
26
- 'empty red and white circle',
27
- 'red and white circle no truck entry',
28
- 'red circle with white horizonal stripe no entry',
29
- 'red and white triangle with exclamation mark warning',
30
- 'red and white triangle with black left curve approaching warning',
31
- 'red and white triangle with black right curve approaching warning',
32
- 'red and white triangle with black double curve approaching warning',
33
- 'red and white triangle rough / bumpy road warning',
34
- 'red and white triangle car skidding / slipping warning',
35
- 'red and white triangle with merging / narrow lanes warning',
36
- 'red and white triangle with person digging / construction / road work warning',
37
- 'red and white triangle with traffic light approaching warning',
38
- 'red and white triangle with person walking warning',
39
- 'red and white triangle with child and person walking warning',
40
- 'red and white triangle with bicyle warning',
41
- 'red and white triangle with snowflake / ice warning',
42
- 'red and white triangle with deer warning',
43
- 'white circle with gray strike bar no speed limit',
44
- 'blue circle with white right turn arrow mandatory',
45
- 'blue circle with white left turn arrow mandatory',
46
- 'blue circle with white forward arrow mandatory',
47
- 'blue circle with white forward or right turn arrow mandatory',
48
- 'blue circle with white forward or left turn arrow mandatory',
49
- 'blue circle with white keep right arrow mandatory',
50
- 'blue circle with white keep left arrow mandatory',
51
- 'blue circle with white arrows indicating a traffic circle',
52
- 'white circle with gray strike bar indicating no passing for cars has ended',
53
- 'white circle with gray strike bar indicating no passing for trucks has ended',
54
- ]
55
-
56
-
57
- DEBUG = os.environ.get("GTSRB_DEBUG", "0") not in ("0", "false", "False", "")
58
-
59
-
60
- def _dbg(msg: str):
61
- if DEBUG:
62
- print(f"[GTSRB][DEBUG] {msg}")
63
-
64
-
65
- def _is_image_file(name):
66
- name = name.lower()
67
- return any(name.endswith(ext) for ext in ['.ppm', '.png', '.jpg', '.jpeg', '.bmp', '.webp'])
68
-
69
-
70
- class GTSRB(DatasetBase):
71
- """
72
- German Traffic Sign Recognition Benchmark (GTSRB)
73
-
74
- Expected structure:
75
- <root>/gtsrb/
76
- ├─ 00000/*.ppm
77
- ├─ 00001/*.ppm
78
- └─ ... up to 00042/
79
-
80
- Note: The dataset has no official test split in this layout, so we will
81
- randomly split each class into train/val/test (50%/20%/30%), cached to json.
82
- """
83
-
84
- dataset_dir = "gtsrb"
85
-
86
- def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'):
87
- root = os.path.abspath(os.path.expanduser(root))
88
- self.dataset_dir = os.path.join(root, self.dataset_dir)
89
- self.image_dir = self.dataset_dir
90
- self.split_path = os.path.join(self.dataset_dir, "split_custom_GTSRB.json")
91
- self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot")
92
- mkdir_if_missing(self.split_fewshot_dir)
93
- _dbg(f"dataset_dir={self.dataset_dir}")
94
- _dbg(f"image_dir={self.image_dir}")
95
- _dbg(f"split_path exists? {os.path.exists(self.split_path)}")
96
-
97
- if os.path.exists(self.split_path):
98
- try:
99
- train, val, test = OxfordPets.read_split(self.split_path, self.image_dir)
100
- except Exception as e:
101
- warnings.warn(f"GTSRB: failed to read split file '{self.split_path}'; rebuilding split. Error: {e}")
102
- train, val, test = self.read_and_split_data(self.image_dir)
103
- try:
104
- OxfordPets.save_split(train, val, test, self.split_path, self.image_dir)
105
- except Exception as e2:
106
- warnings.warn(f"GTSRB: failed to save rebuilt split to '{self.split_path}': {e2}")
107
- else:
108
- train, val, test = self.read_and_split_data(self.image_dir)
109
- try:
110
- OxfordPets.save_split(train, val, test, self.split_path, self.image_dir)
111
- except Exception as e:
112
- warnings.warn(f"GTSRB: failed to save split to '{self.split_path}': {e}")
113
-
114
- _dbg(f"loaded counts: train={len(train)}, val={len(val)}, test={len(test)}")
115
-
116
- if num_shots >= 1:
117
- preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
118
- if os.path.exists(preprocessed):
119
- print(f"Loading preprocessed few-shot data from {preprocessed}")
120
- with open(preprocessed, "rb") as file:
121
- data = pickle.load(file)
122
- train, val = data["train"], data["val"]
123
- else:
124
- train = self.generate_fewshot_dataset(train, num_shots=num_shots)
125
- val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
126
- data = {"train": train, "val": val}
127
- print(f"Saving preprocessed few-shot data to {preprocessed}")
128
- with open(preprocessed, "wb") as file:
129
- pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)
130
- _dbg(f"few-shot applied: train={len(train)}, val={len(val)} (shots={num_shots})")
131
-
132
- # Ensure class subsampling behavior matches others
133
- train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample_classes)
134
- _dbg(f"after subsample='{subsample_classes}': train={len(train)}, val={len(val)}, test={len(test)})")
135
-
136
- # Prompt templates provided by user
137
- self.templates = [
138
- lambda c: f'a zoomed in photo of a "{c}" traffic sign.',
139
- lambda c: f'a centered photo of a "{c}" traffic sign.',
140
- lambda c: f'a close up photo of a "{c}" traffic sign.',
141
- ]
142
-
143
- super().__init__(train_x=train, val=val, test=test)
144
-
145
- @staticmethod
146
- def read_and_split_data(image_root, p_trn=0.5, p_val=0.2):
147
- # Discover class directories (e.g., 00000 .. 00042)
148
- try:
149
- all_entries = listdir_nohidden(image_root, sort=True)
150
- except Exception as e:
151
- warnings.warn(f"GTSRB: failed to list directory '{image_root}': {e}")
152
- raise
153
- class_dirs = [d for d in all_entries if os.path.isdir(os.path.join(image_root, d))]
154
- class_dirs.sort()
155
- assert len(class_dirs) > 0, f"GTSRB: no class folders found under {image_root}"
156
- _dbg(f"found {len(class_dirs)} class folders; head={class_dirs[:5]}")
157
-
158
- # Map sorted class dirs to labels 0..N-1 and names from 'classes'
159
- if len(class_dirs) != len(classes):
160
- print(f"Warning: detected {len(class_dirs)} class folders but classes list has {len(classes)} entries. Proceeding with min overlap.")
161
- num_labels = min(len(class_dirs), len(classes))
162
-
163
- def _collate(paths, y, cname):
164
- return [Datum(impath=p, label=y, classname=cname) for p in paths]
165
-
166
- train, val, test = [], [], []
167
- for y, cls_dir in enumerate(class_dirs[:num_labels]):
168
- cname = classes[y]
169
- cdir = os.path.join(image_root, cls_dir)
170
- try:
171
- files = listdir_nohidden(cdir, sort=False)
172
- except Exception as e:
173
- warnings.warn(f"GTSRB: failed to list class folder '{cdir}': {e}")
174
- files = []
175
- imgs = [os.path.join(cdir, f) for f in files if _is_image_file(f)]
176
- random.shuffle(imgs)
177
- n_total = len(imgs)
178
- if n_total == 0:
179
- warnings.warn(f"GTSRB: empty or unreadable class folder: {cdir}; skipping this class")
180
- continue
181
- if n_total < 5:
182
- warnings.warn(f"GTSRB: very few images in class '{cls_dir}' (n={n_total}); splits may be unstable")
183
- n_train = round(n_total * p_trn)
184
- n_val = round(n_total * p_val)
185
- n_test = n_total - n_train - n_val
186
- if not (n_train > 0 and n_val > 0 and n_test > 0):
187
- warnings.warn(f"GTSRB: split would create empty split for class {cls_dir} (n={n_total}); adjusting strategy to keep at least 1 per split")
188
- # Fallback: enforce at least 1 per split if possible
189
- if n_total >= 3:
190
- n_train, n_val, n_test = 1, 1, n_total - 2
191
- elif n_total == 2:
192
- n_train, n_val, n_test = 1, 1, 0
193
- else: # n_total == 1
194
- n_train, n_val, n_test = 1, 0, 0
195
-
196
- train.extend(_collate(imgs[:n_train], y, cname))
197
- val.extend(_collate(imgs[n_train:n_train + n_val], y, cname))
198
- test.extend(_collate(imgs[n_train + n_val:], y, cname))
199
- _dbg(f"class {cls_dir} -> label {y}: total={n_total}, train={n_train}, val={n_val}, test={n_test}")
200
-
201
- _dbg(f"aggregate sizes: train={len(train)}, val={len(val)}, test={len(test)}")
202
- return train, val, test
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/MTIL_datasets/hatefulmemes.py DELETED
@@ -1,117 +0,0 @@
1
- import os
2
- import json
3
- import pickle
4
-
5
- from .utils import *
6
- from .oxford_pets import OxfordPets
7
-
8
-
9
- class HatefulMemes(DatasetBase):
10
- """
11
- Hateful Memes dataset loader (image-only for MTIL).
12
-
13
- Expected structure:
14
- <root>/hatefulmemes/
15
- ├─ img/*.png|jpg
16
- ├─ train.jsonl
17
- ├─ dev.jsonl
18
- └─ test.jsonl
19
-
20
- JSONL lines example:
21
- {"id":85362, "img":"img/85362.png", "label":0, "text":"..."}
22
-
23
- We DO NOT use the 'text' field. Only image path and label are used.
24
-
25
- Classes: ['meme', 'hatespeech meme']
26
- Templates: ['a {}.']
27
- """
28
-
29
- dataset_dir = "hatefulmemes"
30
-
31
- def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'):
32
- root = os.path.abspath(os.path.expanduser(root))
33
- self.dataset_dir = os.path.join(root, self.dataset_dir)
34
- if not os.path.isdir(self.dataset_dir):
35
- # allow alias without trailing 's'
36
- alt = os.path.join(root, "hatefulmeme")
37
- if os.path.isdir(alt):
38
- self.dataset_dir = alt
39
- else:
40
- raise ValueError(
41
- f"HatefulMemes: dataset folder not found at '{self.dataset_dir}' or '{alt}'"
42
- )
43
-
44
- split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot")
45
- mkdir_if_missing(split_fewshot_dir)
46
-
47
- # paths
48
- train_json = os.path.join(self.dataset_dir, "train.jsonl")
49
- dev_json = os.path.join(self.dataset_dir, "dev.jsonl")
50
- test_json = os.path.join(self.dataset_dir, "test.jsonl")
51
-
52
- if not os.path.isfile(train_json) or not os.path.isfile(dev_json) or not os.path.isfile(test_json):
53
- raise ValueError(
54
- f"HatefulMemes: missing jsonl files. Expected train/dev/test at '{self.dataset_dir}'."
55
- )
56
-
57
- # fixed classes
58
- classes = ["meme", "hatespeech meme"]
59
- lab_to_name = {0: classes[0], 1: classes[1]}
60
-
61
- # read splits
62
- train = self._read_jsonl(train_json, lab_to_name)
63
- val = self._read_jsonl(dev_json, lab_to_name)
64
- test = self._read_jsonl(test_json, lab_to_name)
65
- # Some public releases of Hateful Memes do not include labels for test.jsonl.
66
- # In that case, fallback to use dev.jsonl for evaluation so zero-shot works.
67
- if len(test) == 0 and len(val) > 0:
68
- test = list(val)
69
-
70
- # few-shot
71
- if num_shots >= 1:
72
- preprocessed = os.path.join(split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
73
- if os.path.exists(preprocessed):
74
- with open(preprocessed, "rb") as file:
75
- data = pickle.load(file)
76
- train, val = data["train"], data["val"]
77
- else:
78
- train = self.generate_fewshot_dataset(train, num_shots=num_shots)
79
- val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
80
- data = {"train": train, "val": val}
81
- with open(preprocessed, "wb") as file:
82
- pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)
83
-
84
- # subsample behavior consistent with others
85
- train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample_classes)
86
-
87
- # templates per user spec
88
- self.templates = [
89
- lambda c: f'a {c}.',
90
- ]
91
-
92
- super().__init__(train_x=train, val=val, test=test)
93
-
94
- def _read_jsonl(self, filepath, lab_to_name):
95
- items = []
96
- with open(filepath, 'r', encoding='utf-8') as f:
97
- for line in f:
98
- line = line.strip()
99
- if not line:
100
- continue
101
- try:
102
- obj = json.loads(line)
103
- except Exception:
104
- continue
105
- # fields: id, img, label, text (ignored)
106
- img_rel = obj.get('img', '')
107
- label = obj.get('label', None)
108
- if img_rel is None or label is None:
109
- continue
110
- if label not in (0, 1):
111
- raise ValueError(f"HatefulMemes: unexpected label {label} in {filepath}")
112
- impath = os.path.join(self.dataset_dir, img_rel.replace('/', os.sep))
113
- if not os.path.isfile(impath):
114
- continue
115
- cname = lab_to_name[label]
116
- items.append(Datum(impath=impath, label=label, classname=cname))
117
- return items
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/MTIL_datasets/imagenet_r.py DELETED
@@ -1,231 +0,0 @@
1
- import os
2
- import random
3
- from typing import List, Tuple, Dict
4
-
5
- from .utils import * # Datum, DatasetBase, listdir_nohidden, mkdir_if_missing
6
- from .oxford_pets import OxfordPets
7
-
8
-
9
- IMAGENETR_TEMPLATES: List[str] = [
10
- 'a bad photo of a {}.',
11
- 'a photo of many {}.',
12
- 'a sculpture of a {}.',
13
- 'a photo of the hard to see {}.',
14
- 'a low resolution photo of the {}.',
15
- 'a rendering of a {}.',
16
- 'graffiti of a {}.',
17
- 'a bad photo of the {}.',
18
- 'a cropped photo of the {}.',
19
- 'a tattoo of a {}.',
20
- 'the embroidered {}.',
21
- 'a photo of a hard to see {}.',
22
- 'a bright photo of a {}.',
23
- 'a photo of a clean {}.',
24
- 'a photo of a dirty {}.',
25
- 'a dark photo of the {}.',
26
- 'a drawing of a {}.',
27
- 'a photo of my {}.',
28
- 'the plastic {}.',
29
- 'a photo of the cool {}.',
30
- 'a close-up photo of a {}.',
31
- 'a black and white photo of the {}.',
32
- 'a painting of the {}.',
33
- 'a painting of a {}.',
34
- 'a pixelated photo of the {}.',
35
- 'a sculpture of the {}.',
36
- 'a bright photo of the {}.',
37
- 'a cropped photo of a {}.',
38
- 'a plastic {}.',
39
- 'a photo of the dirty {}.',
40
- 'a jpeg corrupted photo of a {}.',
41
- 'a blurry photo of the {}.',
42
- 'a photo of the {}.',
43
- 'a good photo of the {}.',
44
- 'a rendering of the {}.',
45
- 'a {} in a video game.',
46
- 'a photo of one {}.',
47
- 'a doodle of a {}.',
48
- 'a close-up photo of the {}.',
49
- 'a photo of a {}.',
50
- 'the origami {}.',
51
- 'the {} in a video game.',
52
- 'a sketch of a {}.',
53
- 'a doodle of the {}.',
54
- 'a origami {}.',
55
- 'a low resolution photo of a {}.',
56
- 'the toy {}.',
57
- 'a rendition of the {}.',
58
- 'a photo of the clean {}.',
59
- 'a photo of a large {}.',
60
- 'a rendition of a {}.',
61
- 'a photo of a nice {}.',
62
- 'a photo of a weird {}.',
63
- 'a blurry photo of a {}.',
64
- 'a cartoon {}.',
65
- 'art of a {}.',
66
- 'a sketch of the {}.',
67
- 'a embroidered {}.',
68
- 'a pixelated photo of a {}.',
69
- 'itap of the {}.',
70
- 'a jpeg corrupted photo of the {}.',
71
- 'a good photo of a {}.',
72
- 'a plushie {}.',
73
- 'a photo of the nice {}.',
74
- 'a photo of the small {}.',
75
- 'a photo of the weird {}.',
76
- 'the cartoon {}.',
77
- 'art of the {}.',
78
- 'a drawing of the {}.',
79
- 'a photo of the large {}.',
80
- 'a black and white photo of a {}.',
81
- 'the plushie {}.',
82
- 'a dark photo of a {}.',
83
- 'itap of a {}.',
84
- 'graffiti of the {}.',
85
- 'a toy {}.',
86
- 'itap of my {}.',
87
- 'a photo of a cool {}.',
88
- 'a photo of a small {}.',
89
- 'a tattoo of the {}.',
90
- ]
91
-
92
-
93
- class ImageNetR(DatasetBase):
94
-
95
- dataset_dir = "imagenet-r"
96
-
97
- def __init__(self, root: str, num_shots: int = 0, seed: int = 1, subsample_classes: str = 'all', test_ratio: float = 0.2):
98
- """
99
- Expect directory structure:
100
- {root}/imagenet-r/<wnid>/*.jpg|png|jpeg|bmp|webp
101
- {root}/imagenet-r/classname.txt # lines: "<wnid> <human_readable_name>"
102
- There is no official test split; we perform a per-class split into train/test, then
103
- split train into train/val using OxfordPets.split_trainval.
104
- """
105
- rnd = random.Random(seed)
106
-
107
- root = os.path.abspath(os.path.expanduser(root))
108
- self.dataset_dir = os.path.join(root, self.dataset_dir)
109
- self.split_path = os.path.join(self.dataset_dir, "split_custom_ImageNetR.json")
110
- self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot")
111
- mkdir_if_missing(self.split_fewshot_dir)
112
-
113
- # Read class mapping from classname.txt
114
- class_map_path = os.path.join(self.dataset_dir, "classname.txt")
115
- if not os.path.exists(class_map_path):
116
- raise FileNotFoundError(f"Class mapping file not found: {class_map_path}")
117
- wnids, classnames = self._read_class_map(class_map_path)
118
- self._wnids = wnids
119
- self._classnames_ref = classnames
120
-
121
- # Build or load split
122
- if os.path.exists(self.split_path):
123
- train, val, test = OxfordPets.read_split(self.split_path, self.dataset_dir)
124
- else:
125
- trainval, test = self._read_from_folders(self.dataset_dir, wnids, classnames, rnd=rnd, test_ratio=test_ratio)
126
- train, val = self._split_trainval_safe(trainval)
127
- OxfordPets.save_split(train, val, test, self.split_path, self.dataset_dir)
128
-
129
- if num_shots >= 1:
130
- preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
131
- if os.path.exists(preprocessed):
132
- print(f"Loading preprocessed few-shot data from {preprocessed}")
133
- import pickle
134
- with open(preprocessed, "rb") as file:
135
- data = pickle.load(file)
136
- train, val = data["train"], data["val"]
137
- else:
138
- train = self.generate_fewshot_dataset(train, num_shots=num_shots)
139
- val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
140
- data = {"train": train, "val": val}
141
- print(f"Saving preprocessed few-shot data to {preprocessed}")
142
- import pickle
143
- with open(preprocessed, "wb") as file:
144
- pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)
145
-
146
- train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample_classes)
147
-
148
- # Expose templates
149
- self.templates = IMAGENETR_TEMPLATES
150
-
151
- super().__init__(train_x=train, val=val, test=test)
152
-
153
- @staticmethod
154
- def _read_class_map(filepath: str) -> Tuple[List[str], List[str]]:
155
- wnids: List[str] = []
156
- cnames: List[str] = []
157
- with open(filepath, 'r', encoding='utf-8') as f:
158
- for line in f:
159
- line = line.strip()
160
- if not line:
161
- continue
162
- parts = line.split()
163
- wnid = parts[0]
164
- cname = ' '.join(parts[1:]) if len(parts) > 1 else wnid
165
- wnids.append(wnid)
166
- cnames.append(cname)
167
- return wnids, cnames
168
-
169
- @staticmethod
170
- def _split_trainval_safe(trainval: List[Datum], p_val: float = 0.2) -> Tuple[List[Datum], List[Datum]]:
171
- from collections import defaultdict
172
- tracker: Dict[int, List[int]] = defaultdict(list)
173
- for idx, item in enumerate(trainval):
174
- tracker[item.label].append(idx)
175
-
176
- train, val = [], []
177
- for label, idxs in tracker.items():
178
- n = len(idxs)
179
- if n <= 1:
180
- for idx in idxs:
181
- train.append(trainval[idx])
182
- continue
183
- n_val = max(1, int(round(n * p_val)))
184
- if n_val >= n:
185
- n_val = n - 1
186
- random.shuffle(idxs)
187
- for i, idx in enumerate(idxs):
188
- if i < n_val:
189
- val.append(trainval[idx])
190
- else:
191
- train.append(trainval[idx])
192
- if len(val) == 0 and len(train) > 0:
193
- val.append(train[-1])
194
- train = train[:-1]
195
- return train, val
196
-
197
- def _read_from_folders(self, data_dir: str, wnids: List[str], classnames: List[str], rnd: random.Random, test_ratio: float) -> Tuple[List[Datum], List[Datum]]:
198
- exts = {".png", ".jpg", ".jpeg", ".bmp", ".webp"}
199
- class_to_label = {wnid: i for i, wnid in enumerate(wnids)}
200
- label_to_cname = {i: classnames[i] for i in range(len(classnames))}
201
-
202
- items_by_label: Dict[int, List[Datum]] = {i: [] for i in range(len(wnids))}
203
- for wnid in wnids:
204
- cdir = os.path.join(data_dir, wnid)
205
- if not os.path.isdir(cdir):
206
- # If a class listed in mapping has no folder, skip gracefully
207
- continue
208
- label = class_to_label[wnid]
209
- cname = label_to_cname[label]
210
- for fname in listdir_nohidden(cdir, sort=True):
211
- fext = os.path.splitext(fname)[1].lower()
212
- if fext not in exts:
213
- continue
214
- impath = os.path.join(cdir, fname)
215
- items_by_label[label].append(Datum(impath=impath, label=label, classname=cname))
216
-
217
- trainval: List[Datum] = []
218
- test: List[Datum] = []
219
- for label, items in items_by_label.items():
220
- if not items:
221
- continue
222
- rnd.shuffle(items)
223
- if len(items) == 1:
224
- trainval.extend(items)
225
- continue
226
- n_test = max(1, int(round(len(items) * test_ratio)))
227
- if n_test >= len(items):
228
- n_test = len(items) - 1
229
- test.extend(items[:n_test])
230
- trainval.extend(items[n_test:])
231
- return trainval, test
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/MTIL_datasets/kitti_distance.py DELETED
@@ -1,228 +0,0 @@
1
- import os
2
- import random
3
- from typing import List, Tuple
4
-
5
- from .utils import * # Datum, DatasetBase, listdir_nohidden, mkdir_if_missing
6
- from .oxford_pets import OxfordPets
7
-
8
- # Class names per user specification (ordered)
9
- KITTI_DISTANCE_CLASSES: List[str] = [
10
- 'a photo i took of a car nearby',
11
- 'a photo i took with a car in the middle distance',
12
- 'a photo i took with a car faraway',
13
- 'a photo i took with no car.',
14
- ]
15
-
16
- # Keep templates as strings with {}
17
- KITTI_DISTANCE_TEMPLATES: List[str] = [
18
- '{}',
19
- ]
20
-
21
-
22
- class KittiDistance(DatasetBase):
23
-
24
- dataset_dir = "kitti"
25
-
26
- def __init__(self, root, num_shots=0, seed=1, subsample_classes='all', test_ratio: float = 0.2):
27
- """
28
- Predict the distance category of the closest car from KITTI labels.
29
-
30
- Expected structure:
31
- {root}/kitti/
32
- image/000004.png
33
- label/000004.txt
34
-
35
- Label file format: KITTI object detection label lines.
36
- We retain only lines with type == 'Car'.
37
- For these, we read the 14th column (index 13, loc_z) as the camera-depth in meters.
38
- We take the minimum positive loc_z across all 'Car' lines as z_min for the image.
39
- If there is no valid positive loc_z for 'Car', we assign the 'no car' class.
40
-
41
- Discretization into classes:
42
- 0: 0 < z_min < 10 -> 'nearby'
43
- 1: 10 <= z_min < 30 -> 'middle distance'
44
- 2: z_min >= 30 -> 'faraway'
45
- 3: no car -> 'no car.'
46
-
47
- No official test set; we split randomly (per class) into train/test by test_ratio.
48
- Then we split train into train/val using a safe per-class split.
49
- Splits are saved to JSON for reproducibility.
50
- """
51
- rnd = random.Random(seed)
52
-
53
- root = os.path.abspath(os.path.expanduser(root))
54
- self.dataset_dir = os.path.join(root, self.dataset_dir)
55
-
56
- # Image and label directories
57
- image_dir = os.path.join(self.dataset_dir, 'image')
58
- if not os.path.isdir(image_dir):
59
- # Graceful fallback to standard KITTI naming variants if provided
60
- alt1 = os.path.join(self.dataset_dir, 'image_2')
61
- alt2 = os.path.join(self.dataset_dir, 'images')
62
- if os.path.isdir(alt1):
63
- image_dir = alt1
64
- elif os.path.isdir(alt2):
65
- image_dir = alt2
66
- label_dir = os.path.join(self.dataset_dir, 'label')
67
- if not os.path.isdir(label_dir):
68
- alt_l1 = os.path.join(self.dataset_dir, 'label_2')
69
- alt_l2 = os.path.join(self.dataset_dir, 'labels')
70
- if os.path.isdir(alt_l1):
71
- label_dir = alt_l1
72
- elif os.path.isdir(alt_l2):
73
- label_dir = alt_l2
74
-
75
- if not os.path.isdir(image_dir):
76
- raise FileNotFoundError(f"KittiDistance: image dir not found: {image_dir}")
77
- if not os.path.isdir(label_dir):
78
- raise FileNotFoundError(f"KittiDistance: label dir not found: {label_dir}")
79
-
80
- self.image_dir = image_dir
81
- self.label_dir = label_dir
82
-
83
- self.split_path = os.path.join(self.dataset_dir, "split_custom_KittiDistance.json")
84
- self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot")
85
- mkdir_if_missing(self.split_fewshot_dir)
86
-
87
- if os.path.exists(self.split_path):
88
- train, val, test = OxfordPets.read_split(self.split_path, self.image_dir)
89
- else:
90
- trainval, test = self._read_and_split(self.image_dir, self.label_dir, rnd=rnd, test_ratio=test_ratio)
91
- train, val = self._split_trainval_safe(trainval)
92
- OxfordPets.save_split(train, val, test, self.split_path, self.image_dir)
93
-
94
- if num_shots >= 1:
95
- preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
96
- if os.path.exists(preprocessed):
97
- print(f"Loading preprocessed few-shot data from {preprocessed}")
98
- import pickle
99
- with open(preprocessed, "rb") as file:
100
- data = pickle.load(file)
101
- train, val = data["train"], data["val"]
102
- else:
103
- train = self.generate_fewshot_dataset(train, num_shots=num_shots)
104
- val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
105
- data = {"train": train, "val": val}
106
- print(f"Saving preprocessed few-shot data to {preprocessed}")
107
- import pickle
108
- with open(preprocessed, "wb") as file:
109
- pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)
110
-
111
- # Allow class subsampling if requested
112
- train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample_classes)
113
-
114
- # Templates per user specification
115
- self.templates = KITTI_DISTANCE_TEMPLATES
116
-
117
- super().__init__(train_x=train, val=val, test=test)
118
- # Override inferred metadata to ensure stable 4-way classification regardless of train label coverage
119
- self._classnames = KITTI_DISTANCE_CLASSES
120
- self._lab2cname = {i: c for i, c in enumerate(KITTI_DISTANCE_CLASSES)}
121
- self._num_classes = len(KITTI_DISTANCE_CLASSES)
122
-
123
- def _read_and_split(self, image_dir: str, label_dir: str, rnd: random.Random, test_ratio: float) -> Tuple[List[Datum], List[Datum]]:
124
- exts = {".png", ".jpg", ".jpeg", ".bmp", ".webp"}
125
- # Collect items with computed labels
126
- items_by_label = {i: [] for i in range(len(KITTI_DISTANCE_CLASSES))}
127
-
128
- for fname in listdir_nohidden(image_dir, sort=True):
129
- fext = os.path.splitext(fname)[1].lower()
130
- if fext not in exts:
131
- continue
132
- impath = os.path.join(image_dir, fname)
133
- stem = os.path.splitext(fname)[0]
134
- label_path = os.path.join(label_dir, stem + '.txt')
135
- label_idx = self._compute_label_from_kitti(label_path)
136
- cname = KITTI_DISTANCE_CLASSES[label_idx]
137
- items_by_label[label_idx].append(Datum(impath=impath, label=label_idx, classname=cname))
138
-
139
- # Per-class balanced split into trainval/test
140
- trainval, test = [], []
141
- for label, items in items_by_label.items():
142
- if not items:
143
- continue
144
- rnd.shuffle(items)
145
- if len(items) == 1:
146
- trainval.extend(items)
147
- continue
148
- n_test = max(1, int(round(len(items) * test_ratio)))
149
- if n_test >= len(items):
150
- n_test = len(items) - 1
151
- test.extend(items[:n_test])
152
- trainval.extend(items[n_test:])
153
-
154
- return trainval, test
155
-
156
- @staticmethod
157
- def _compute_label_from_kitti(label_path: str) -> int:
158
- """
159
- Parse KITTI label file; focus only on lines with type == 'Car'.
160
- Extract the 14th column (index 13, loc_z) as meters; consider only positive values.
161
- If no positive loc_z found -> class 'no car' (index 3).
162
-
163
- Thresholds:
164
- 0: 0 < z < 10
165
- 1: 10 <= z < 30
166
- 2: z >= 30
167
- """
168
- z_vals = []
169
- if os.path.isfile(label_path):
170
- with open(label_path, 'r') as f:
171
- for line in f:
172
- line = line.strip()
173
- if not line:
174
- continue
175
- parts = line.split()
176
- obj_type = parts[0]
177
- if obj_type != 'Car':
178
- continue
179
- if len(parts) < 14:
180
- # Not a valid KITTI detection line; skip
181
- continue
182
- try:
183
- # parts[13] is loc_z (0-based index), per KITTI format
184
- z = float(parts[13])
185
- if z > 0:
186
- z_vals.append(z)
187
- except Exception:
188
- continue
189
- # Determine class index
190
- if not z_vals:
191
- return 3 # no car
192
- z_min = min(z_vals)
193
- if z_min < 10:
194
- return 0
195
- elif z_min < 30:
196
- return 1
197
- else:
198
- return 2
199
-
200
- @staticmethod
201
- def _split_trainval_safe(trainval: List[Datum], p_val: float = 0.2) -> Tuple[List[Datum], List[Datum]]:
202
- from collections import defaultdict
203
- tracker = defaultdict(list)
204
- for idx, item in enumerate(trainval):
205
- tracker[item.label].append(idx)
206
-
207
- train, val = [], []
208
- for label, idxs in tracker.items():
209
- n = len(idxs)
210
- if n <= 1:
211
- for idx in idxs:
212
- train.append(trainval[idx])
213
- continue
214
- n_val = max(1, int(round(n * p_val)))
215
- if n_val >= n:
216
- n_val = n - 1
217
- random.shuffle(idxs)
218
- for i, idx in enumerate(idxs):
219
- if i < n_val:
220
- val.append(trainval[idx])
221
- else:
222
- train.append(trainval[idx])
223
-
224
- if len(val) == 0 and len(train) > 0:
225
- val.append(train[-1])
226
- train = train[:-1]
227
-
228
- return train, val
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/MTIL_datasets/mnist.py DELETED
@@ -1,132 +0,0 @@
1
- import os
2
- import pickle
3
-
4
- from .utils import *
5
-
6
- from .oxford_pets import OxfordPets
7
- from .dtd import DescribableTextures as DTD
8
- from .oxford_pets import OxfordPets
9
-
10
- import torch
11
- import codecs
12
- import numpy as np
13
- import sys
14
-
15
- classes = [
16
- "0 - zero",
17
- "1 - one",
18
- "2 - two",
19
- "3 - three",
20
- "4 - four",
21
- "5 - five",
22
- "6 - six",
23
- "7 - seven",
24
- "8 - eight",
25
- "9 - nine",
26
- ]
27
-
28
- class MNIST(DatasetBase):
29
-
30
- dataset_dir = "mnist"
31
-
32
- def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'):
33
- root = os.path.abspath(os.path.expanduser(root))
34
- self.dataset_dir = os.path.join(root, self.dataset_dir)
35
- self.image_dir = self.dataset_dir
36
-
37
- trainval_image_file = "train-images-idx3-ubyte"
38
- trainval_data = read_image_file(os.path.join(self.image_dir, trainval_image_file)) # Size([60000, 28, 28]) torch.uint8
39
- trainval_label_file = "train-labels-idx1-ubyte"
40
- trainval_targets = read_label_file(os.path.join(self.image_dir, trainval_label_file)) # Size([60000]) torch.int64
41
- # trainval_names =
42
-
43
- test_image_file = "t10k-images-idx3-ubyte"
44
- test_data = read_image_file(os.path.join(self.image_dir, test_image_file))
45
- test_label_file = "t10k-labels-idx1-ubyte"
46
- test_targets = read_label_file(os.path.join(self.image_dir, test_label_file))
47
-
48
- trainval = []
49
- for idx in range(trainval_data.size(0)):
50
- item = Datum(impath=Image.fromarray(trainval_data[idx].numpy(), mode="L"),
51
- label=int(trainval_targets[idx]), classname=classes[trainval_targets[idx]])
52
- trainval.append(item)
53
-
54
- test = []
55
- for idx in range(test_data.size(0)):
56
- item = Datum(impath=Image.fromarray(test_data[idx].numpy(), mode="L"),
57
- label=int(test_targets[idx]), classname=classes[test_targets[idx]])
58
- test.append(item)
59
-
60
- train, val = OxfordPets.split_trainval(trainval)
61
-
62
- if num_shots >= 1:
63
- train = self.generate_fewshot_dataset(train, num_shots=num_shots)
64
- val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
65
-
66
- subsample = subsample_classes
67
- train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample)
68
-
69
- self.templates = [
70
- lambda c: f'a photo of the number: "{c}".',
71
- ]
72
-
73
- super().__init__(train_x=train, val=val, test=test)
74
-
75
-
76
- def _flip_byte_order(t: torch.Tensor) -> torch.Tensor:
77
- return (
78
- t.contiguous().view(torch.uint8).view(*t.shape, t.element_size()).flip(-1).view(*t.shape[:-1], -1).view(t.dtype)
79
- )
80
-
81
- def get_int(b: bytes) -> int:
82
- return int(codecs.encode(b, "hex"), 16)
83
-
84
- SN3_PASCALVINCENT_TYPEMAP = {
85
- 8: torch.uint8,
86
- 9: torch.int8,
87
- 11: torch.int16,
88
- 12: torch.int32,
89
- 13: torch.float32,
90
- 14: torch.float64,
91
- }
92
-
93
- def read_sn3_pascalvincent_tensor(path: str, strict: bool = True) -> torch.Tensor:
94
- """Read a SN3 file in "Pascal Vincent" format (Lush file 'libidx/idx-io.lsh').
95
- Argument may be a filename, compressed filename, or file object.
96
- """
97
- # read
98
- with open(path, "rb") as f:
99
- data = f.read()
100
- # parse
101
- magic = get_int(data[0:4])
102
- nd = magic % 256
103
- ty = magic // 256
104
- assert 1 <= nd <= 3
105
- assert 8 <= ty <= 14
106
- torch_type = SN3_PASCALVINCENT_TYPEMAP[ty]
107
- s = [get_int(data[4 * (i + 1) : 4 * (i + 2)]) for i in range(nd)]
108
-
109
- parsed = torch.frombuffer(bytearray(data), dtype=torch_type, offset=(4 * (nd + 1)))
110
- if sys.byteorder == "little" and parsed.element_size() > 1:
111
- parsed = _flip_byte_order(parsed)
112
-
113
- assert parsed.shape[0] == np.prod(s) or not strict
114
- return parsed.view(*s)
115
-
116
-
117
- def read_label_file(path: str) -> torch.Tensor:
118
- x = read_sn3_pascalvincent_tensor(path, strict=False)
119
- if x.dtype != torch.uint8:
120
- raise TypeError(f"x should be of dtype torch.uint8 instead of {x.dtype}")
121
- if x.ndimension() != 1:
122
- raise ValueError(f"x should have 1 dimension instead of {x.ndimension()}")
123
- return x.long()
124
-
125
-
126
- def read_image_file(path: str) -> torch.Tensor:
127
- x = read_sn3_pascalvincent_tensor(path, strict=False)
128
- if x.dtype != torch.uint8:
129
- raise TypeError(f"x should be of dtype torch.uint8 instead of {x.dtype}")
130
- if x.ndimension() != 3:
131
- raise ValueError(f"x should have 3 dimension instead of {x.ndimension()}")
132
- return x
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/MTIL_datasets/oxford_flowers.py DELETED
@@ -1,89 +0,0 @@
1
- import os
2
- import pickle
3
- import random
4
- from scipy.io import loadmat
5
- from collections import defaultdict
6
-
7
- from .utils import *
8
-
9
- from .oxford_pets import OxfordPets
10
-
11
-
12
- class OxfordFlowers(DatasetBase):
13
-
14
- dataset_dir = "oxford_flowers"
15
-
16
- def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'):
17
- root = os.path.abspath(os.path.expanduser(root))
18
- self.dataset_dir = os.path.join(root, self.dataset_dir)
19
- self.image_dir = os.path.join(self.dataset_dir, "jpg")
20
- self.label_file = os.path.join(self.dataset_dir, "imagelabels.mat")
21
- self.lab2cname_file = os.path.join(self.dataset_dir, "cat_to_name.json")
22
- self.split_path = os.path.join(self.dataset_dir, "split_zhou_OxfordFlowers.json")
23
- self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot")
24
- mkdir_if_missing(self.split_fewshot_dir)
25
-
26
- if os.path.exists(self.split_path):
27
- train, val, test = OxfordPets.read_split(self.split_path, self.image_dir)
28
- else:
29
- train, val, test = self.read_data()
30
- OxfordPets.save_split(train, val, test, self.split_path, self.image_dir)
31
-
32
- if num_shots >= 1:
33
- preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
34
-
35
- if os.path.exists(preprocessed):
36
- print(f"Loading preprocessed few-shot data from {preprocessed}")
37
- with open(preprocessed, "rb") as file:
38
- data = pickle.load(file)
39
- train, val = data["train"], data["val"]
40
- else:
41
- train = self.generate_fewshot_dataset(train, num_shots=num_shots)
42
- val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
43
- data = {"train": train, "val": val}
44
- print(f"Saving preprocessed few-shot data to {preprocessed}")
45
- with open(preprocessed, "wb") as file:
46
- pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)
47
-
48
- subsample = subsample_classes
49
- train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample)
50
-
51
- self.templates = [
52
- lambda c: f"a photo of a {c}, a type of flower.",
53
- ]
54
-
55
- super().__init__(train_x=train, val=val, test=test)
56
-
57
- def read_data(self):
58
- tracker = defaultdict(list)
59
- label_file = loadmat(self.label_file)["labels"][0]
60
- for i, label in enumerate(label_file):
61
- imname = f"image_{str(i + 1).zfill(5)}.jpg"
62
- impath = os.path.join(self.image_dir, imname)
63
- label = int(label)
64
- tracker[label].append(impath)
65
-
66
- print("Splitting data into 50% train, 20% val, and 30% test")
67
-
68
- def _collate(ims, y, c):
69
- items = []
70
- for im in ims:
71
- item = Datum(impath=im, label=y - 1, classname=c)
72
- items.append(item)
73
- return items
74
-
75
- lab2cname = read_json(self.lab2cname_file)
76
- train, val, test = [], [], []
77
- for label, impaths in tracker.items():
78
- random.shuffle(impaths)
79
- n_total = len(impaths)
80
- n_train = round(n_total * 0.5)
81
- n_val = round(n_total * 0.2)
82
- n_test = n_total - n_train - n_val
83
- assert n_train > 0 and n_val > 0 and n_test > 0
84
- cname = lab2cname[str(label)]
85
- train.extend(_collate(impaths[:n_train], label, cname))
86
- val.extend(_collate(impaths[n_train : n_train + n_val], label, cname))
87
- test.extend(_collate(impaths[n_train + n_val :], label, cname))
88
-
89
- return train, val, test
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/MTIL_datasets/oxford_pets.py DELETED
@@ -1,176 +0,0 @@
1
- import os
2
- import pickle
3
- import math
4
- import random
5
- from collections import defaultdict
6
-
7
- from .utils import *
8
-
9
-
10
- class OxfordPets(DatasetBase):
11
-
12
- dataset_dir = "oxford_pets"
13
-
14
- def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'):
15
- root = os.path.abspath(os.path.expanduser(root))
16
- self.dataset_dir = os.path.join(root, self.dataset_dir)
17
- self.image_dir = os.path.join(self.dataset_dir, "images")
18
- self.anno_dir = os.path.join(self.dataset_dir, "annotations")
19
- self.split_path = os.path.join(self.dataset_dir, "split_zhou_OxfordPets.json")
20
- self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot")
21
- mkdir_if_missing(self.split_fewshot_dir)
22
-
23
- if os.path.exists(self.split_path):
24
- train, val, test = self.read_split(self.split_path, self.image_dir)
25
- else:
26
- trainval = self.read_data(split_file="trainval.txt")
27
- test = self.read_data(split_file="test.txt")
28
- train, val = self.split_trainval(trainval)
29
- self.save_split(train, val, test, self.split_path, self.image_dir)
30
-
31
- if num_shots >= 1:
32
- preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
33
-
34
- if os.path.exists(preprocessed):
35
- print(f"Loading preprocessed few-shot data from {preprocessed}")
36
- with open(preprocessed, "rb") as file:
37
- data = pickle.load(file)
38
- train, val = data["train"], data["val"]
39
- else:
40
- train = self.generate_fewshot_dataset(train, num_shots=num_shots)
41
- val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
42
- data = {"train": train, "val": val}
43
- print(f"Saving preprocessed few-shot data to {preprocessed}")
44
- with open(preprocessed, "wb") as file:
45
- pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)
46
-
47
- subsample = subsample_classes
48
- train, val, test = self.subsample_classes(train, val, test, subsample=subsample)
49
-
50
- self.templates = [
51
- lambda c: f"a photo of a {c}, a type of pet.",
52
- ]
53
-
54
- super().__init__(train_x=train, val=val, test=test)
55
-
56
- def read_data(self, split_file):
57
- filepath = os.path.join(self.anno_dir, split_file)
58
- items = []
59
-
60
- with open(filepath, "r") as f:
61
- lines = f.readlines()
62
- for line in lines:
63
- line = line.strip()
64
- imname, label, species, _ = line.split(" ")
65
- breed = imname.split("_")[:-1]
66
- breed = "_".join(breed)
67
- breed = breed.lower()
68
- imname += ".jpg"
69
- impath = os.path.join(self.image_dir, imname)
70
- label = int(label) - 1 # convert to 0-based index
71
- item = Datum(impath=impath, label=label, classname=breed)
72
- items.append(item)
73
-
74
- return items
75
-
76
- @staticmethod
77
- def split_trainval(trainval, p_val=0.2):
78
- p_trn = 1 - p_val
79
- tracker = defaultdict(list)
80
- for idx, item in enumerate(trainval):
81
- label = item.label
82
- tracker[label].append(idx)
83
-
84
- train, val = [], []
85
- for label, idxs in tracker.items():
86
- n_val = round(len(idxs) * p_val)
87
- assert n_val > 0
88
- random.shuffle(idxs)
89
- for n, idx in enumerate(idxs):
90
- item = trainval[idx]
91
- if n < n_val:
92
- val.append(item)
93
- else:
94
- train.append(item)
95
-
96
- return train, val
97
-
98
- @staticmethod
99
- def save_split(train, val, test, filepath, path_prefix):
100
- def _extract(items):
101
- out = []
102
- for item in items:
103
- impath = item.impath
104
- label = item.label
105
- classname = item.classname
106
- impath = impath.replace(path_prefix, "")
107
- if impath.startswith("/"):
108
- impath = impath[1:]
109
- out.append((impath, label, classname))
110
- return out
111
-
112
- train = _extract(train)
113
- val = _extract(val)
114
- test = _extract(test)
115
-
116
- split = {"train": train, "val": val, "test": test}
117
-
118
- write_json(split, filepath)
119
- print(f"Saved split to {filepath}")
120
-
121
- @staticmethod
122
- def read_split(filepath, path_prefix):
123
- def _convert(items):
124
- out = []
125
- for impath, label, classname in items:
126
- impath = os.path.join(path_prefix, impath)
127
- item = Datum(impath=impath, label=int(label), classname=classname)
128
- out.append(item)
129
- return out
130
-
131
- print(f"Reading split from {filepath}")
132
- split = read_json(filepath)
133
- train = _convert(split["train"])
134
- val = _convert(split["val"])
135
- test = _convert(split["test"])
136
-
137
- return train, val, test
138
-
139
- @staticmethod
140
- def subsample_classes(*args, subsample="all"):
141
- assert subsample in ["all", "base", "new"]
142
-
143
- if subsample == "all":
144
- return args
145
-
146
- dataset = args[0]
147
- labels = set()
148
- for item in dataset:
149
- labels.add(item.label)
150
- labels = list(labels)
151
- labels.sort()
152
- n = len(labels)
153
- m = math.ceil(n / 2)
154
-
155
- print(f"SUBSAMPLE {subsample.upper()} CLASSES!")
156
- if subsample == "base":
157
- selected = labels[:m]
158
- else:
159
- selected = labels[m:]
160
- relabeler = {y: y_new for y_new, y in enumerate(selected)}
161
-
162
- output = []
163
- for dataset in args:
164
- dataset_new = []
165
- for item in dataset:
166
- if item.label not in selected:
167
- continue
168
- item_new = Datum(
169
- impath=item.impath,
170
- label=relabeler[item.label],
171
- classname=item.classname
172
- )
173
- dataset_new.append(item_new)
174
- output.append(dataset_new)
175
-
176
- return output
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/MTIL_datasets/pcam.py DELETED
@@ -1,198 +0,0 @@
1
- import os
2
- import pickle
3
- import random
4
- from typing import List, Tuple
5
-
6
- import h5py
7
- import numpy as np
8
-
9
- from .utils import Datum, DatasetBase, mkdir_if_missing, read_json, write_json
10
- from .oxford_pets import OxfordPets
11
-
12
-
13
- PCAM_CLASSES = [
14
- 'lymph node tissue without metastatic tumor',
15
- 'metastatic tumor in lymph node tissue',
16
- ]
17
-
18
- # Multiple domain-specific templates for histopathology microscopy images
19
- PCAM_TEMPLATES = [
20
- 'a microscopy image patch of {}',
21
- 'a histopathology image of {}',
22
- 'a hematoxylin and eosin stained image of {}',
23
- 'a high-resolution histology patch of {}',
24
- 'a digital pathology slide patch of {}',
25
- 'this is a microscopy image of {}',
26
- 'this is a histopathology image of {}',
27
- ]
28
-
29
-
30
- class PCam(DatasetBase):
31
-
32
- dataset_dir = 'pcam'
33
-
34
- def __init__(self, root, num_shots: int = 0, seed: int = 1, subsample_classes: str = 'all', val_ratio: float = 0.2):
35
- root = os.path.abspath(os.path.expanduser(root))
36
- self.dataset_dir = os.path.join(root, self.dataset_dir)
37
- mkdir_if_missing(self.dataset_dir)
38
-
39
- # HDF5 file paths
40
- self.train_x_path = os.path.join(self.dataset_dir, 'camelyonpatch_level_2_split_train_x.h5')
41
- self.train_y_path = os.path.join(self.dataset_dir, 'camelyonpatch_level_2_split_train_y.h5')
42
- self.test_x_path = os.path.join(self.dataset_dir, 'camelyonpatch_level_2_split_test_x.h5')
43
- self.test_y_path = os.path.join(self.dataset_dir, 'camelyonpatch_level_2_split_test_y.h5')
44
-
45
- for p in [self.train_x_path, self.train_y_path, self.test_x_path, self.test_y_path]:
46
- if not os.path.isfile(p):
47
- raise FileNotFoundError(f"PCam: expected file not found: {p}")
48
-
49
- self.split_path = os.path.join(self.dataset_dir, 'split_custom_PCam.json')
50
- self.split_fewshot_dir = os.path.join(self.dataset_dir, 'split_fewshot')
51
- mkdir_if_missing(self.split_fewshot_dir)
52
-
53
- random.seed(seed)
54
- np.random.seed(seed)
55
-
56
- if os.path.exists(self.split_path):
57
- train, val, test = self.read_split(self.split_path)
58
- else:
59
- trainval = self._read_train(self.train_x_path, self.train_y_path)
60
- test = self._read_test(self.test_x_path, self.test_y_path)
61
- train, val = self.split_trainval(trainval, p_val=val_ratio)
62
- self.save_split(train, val, test, self.split_path)
63
-
64
- if num_shots >= 1:
65
- preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
66
- if os.path.exists(preprocessed):
67
- print(f"Loading preprocessed few-shot data from {preprocessed}")
68
- with open(preprocessed, 'rb') as f:
69
- data = pickle.load(f)
70
- train, val = data['train'], data['val']
71
- else:
72
- train = self.generate_fewshot_dataset(train, num_shots=num_shots)
73
- val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
74
- data = {'train': train, 'val': val}
75
- print(f"Saving preprocessed few-shot data to {preprocessed}")
76
- with open(preprocessed, 'wb') as f:
77
- pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL)
78
-
79
- # Optional class subsampling (kept for consistency with other datasets)
80
- train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample_classes)
81
-
82
- # Templates per user specification
83
- self.templates = PCAM_TEMPLATES
84
-
85
- # Debug: print split sizes and label histograms
86
- def _hist(items):
87
- from collections import Counter
88
- cnt = Counter([it.label for it in items])
89
- # ensure keys present for both binary classes
90
- out = {i: int(cnt.get(i, 0)) for i in range(len(PCAM_CLASSES))}
91
- return out
92
-
93
- super().__init__(train_x=train, val=val, test=test)
94
- # Ensure stable binary classification metadata
95
- self._classnames = PCAM_CLASSES
96
- self._lab2cname = {i: c for i, c in enumerate(PCAM_CLASSES)}
97
- self._num_classes = len(PCAM_CLASSES)
98
-
99
- @staticmethod
100
- def _first_key(h5_path: str) -> str:
101
- with h5py.File(h5_path, 'r') as f:
102
- keys = list(f.keys())
103
- if not keys:
104
- raise RuntimeError(f"No datasets found in H5 file: {h5_path}")
105
- return keys[0]
106
-
107
- @staticmethod
108
- def _read_labels(h5_path: str) -> np.ndarray:
109
- key = PCam._first_key(h5_path)
110
- with h5py.File(h5_path, 'r') as f:
111
- y = f[key][...]
112
- y = np.asarray(y).squeeze()
113
- y = y.astype(np.int64)
114
- return y
115
-
116
- def _read_train(self, x_path: str, y_path: str) -> List[Datum]:
117
- x_key = self._first_key(x_path)
118
- y = self._read_labels(y_path)
119
- items: List[Datum] = []
120
- for i, label in enumerate(y.tolist()):
121
- label_i = int(label)
122
- classname = PCAM_CLASSES[label_i]
123
- # Lazy image reference: ('h5', abs_h5_path, dataset_key, index)
124
- abs_path = os.path.abspath(x_path)
125
- impath = ('h5', abs_path, x_key, i)
126
- items.append(Datum(impath=impath, label=label_i, classname=classname))
127
- return items
128
-
129
- def _read_test(self, x_path: str, y_path: str) -> List[Datum]:
130
- x_key = self._first_key(x_path)
131
- y = self._read_labels(y_path)
132
- items: List[Datum] = []
133
- for i, label in enumerate(y.tolist()):
134
- label_i = int(label)
135
- classname = PCAM_CLASSES[label_i]
136
- abs_path = os.path.abspath(x_path)
137
- impath = ('h5', abs_path, x_key, i)
138
- items.append(Datum(impath=impath, label=label_i, classname=classname))
139
- return items
140
-
141
- @staticmethod
142
- def split_trainval(trainval: List[Datum], p_val: float = 0.2) -> Tuple[List[Datum], List[Datum]]:
143
- from collections import defaultdict
144
- p_trn = 1 - p_val
145
- print(f"Splitting PCam train into {p_trn:.0%} train and {p_val:.0%} val")
146
- tracker = defaultdict(list)
147
- for idx, item in enumerate(trainval):
148
- tracker[item.label].append(idx)
149
- train, val = [], []
150
- for _, idxs in tracker.items():
151
- n_val = max(1, round(len(idxs) * p_val))
152
- random.shuffle(idxs)
153
- for n, i in enumerate(idxs):
154
- if n < n_val:
155
- val.append(trainval[i])
156
- else:
157
- train.append(trainval[i])
158
- return train, val
159
-
160
- def save_split(self, train: List[Datum], val: List[Datum], test: List[Datum], filepath: str):
161
- def _ser(items: List[Datum]):
162
- out = []
163
- for it in items:
164
- impath = it.impath
165
- if isinstance(impath, tuple) and len(impath) == 4 and impath[0] == 'h5':
166
- tag, abs_path, key, idx = impath
167
- # store relative path for portability
168
- rel = os.path.relpath(abs_path, self.dataset_dir)
169
- impath_ser = [tag, rel, key, int(idx)]
170
- else:
171
- raise ValueError('PCam expects H5 tuple paths')
172
- out.append((impath_ser, int(it.label), it.classname))
173
- return out
174
- split = {
175
- 'train': _ser(train),
176
- 'val': _ser(val),
177
- 'test': _ser(test),
178
- }
179
- write_json(split, filepath)
180
- print(f"Saved PCam split to {filepath}")
181
-
182
- def read_split(self, filepath: str):
183
- def _deser(items):
184
- out = []
185
- for impath_ser, label, classname in items:
186
- if isinstance(impath_ser, (list, tuple)) and len(impath_ser) == 4 and impath_ser[0] == 'h5':
187
- tag, rel, key, idx = impath_ser
188
- fpath = os.path.join(self.dataset_dir, rel)
189
- impath = (tag, fpath, key, int(idx))
190
- else:
191
- raise ValueError('PCam split contains invalid path entries')
192
- out.append(Datum(impath=impath, label=int(label), classname=classname))
193
- return out
194
- split = read_json(filepath)
195
- train = _deser(split['train'])
196
- val = _deser(split['val'])
197
- test = _deser(split['test'])
198
- return train, val, test
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/MTIL_datasets/resisc.py DELETED
@@ -1,199 +0,0 @@
1
- import os
2
- import pickle
3
- import random
4
- import warnings
5
-
6
- from .utils import *
7
- from .oxford_pets import OxfordPets
8
-
9
- # Canonical class list for RESISC45 (order defines label ids 0..44)
10
- RESISC_CLASSES = [
11
- 'airplane',
12
- 'airport',
13
- 'baseball diamond',
14
- 'basketball court',
15
- 'beach',
16
- 'bridge',
17
- 'chaparral',
18
- 'church',
19
- 'circular farmland',
20
- 'cloud',
21
- 'commercial area',
22
- 'dense residential',
23
- 'desert',
24
- 'forest',
25
- 'freeway',
26
- 'golf course',
27
- 'ground track field',
28
- 'harbor',
29
- 'industrial area',
30
- 'intersection',
31
- 'island',
32
- 'lake',
33
- 'meadow',
34
- 'medium residential',
35
- 'mobile home park',
36
- 'mountain',
37
- 'overpass',
38
- 'palace',
39
- 'parking lot',
40
- 'railway',
41
- 'railway station',
42
- 'rectangular farmland',
43
- 'river',
44
- 'roundabout',
45
- 'runway',
46
- 'sea ice',
47
- 'ship',
48
- 'snowberg',
49
- 'sparse residential',
50
- 'stadium',
51
- 'storage tank',
52
- 'tennis court',
53
- 'terrace',
54
- 'thermal power station',
55
- 'wetland',
56
- ]
57
-
58
- # Prompt templates (strings) as provided
59
- RESISC_TEMPLATES = [
60
- 'satellite imagery of {}.',
61
- 'aerial imagery of {}.',
62
- 'satellite photo of {}.',
63
- 'aerial photo of {}.',
64
- 'satellite view of {}.',
65
- 'aerial view of {}.',
66
- 'satellite imagery of a {}.',
67
- 'aerial imagery of a {}.',
68
- 'satellite photo of a {}.',
69
- 'aerial photo of a {}.',
70
- 'satellite view of a {}.',
71
- 'aerial view of a {}.',
72
- 'satellite imagery of the {}.',
73
- 'aerial imagery of the {}.',
74
- 'satellite photo of the {}.',
75
- 'aerial photo of the {}.',
76
- 'satellite view of the {}.',
77
- 'aerial view of the {}.',
78
- ]
79
-
80
- RESISC_DEBUG = os.environ.get("RESISC_DEBUG", "0") not in ("0", "false", "False", "")
81
-
82
- def _dbg(msg: str):
83
- if RESISC_DEBUG:
84
- print(f"[RESISC45][DEBUG] {msg}")
85
-
86
-
87
- def _norm_name(s: str) -> str:
88
- s = s.lower().strip()
89
- for ch in [" ", "_", "-", "."]:
90
- s = s.replace(ch, "")
91
- return s
92
-
93
-
94
- class RESISC45(DatasetBase):
95
-
96
- dataset_dir = "resisc45"
97
-
98
- def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'):
99
- root = os.path.abspath(os.path.expanduser(root))
100
- self.dataset_dir = os.path.join(root, self.dataset_dir)
101
- self.image_dir = self.dataset_dir # images are under class subfolders directly
102
- self.split_path = os.path.join(self.dataset_dir, "split_custom_RESISC45.json")
103
- self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot")
104
- mkdir_if_missing(self.split_fewshot_dir)
105
- _dbg(f"dataset_dir={self.dataset_dir}")
106
-
107
- if os.path.exists(self.split_path):
108
- try:
109
- train, val, test = OxfordPets.read_split(self.split_path, self.image_dir)
110
- except Exception as e:
111
- warnings.warn(f"RESISC45: failed to read split file '{self.split_path}'; rebuilding. Error: {e}")
112
- train, val, test = self.read_and_split_data(self.image_dir)
113
- try:
114
- OxfordPets.save_split(train, val, test, self.split_path, self.image_dir)
115
- except Exception as e2:
116
- warnings.warn(f"RESISC45: failed to save rebuilt split to '{self.split_path}': {e2}")
117
- else:
118
- train, val, test = self.read_and_split_data(self.image_dir)
119
- try:
120
- OxfordPets.save_split(train, val, test, self.split_path, self.image_dir)
121
- except Exception as e:
122
- warnings.warn(f"RESISC45: failed to save split to '{self.split_path}': {e}")
123
-
124
- if num_shots >= 1:
125
- preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
126
- if os.path.exists(preprocessed):
127
- print(f"Loading preprocessed few-shot data from {preprocessed}")
128
- with open(preprocessed, "rb") as file:
129
- data = pickle.load(file)
130
- train, val = data["train"], data["val"]
131
- else:
132
- train = self.generate_fewshot_dataset(train, num_shots=num_shots)
133
- val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
134
- data = {"train": train, "val": val}
135
- print(f"Saving preprocessed few-shot data to {preprocessed}")
136
- with open(preprocessed, "wb") as file:
137
- pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)
138
-
139
- train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample_classes)
140
- self.templates = RESISC_TEMPLATES
141
- super().__init__(train_x=train, val=val, test=test)
142
-
143
- @staticmethod
144
- def read_and_split_data(image_dir, p_trn=0.5, p_val=0.2):
145
- try:
146
- categories = listdir_nohidden(image_dir, sort=True)
147
- except Exception as e:
148
- warnings.warn(f"RESISC45: failed to list directory '{image_dir}': {e}")
149
- raise
150
- categories = [c for c in categories if os.path.isdir(os.path.join(image_dir, c))]
151
- cat_norm = {_norm_name(c): c for c in categories}
152
-
153
- missing = []
154
- class_to_dir = {}
155
- for cname in RESISC_CLASSES:
156
- key = _norm_name(cname)
157
- if key in cat_norm:
158
- class_to_dir[cname] = cat_norm[key]
159
- else:
160
- missing.append(cname)
161
- if missing:
162
- warnings.warn(f"RESISC45: missing class folders for {len(missing)} classes: {missing[:5]}{' ...' if len(missing)>5 else ''}")
163
-
164
- def _collate(paths, y, cname):
165
- return [Datum(impath=p, label=y, classname=cname) for p in paths]
166
-
167
- train, val, test = [], [], []
168
- for y, cname in enumerate(RESISC_CLASSES):
169
- if cname not in class_to_dir:
170
- continue
171
- cdir = os.path.join(image_dir, class_to_dir[cname])
172
- try:
173
- images = listdir_nohidden(cdir, sort=False)
174
- except Exception as e:
175
- warnings.warn(f"RESISC45: failed to list class folder '{cdir}': {e}")
176
- images = []
177
- images = [os.path.join(cdir, im) for im in images]
178
- random.shuffle(images)
179
- n_total = len(images)
180
- if n_total == 0:
181
- warnings.warn(f"RESISC45: empty class folder {cdir}; skipping")
182
- continue
183
- n_train = round(n_total * p_trn)
184
- n_val = round(n_total * p_val)
185
- n_test = n_total - n_train - n_val
186
- if not (n_train > 0 and n_val > 0 and n_test > 0):
187
- # Fallback to keep all splits non-empty where possible
188
- if n_total >= 3:
189
- n_train, n_val, n_test = 1, 1, n_total - 2
190
- elif n_total == 2:
191
- n_train, n_val, n_test = 1, 1, 0
192
- else: # 1
193
- n_train, n_val, n_test = 1, 0, 0
194
-
195
- train.extend(_collate(images[:n_train], y, cname))
196
- val.extend(_collate(images[n_train:n_train + n_val], y, cname))
197
- test.extend(_collate(images[n_train + n_val:], y, cname))
198
-
199
- return train, val, test
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/MTIL_datasets/sst2.py DELETED
@@ -1,122 +0,0 @@
1
- import os
2
- import pickle
3
-
4
- from .utils import *
5
- from .oxford_pets import OxfordPets
6
- from .dtd import DescribableTextures as DTD
7
-
8
-
9
- class SST2(DatasetBase):
10
- """
11
- SST2 (rendered) dataset loader for MTIL.
12
-
13
- Expected structure (either of the following roots):
14
- <root>/sst2/
15
- ├─ train/{negative,positive}/*.png
16
- ├─ valid/{negative,positive}/*.png
17
- └─ test/{negative,positive}/*.png
18
-
19
- <root>/rendered-sst2/ (alias supported)
20
- ├─ train/{negative,positive}/*.png
21
- ├─ valid/{negative,positive}/*.png
22
- └─ test/{negative,positive}/*.png
23
-
24
- Classes: ['negative', 'positive']
25
- Templates: ['a {} review of a movie.']
26
- """
27
-
28
- dataset_dir = "sst2"
29
-
30
- def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'):
31
- root = os.path.abspath(os.path.expanduser(root))
32
- primary_dir = os.path.join(root, self.dataset_dir)
33
- alt_dir = os.path.join(root, "rendered-sst2")
34
-
35
- if os.path.isdir(primary_dir):
36
- self.dataset_dir = primary_dir
37
- elif os.path.isdir(alt_dir):
38
- self.dataset_dir = alt_dir
39
- else:
40
- raise ValueError(
41
- "SST2: dataset folder not found. Expected one of: '{}' or '{}'".format(primary_dir, alt_dir)
42
- )
43
-
44
- self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot")
45
- mkdir_if_missing(self.split_fewshot_dir)
46
-
47
- train_dir = os.path.join(self.dataset_dir, "train")
48
- valid_dir = os.path.join(self.dataset_dir, "valid")
49
- test_dir = os.path.join(self.dataset_dir, "test")
50
-
51
- use_folder_splits = os.path.isdir(train_dir) and os.path.isdir(valid_dir) and os.path.isdir(test_dir)
52
-
53
- # fixed class order and validation
54
- classes = ["negative", "positive"]
55
- class_to_label = {c: i for i, c in enumerate(classes)}
56
-
57
- if use_folder_splits:
58
- train = self._read_split_dir(train_dir, class_to_label)
59
- val = self._read_split_dir(valid_dir, class_to_label)
60
- test = self._read_split_dir(test_dir, class_to_label)
61
- else:
62
- # Fallbacks consistent with other datasets: look for a JSON split, else naive split
63
- image_dir = os.path.join(self.dataset_dir, "images")
64
- self.image_dir = image_dir if os.path.isdir(image_dir) else self.dataset_dir
65
- split_path = os.path.join(self.dataset_dir, "split_zhou_SST2.json")
66
- if os.path.exists(split_path):
67
- train, val, test = OxfordPets.read_split(split_path, self.image_dir)
68
- else:
69
- train, val, test = DTD.read_and_split_data(self.image_dir)
70
- OxfordPets.save_split(train, val, test, split_path, self.image_dir)
71
-
72
- if num_shots >= 1:
73
- preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
74
- if os.path.exists(preprocessed):
75
- print(f"Loading preprocessed few-shot data from {preprocessed}")
76
- with open(preprocessed, "rb") as file:
77
- data = pickle.load(file)
78
- train, val = data["train"], data["val"]
79
- else:
80
- train = self.generate_fewshot_dataset(train, num_shots=num_shots)
81
- val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
82
- data = {"train": train, "val": val}
83
- print(f"Saving preprocessed few-shot data to {preprocessed}")
84
- with open(preprocessed, "wb") as file:
85
- pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)
86
-
87
- subsample = subsample_classes
88
- train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample)
89
-
90
- # Template per user spec
91
- self.templates = [
92
- lambda c: f'a {c} review of a movie.',
93
- ]
94
-
95
- super().__init__(train_x=train, val=val, test=test)
96
-
97
- def _read_split_dir(self, split_dir, class_to_label):
98
- items = []
99
- if not os.path.isdir(split_dir):
100
- return items
101
- codes = listdir_nohidden(split_dir)
102
- # Validate folders: must be subset of expected classes and cover at least one class
103
- unexpected = sorted([c for c in codes if c not in class_to_label])
104
- if unexpected:
105
- raise ValueError(
106
- f"SST2: Found unexpected class folders in '{split_dir}': {unexpected}. "
107
- f"Expected only {list(class_to_label.keys())}."
108
- )
109
- for cls in codes:
110
- class_dir = os.path.join(split_dir, cls)
111
- if not os.path.isdir(class_dir):
112
- continue
113
- label = class_to_label.get(cls)
114
- if label is None:
115
- raise ValueError(
116
- f"SST2: Inconsistent label mapping for class '{cls}' in split '{split_dir}'."
117
- )
118
- cname = cls
119
- for fname in listdir_nohidden(class_dir):
120
- impath = os.path.join(class_dir, fname)
121
- items.append(Datum(impath=impath, label=label, classname=cname))
122
- return items
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/MTIL_datasets/stanford_cars.py DELETED
@@ -1,83 +0,0 @@
1
- import os
2
- import pickle
3
- from scipy.io import loadmat
4
-
5
- from .utils import *
6
-
7
- from .oxford_pets import OxfordPets
8
-
9
-
10
- class StanfordCars(DatasetBase):
11
-
12
- dataset_dir = "stanford_cars"
13
-
14
- def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'):
15
- root = os.path.abspath(os.path.expanduser(root))
16
- self.dataset_dir = os.path.join(root, self.dataset_dir)
17
- self.split_path = os.path.join(self.dataset_dir, "split_zhou_StanfordCars.json")
18
- self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot")
19
- mkdir_if_missing(self.split_fewshot_dir)
20
-
21
- if os.path.exists(self.split_path):
22
- train, val, test = OxfordPets.read_split(self.split_path, self.dataset_dir)
23
- else:
24
- trainval_file = os.path.join(self.dataset_dir, "devkit", "cars_train_annos.mat")
25
- test_file = os.path.join(self.dataset_dir, "cars_test_annos_withlabels.mat")
26
- meta_file = os.path.join(self.dataset_dir, "devkit", "cars_meta.mat")
27
- trainval = self.read_data("cars_train", trainval_file, meta_file)
28
- test = self.read_data("cars_test", test_file, meta_file)
29
- train, val = OxfordPets.split_trainval(trainval)
30
- OxfordPets.save_split(train, val, test, self.split_path, self.dataset_dir)
31
-
32
- if num_shots >= 1:
33
- preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
34
-
35
- if os.path.exists(preprocessed):
36
- print(f"Loading preprocessed few-shot data from {preprocessed}")
37
- with open(preprocessed, "rb") as file:
38
- data = pickle.load(file)
39
- train, val = data["train"], data["val"]
40
- else:
41
- train = self.generate_fewshot_dataset(train, num_shots=num_shots)
42
- val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
43
- data = {"train": train, "val": val}
44
- print(f"Saving preprocessed few-shot data to {preprocessed}")
45
- with open(preprocessed, "wb") as file:
46
- pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)
47
-
48
- subsample = subsample_classes
49
- train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample)
50
-
51
- self.templates = [
52
- lambda c: f"a photo of a {c}, a type of car.",
53
- lambda c: f"a photo of a {c}.",
54
- lambda c: f"a photo of the {c}.",
55
- lambda c: f"a photo of my {c}.",
56
- lambda c: f"i love my {c}!",
57
- lambda c: f"a photo of my dirty {c}.",
58
- lambda c: f"a photo of my clean {c}.",
59
- lambda c: f"a photo of my new {c}.",
60
- lambda c: f"a photo of my old {c}.",
61
- ]
62
-
63
- super().__init__(train_x=train, val=val, test=test)
64
-
65
- def read_data(self, image_dir, anno_file, meta_file):
66
- anno_file = loadmat(anno_file)["annotations"][0]
67
- meta_file = loadmat(meta_file)["class_names"][0]
68
- items = []
69
-
70
- for i in range(len(anno_file)):
71
- imname = anno_file[i]["fname"][0]
72
- impath = os.path.join(self.dataset_dir, image_dir, imname)
73
- label = anno_file[i]["class"][0, 0]
74
- label = int(label) - 1
75
- classname = meta_file[label][0]
76
- names = classname.split(" ")
77
- year = names.pop(-1)
78
- names.insert(0, year)
79
- classname = " ".join(names)
80
- item = Datum(impath=impath, label=label, classname=classname)
81
- items.append(item)
82
-
83
- return items
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/MTIL_datasets/stl10.py DELETED
@@ -1,143 +0,0 @@
1
- import os
2
- import math
3
- import random
4
- from typing import List, Tuple
5
-
6
- from .utils import * # Datum, DatasetBase, listdir_nohidden
7
- from .oxford_pets import OxfordPets
8
-
9
- STL10_CLASSES: List[str] = [
10
- 'airplane',
11
- 'bird',
12
- 'car',
13
- 'cat',
14
- 'deer',
15
- 'dog',
16
- 'horse',
17
- 'monkey',
18
- 'ship',
19
- 'truck',
20
- ]
21
-
22
- # keep templates as strings with {}
23
- STL10_TEMPLATES: List[str] = [
24
- 'a photo of a {}.',
25
- 'a photo of the {}.',
26
- ]
27
-
28
-
29
- class STL10(DatasetBase):
30
-
31
- dataset_dir = "stl10"
32
-
33
- def __init__(self, root, num_shots=0, seed=1, subsample_classes='all', test_ratio: float = 0.2):
34
- """
35
- Expect directory structure:
36
- {root}/stl10/<class_name>/*.png|jpg|jpeg|bmp|webp
37
- No official test set provided; we split per-class into train/test.
38
- Then we further split train into train/val using OxfordPets.split_trainval.
39
- """
40
- rnd = random.Random(seed)
41
-
42
- root = os.path.abspath(os.path.expanduser(root))
43
- self.dataset_dir = os.path.join(root, self.dataset_dir)
44
- self.split_path = os.path.join(self.dataset_dir, "split_custom_STL10.json")
45
- self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot")
46
- mkdir_if_missing(self.split_fewshot_dir)
47
-
48
- # If we have a saved split, reuse for reproducibility
49
- if os.path.exists(self.split_path):
50
- train, val, test = OxfordPets.read_split(self.split_path, self.dataset_dir)
51
- else:
52
- trainval, test = self._read_from_folders(self.dataset_dir, rnd=rnd, test_ratio=test_ratio)
53
- train, val = self._split_trainval_safe(trainval)
54
- OxfordPets.save_split(train, val, test, self.split_path, self.dataset_dir)
55
-
56
- if num_shots >= 1:
57
- preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
58
- if os.path.exists(preprocessed):
59
- print(f"Loading preprocessed few-shot data from {preprocessed}")
60
- import pickle
61
- with open(preprocessed, "rb") as file:
62
- data = pickle.load(file)
63
- train, val = data["train"], data["val"]
64
- else:
65
- train = self.generate_fewshot_dataset(train, num_shots=num_shots)
66
- val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
67
- data = {"train": train, "val": val}
68
- print(f"Saving preprocessed few-shot data to {preprocessed}")
69
- import pickle
70
- with open(preprocessed, "wb") as file:
71
- pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)
72
-
73
- train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample_classes)
74
-
75
- self.templates = STL10_TEMPLATES
76
-
77
- super().__init__(train_x=train, val=val, test=test)
78
-
79
- def _read_from_folders(self, data_dir: str, rnd: random.Random, test_ratio: float) -> Tuple[List[Datum], List[Datum]]:
80
- exts = {".png", ".jpg", ".jpeg", ".bmp", ".webp"}
81
- class_to_label = {name: i for i, name in enumerate(STL10_CLASSES)}
82
-
83
- items_by_label = {i: [] for i in range(len(STL10_CLASSES))}
84
- for cname in STL10_CLASSES:
85
- cdir = os.path.join(data_dir, cname)
86
- if not os.path.isdir(cdir):
87
- raise FileNotFoundError(f"Class folder not found: {cdir}")
88
- for fname in listdir_nohidden(cdir, sort=True):
89
- fext = os.path.splitext(fname)[1].lower()
90
- if fext not in exts:
91
- continue
92
- impath = os.path.join(cdir, fname)
93
- label = class_to_label[cname]
94
- items_by_label[label].append(Datum(impath=impath, label=label, classname=cname))
95
-
96
- trainval, test = [], []
97
- for label, items in items_by_label.items():
98
- if not items:
99
- continue
100
- rnd.shuffle(items)
101
- if len(items) == 1:
102
- # keep the only sample for training to preserve class presence in train
103
- trainval.extend(items)
104
- continue
105
- n_test = max(1, int(round(len(items) * test_ratio)))
106
- if n_test >= len(items):
107
- n_test = len(items) - 1
108
- test.extend(items[:n_test])
109
- trainval.extend(items[n_test:])
110
-
111
- return trainval, test
112
-
113
- @staticmethod
114
- def _split_trainval_safe(trainval: List[Datum], p_val: float = 0.2) -> Tuple[List[Datum], List[Datum]]:
115
- from collections import defaultdict
116
- tracker = defaultdict(list)
117
- for idx, item in enumerate(trainval):
118
- tracker[item.label].append(idx)
119
-
120
- train, val = [], []
121
- for label, idxs in tracker.items():
122
- n = len(idxs)
123
- if n <= 1:
124
- # not enough to create a val sample for this class
125
- for idx in idxs:
126
- train.append(trainval[idx])
127
- continue
128
- n_val = max(1, int(round(n * p_val)))
129
- if n_val >= n:
130
- n_val = n - 1
131
- random.shuffle(idxs)
132
- for i, idx in enumerate(idxs):
133
- if i < n_val:
134
- val.append(trainval[idx])
135
- else:
136
- train.append(trainval[idx])
137
-
138
- # If val ended up empty (degenerate tiny dataset), move one from train to val
139
- if len(val) == 0 and len(train) > 0:
140
- val.append(train[-1])
141
- train = train[:-1]
142
-
143
- return train, val
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/MTIL_datasets/sun397.py DELETED
@@ -1,81 +0,0 @@
1
- import os
2
- import pickle
3
-
4
- from .utils import *
5
-
6
- from .oxford_pets import OxfordPets
7
-
8
-
9
- class SUN397(DatasetBase):
10
-
11
- dataset_dir = "sun397"
12
-
13
- def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'):
14
- root = os.path.abspath(os.path.expanduser(root))
15
- self.dataset_dir = os.path.join(root, self.dataset_dir)
16
- self.image_dir = os.path.join(self.dataset_dir, "SUN397")
17
- self.split_path = os.path.join(self.dataset_dir, "split_zhou_SUN397.json")
18
- self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot")
19
- mkdir_if_missing(self.split_fewshot_dir)
20
-
21
- if os.path.exists(self.split_path):
22
- train, val, test = OxfordPets.read_split(self.split_path, self.image_dir)
23
- else:
24
- classnames = []
25
- with open(os.path.join(self.dataset_dir, "ClassName.txt"), "r") as f:
26
- lines = f.readlines()
27
- for line in lines:
28
- line = line.strip()[1:]
29
- classnames.append(line)
30
- cname2lab = {c: i for i, c in enumerate(classnames)}
31
- trainval = self.read_data(cname2lab, "Training_01.txt")
32
- test = self.read_data(cname2lab, "Testing_01.txt")
33
- train, val = OxfordPets.split_trainval(trainval)
34
- OxfordPets.save_split(train, val, test, self.split_path, self.image_dir)
35
-
36
- if num_shots >= 1:
37
- preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
38
-
39
- if os.path.exists(preprocessed):
40
- print(f"Loading preprocessed few-shot data from {preprocessed}")
41
- with open(preprocessed, "rb") as file:
42
- data = pickle.load(file)
43
- train, val = data["train"], data["val"]
44
- else:
45
- train = self.generate_fewshot_dataset(train, num_shots=num_shots)
46
- val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
47
- data = {"train": train, "val": val}
48
- print(f"Saving preprocessed few-shot data to {preprocessed}")
49
- with open(preprocessed, "wb") as file:
50
- pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)
51
-
52
- subsample = subsample_classes
53
- train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample)
54
-
55
- self.templates = [
56
- lambda c: f"a photo of a {c}.",
57
- lambda c: f"a photo of the {c}.",
58
- ]
59
-
60
- super().__init__(train_x=train, val=val, test=test)
61
-
62
- def read_data(self, cname2lab, text_file):
63
- text_file = os.path.join(self.dataset_dir, text_file)
64
- items = []
65
-
66
- with open(text_file, "r") as f:
67
- lines = f.readlines()
68
- for line in lines:
69
- imname = line.strip()[1:]
70
- classname = os.path.dirname(imname)
71
- label = cname2lab[classname]
72
- impath = os.path.join(self.image_dir, imname)
73
-
74
- names = classname.split("/")[1:]
75
- names = names[::-1]
76
- classname = " ".join(names)
77
-
78
- item = Datum(impath=impath, label=label, classname=classname)
79
- items.append(item)
80
-
81
- return items
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/MTIL_datasets/ucf101.py DELETED
@@ -1,360 +0,0 @@
1
- import os
2
- import pickle
3
- import random
4
- import math
5
- from collections import defaultdict
6
- from typing import List, Tuple
7
-
8
- from .utils import * # Datum, DatasetBase, mkdir_if_missing, read_json, write_json, listdir_nohidden
9
-
10
-
11
- # Canonical class names as provided
12
- UCF101_CANONICAL: List[str] = [
13
- 'Apply Eye Makeup',
14
- 'Apply Lipstick',
15
- 'Archery',
16
- 'Baby Crawling',
17
- 'Balance Beam',
18
- 'Band Marching',
19
- 'Baseball Pitch',
20
- 'Basketball',
21
- 'Basketball Dunk',
22
- 'Bench Press',
23
- 'Biking',
24
- 'Billiards',
25
- 'Blow Dry Hair',
26
- 'Blowing Candles',
27
- 'Body Weight Squats',
28
- 'Bowling',
29
- 'Boxing Punching Bag',
30
- 'Boxing Speed Bag',
31
- 'Breast Stroke',
32
- 'Brushing Teeth',
33
- 'Clean And Jerk',
34
- 'Cliff Diving',
35
- 'Cricket Bowling',
36
- 'Cricket Shot',
37
- 'Cutting In Kitchen',
38
- 'Diving',
39
- 'Drumming',
40
- 'Fencing',
41
- 'Field Hockey Penalty',
42
- 'Floor Gymnastics',
43
- 'Frisbee Catch',
44
- 'Front Crawl',
45
- 'Golf Swing',
46
- 'Haircut',
47
- 'Hammer Throw',
48
- 'Hammering',
49
- 'Hand Stand Pushups',
50
- 'Handstand Walking',
51
- 'Head Massage',
52
- 'High Jump',
53
- 'Horse Race',
54
- 'Horse Riding',
55
- 'Hula Hoop',
56
- 'Ice Dancing',
57
- 'Javelin Throw',
58
- 'Juggling Balls',
59
- 'Jump Rope',
60
- 'Jumping Jack',
61
- 'Kayaking',
62
- 'Knitting',
63
- 'Long Jump',
64
- 'Lunges',
65
- 'Military Parade',
66
- 'Mixing',
67
- 'Mopping Floor',
68
- 'Nunchucks',
69
- 'Parallel Bars',
70
- 'Pizza Tossing',
71
- 'Playing Cello',
72
- 'Playing Daf',
73
- 'Playing Dhol',
74
- 'Playing Flute',
75
- 'Playing Guitar',
76
- 'Playing Piano',
77
- 'Playing Sitar',
78
- 'Playing Tabla',
79
- 'Playing Violin',
80
- 'Pole Vault',
81
- 'Pommel Horse',
82
- 'Pull Ups',
83
- 'Punch',
84
- 'Push Ups',
85
- 'Rafting',
86
- 'Rock Climbing Indoor',
87
- 'Rope Climbing',
88
- 'Rowing',
89
- 'Salsa Spin',
90
- 'Shaving Beard',
91
- 'Shotput',
92
- 'Skate Boarding',
93
- 'Skiing',
94
- 'Skijet',
95
- 'Sky Diving',
96
- 'Soccer Juggling',
97
- 'Soccer Penalty',
98
- 'Still Rings',
99
- 'Sumo Wrestling',
100
- 'Surfing',
101
- 'Swing',
102
- 'Table Tennis Shot',
103
- 'Tai Chi',
104
- 'Tennis Swing',
105
- 'Throw Discus',
106
- 'Trampoline Jumping',
107
- 'Typing',
108
- 'Uneven Bars',
109
- 'Volleyball Spiking',
110
- 'Walking With Dog',
111
- 'Wall Pushups',
112
- 'Writing On Board',
113
- 'Yo Yo',
114
- ]
115
-
116
-
117
- UCF101_TEMPLATES: List[str] = [
118
- 'a photo of a person {}.',
119
- 'a video of a person {}.',
120
- 'a example of a person {}.',
121
- 'a demonstration of a person {}.',
122
- 'a photo of the person {}.',
123
- 'a video of the person {}.',
124
- 'a example of the person {}.',
125
- 'a demonstration of the person {}.',
126
- 'a photo of a person using {}.',
127
- 'a video of a person using {}.',
128
- 'a example of a person using {}.',
129
- 'a demonstration of a person using {}.',
130
- 'a photo of the person using {}.',
131
- 'a video of the person using {}.',
132
- 'a example of the person using {}.',
133
- 'a demonstration of the person using {}.',
134
- 'a photo of a person doing {}.',
135
- 'a video of a person doing {}.',
136
- 'a example of a person doing {}.',
137
- 'a demonstration of a person doing {}.',
138
- 'a photo of the person doing {}.',
139
- 'a video of the person doing {}.',
140
- 'a example of the person doing {}.',
141
- 'a demonstration of the person doing {}.',
142
- 'a photo of a person during {}.',
143
- 'a video of a person during {}.',
144
- 'a example of a person during {}.',
145
- 'a demonstration of a person during {}.',
146
- 'a photo of the person during {}.',
147
- 'a video of the person during {}.',
148
- 'a example of the person during {}.',
149
- 'a demonstration of the person during {}.',
150
- 'a photo of a person performing {}.',
151
- 'a video of a person performing {}.',
152
- 'a example of a person performing {}.',
153
- 'a demonstration of a person performing {}.',
154
- 'a photo of the person performing {}.',
155
- 'a video of the person performing {}.',
156
- 'a example of the person performing {}.',
157
- 'a demonstration of the person performing {}.',
158
- 'a photo of a person practicing {}.',
159
- 'a video of a person practicing {}.',
160
- 'a example of a person practicing {}.',
161
- 'a demonstration of a person practicing {}.',
162
- 'a photo of the person practicing {}.',
163
- 'a video of the person practicing {}.',
164
- 'a example of the person practicing {}.',
165
- 'a demonstration of the person practicing {}.',
166
- ]
167
-
168
-
169
- class UCF101(DatasetBase):
170
- """
171
- UCF101 midframes classification dataset adapter for MTIL.
172
- Expected structure:
173
- root/ucf101/
174
- UCF-101-midframes/<Class_Folder>/<image>.jpg
175
- split_zhou_UCF101.json
176
- """
177
-
178
- dataset_dir = "ucf101"
179
-
180
- def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'):
181
- root = os.path.abspath(os.path.expanduser(root))
182
- self.dataset_dir = os.path.join(root, self.dataset_dir)
183
- self.image_dir = os.path.join(self.dataset_dir, "UCF-101-midframes")
184
- self.split_path = os.path.join(self.dataset_dir, "split_zhou_UCF101.json")
185
- self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot")
186
- mkdir_if_missing(self.split_fewshot_dir)
187
-
188
- if os.path.exists(self.split_path):
189
- train, val, test = self.read_split(self.split_path, self.image_dir)
190
- else:
191
- # Fallback: build from directory and split train/val; use val as test too.
192
- trainval = self._read_from_dir()
193
- train, val = self.split_trainval(trainval)
194
- test = list(val)
195
- self.save_split(train, val, test, self.split_path, self.image_dir)
196
-
197
- if num_shots >= 1:
198
- preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
199
- if os.path.exists(preprocessed):
200
- print(f"Loading preprocessed few-shot data from {preprocessed}")
201
- with open(preprocessed, "rb") as file:
202
- data = pickle.load(file)
203
- train, val = data["train"], data["val"]
204
- else:
205
- train = self.generate_fewshot_dataset(train, num_shots=num_shots)
206
- val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
207
- data = {"train": train, "val": val}
208
- print(f"Saving preprocessed few-shot data to {preprocessed}")
209
- with open(preprocessed, "wb") as file:
210
- pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)
211
-
212
- # Optionally subsample classes (keep interface consistent)
213
- train, val, test = self.subsample_classes(train, val, test, subsample=subsample_classes)
214
-
215
- self.templates = UCF101_TEMPLATES
216
-
217
- super().__init__(train_x=train, val=val, test=test)
218
-
219
- @staticmethod
220
- def subsample_classes(*args, subsample="all"):
221
- assert subsample in ["all", "base", "new"]
222
-
223
- if subsample == "all":
224
- return args
225
-
226
- dataset = args[0]
227
- labels = set()
228
- for item in dataset:
229
- labels.add(item.label)
230
- labels = list(labels)
231
- labels.sort()
232
- n = len(labels)
233
- m = math.ceil(n / 2)
234
-
235
- print(f"SUBSAMPLE {subsample.upper()} CLASSES!")
236
- if subsample == "base":
237
- selected = labels[:m]
238
- else:
239
- selected = labels[m:]
240
- relabeler = {y: y_new for y_new, y in enumerate(selected)}
241
-
242
- output = []
243
- for dataset in args:
244
- dataset_new = []
245
- for item in dataset:
246
- if item.label not in selected:
247
- continue
248
- item_new = Datum(
249
- impath=item.impath,
250
- label=relabeler[item.label],
251
- classname=item.classname
252
- )
253
- dataset_new.append(item_new)
254
- output.append(dataset_new)
255
-
256
- return output
257
-
258
- # ---- IO helpers (compatible with OxfordPets) ----
259
- @staticmethod
260
- def save_split(train, val, test, filepath, path_prefix):
261
- def _extract(items):
262
- out = []
263
- for item in items:
264
- impath = item.impath
265
- label = item.label
266
- classname = item.classname
267
- impath = impath.replace(path_prefix, "")
268
- if impath.startswith("/"):
269
- impath = impath[1:]
270
- out.append((impath, label, classname))
271
- return out
272
-
273
- train = _extract(train)
274
- val = _extract(val)
275
- test = _extract(test)
276
- split = {"train": train, "val": val, "test": test}
277
- write_json(split, filepath)
278
- print(f"Saved split to {filepath}")
279
-
280
- @staticmethod
281
- def read_split(filepath, path_prefix):
282
- def _convert(items):
283
- out = []
284
- for impath, label, classname in items:
285
- impath = os.path.join(path_prefix, impath)
286
- item = Datum(impath=impath, label=int(label), classname=classname)
287
- out.append(item)
288
- return out
289
-
290
- print(f"Reading split from {filepath}")
291
- split = read_json(filepath)
292
- train = _convert(split["train"])
293
- val = _convert(split["val"])
294
- test = _convert(split["test"])
295
- return train, val, test
296
-
297
- @staticmethod
298
- def split_trainval(trainval: List[Datum], p_val=0.2) -> Tuple[List[Datum], List[Datum]]:
299
- p_trn = 1 - p_val
300
- tracker = defaultdict(list)
301
- for idx, item in enumerate(trainval):
302
- tracker[item.label].append(idx)
303
- train, val = [], []
304
- for label, idxs in tracker.items():
305
- n_val = max(1, round(len(idxs) * p_val))
306
- random.shuffle(idxs)
307
- for n, idx in enumerate(idxs):
308
- item = trainval[idx]
309
- if n < n_val:
310
- val.append(item)
311
- else:
312
- train.append(item)
313
- return train, val
314
-
315
- # ---- Directory reader (fallback if JSON is missing) ----
316
- def _read_from_dir(self) -> List[Datum]:
317
- if not os.path.isdir(self.image_dir):
318
- raise FileNotFoundError(f"Image directory not found: {self.image_dir}")
319
-
320
- # Build canonical key mapping (lowercased, remove spaces/underscores)
321
- def canon_key(s: str) -> str:
322
- return ''.join(ch for ch in s.lower() if ch.isalnum())
323
-
324
- canonical_to_label = {name: idx for idx, name in enumerate(UCF101_CANONICAL)}
325
- key_to_canonical = {canon_key(name): name for name in UCF101_CANONICAL}
326
-
327
- items: List[Datum] = []
328
- class_dirs = listdir_nohidden(self.image_dir, sort=True)
329
- for cls_dir in class_dirs:
330
- cls_path = os.path.join(self.image_dir, cls_dir)
331
- if not os.path.isdir(cls_path):
332
- continue
333
- # Try to map folder name to canonical class
334
- k = canon_key(cls_dir.replace('_', ' '))
335
- cname = key_to_canonical.get(k, None)
336
- if cname is None:
337
- # Try removing underscores without space
338
- k2 = canon_key(cls_dir.replace('_', ''))
339
- cname = key_to_canonical.get(k2, None)
340
- if cname is None:
341
- # As a last resort, use the folder name with underscores replaced
342
- cname = cls_dir.replace('_', ' ').strip()
343
- if cname not in canonical_to_label:
344
- # Unknown class; skip
345
- continue
346
- label = canonical_to_label[cname]
347
- # Collect images
348
- for vid in listdir_nohidden(cls_path, sort=False):
349
- vpath = os.path.join(cls_path, vid)
350
- if os.path.isdir(vpath):
351
- # some datasets might nest frames under video folder; include all frames
352
- for frame in listdir_nohidden(vpath, sort=False):
353
- impath = os.path.join(vpath, frame)
354
- if os.path.isfile(impath):
355
- items.append(Datum(impath=impath, label=label, classname=cname))
356
- else:
357
- # direct frames under class folder
358
- if os.path.isfile(vpath):
359
- items.append(Datum(impath=vpath, label=label, classname=cname))
360
- return items
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/MTIL_datasets/utils.py DELETED
@@ -1,299 +0,0 @@
1
- import os
2
- import random
3
- import os.path as osp
4
- import tarfile
5
- import zipfile
6
- from collections import defaultdict
7
- import gdown
8
- import errno
9
- import warnings
10
- import json
11
- import numpy as np
12
- import h5py
13
- from torch.utils.data import Dataset as TorchDataset
14
- from PIL import Image
15
-
16
-
17
- class Datum:
18
- def __init__(self, impath="", label=0, domain=0, classname=""):
19
- self._impath = impath
20
- self._label = label
21
- self._domain = domain
22
- self._classname = classname
23
-
24
- @property
25
- def impath(self):
26
- return self._impath
27
-
28
- @property
29
- def label(self):
30
- return self._label
31
-
32
- @property
33
- def domain(self):
34
- return self._domain
35
-
36
- @property
37
- def classname(self):
38
- return self._classname
39
-
40
-
41
- class DatasetBase:
42
- dataset_dir = ""
43
- domains = []
44
-
45
- def __init__(self, train_x=None, train_u=None, val=None, test=None):
46
- self._train_x = train_x
47
- self._train_u = train_u
48
- self._val = val
49
- self._test = test
50
- self._num_classes = self.get_num_classes(train_x)
51
- self._lab2cname, self._classnames = self.get_lab2cname(train_x)
52
-
53
- @property
54
- def train_x(self):
55
- return self._train_x
56
-
57
- @property
58
- def train_u(self):
59
- return self._train_u
60
-
61
- @property
62
- def val(self):
63
- return self._val
64
-
65
- @property
66
- def test(self):
67
- return self._test
68
-
69
- @property
70
- def lab2cname(self):
71
- return self._lab2cname
72
-
73
- @property
74
- def classnames(self):
75
- return self._classnames
76
-
77
- @property
78
- def num_classes(self):
79
- return self._num_classes
80
-
81
- @staticmethod
82
- def get_num_classes(data_source):
83
- label_set = set()
84
- for item in data_source:
85
- label_set.add(item.label)
86
- return max(label_set) + 1
87
-
88
- @staticmethod
89
- def get_lab2cname(data_source):
90
- container = set()
91
- for item in data_source:
92
- container.add((item.label, item.classname))
93
- mapping = {label: classname for label, classname in container}
94
- labels = list(mapping.keys())
95
- labels.sort()
96
- classnames = [mapping[label] for label in labels]
97
- return mapping, classnames
98
-
99
- @property
100
- def template(self):
101
- return self.templates[0]
102
-
103
- def check_input_domains(self, source_domains, target_domains):
104
- assert len(source_domains) > 0, "source_domains (list) is empty"
105
- assert len(target_domains) > 0, "target_domains (list) is empty"
106
- self.is_input_domain_valid(source_domains)
107
- self.is_input_domain_valid(target_domains)
108
-
109
- def is_input_domain_valid(self, input_domains):
110
- for domain in input_domains:
111
- if domain not in self.domains:
112
- raise ValueError(
113
- "Input domain must belong to {}, "
114
- "but got [{}]".format(self.domains, domain)
115
- )
116
-
117
- def download_data(self, url, dst, from_gdrive=True):
118
- if not osp.exists(osp.dirname(dst)):
119
- os.makedirs(osp.dirname(dst))
120
-
121
- if from_gdrive:
122
- gdown.download(url, dst, quiet=False)
123
- else:
124
- raise NotImplementedError
125
-
126
- print("Extracting file ...")
127
-
128
- if dst.endswith(".zip"):
129
- zip_ref = zipfile.ZipFile(dst, "r")
130
- zip_ref.extractall(osp.dirname(dst))
131
- zip_ref.close()
132
-
133
- elif dst.endswith(".tar"):
134
- tar = tarfile.open(dst, "r:")
135
- tar.extractall(osp.dirname(dst))
136
- tar.close()
137
-
138
- elif dst.endswith(".tar.gz"):
139
- tar = tarfile.open(dst, "r:gz")
140
- tar.extractall(osp.dirname(dst))
141
- tar.close()
142
-
143
- else:
144
- raise NotImplementedError
145
-
146
- print("File extracted to {}".format(osp.dirname(dst)))
147
-
148
- def generate_fewshot_dataset(
149
- self, *data_sources, num_shots=-1, repeat=False
150
- ):
151
- if num_shots < 1:
152
- if len(data_sources) == 1:
153
- return data_sources[0]
154
- return data_sources
155
-
156
- print(f"Creating a {num_shots}-shot dataset")
157
-
158
- output = []
159
-
160
- for data_source in data_sources:
161
- tracker = self.split_dataset_by_label(data_source)
162
- dataset = []
163
-
164
- for label, items in tracker.items():
165
- if len(items) >= num_shots:
166
- sampled_items = random.sample(items, num_shots)
167
- else:
168
- if repeat:
169
- sampled_items = random.choices(items, k=num_shots)
170
- else:
171
- sampled_items = items
172
- dataset.extend(sampled_items)
173
-
174
- output.append(dataset)
175
-
176
- if len(output) == 1:
177
- return output[0]
178
-
179
- return output
180
-
181
- def split_dataset_by_label(self, data_source):
182
- output = defaultdict(list)
183
-
184
- for item in data_source:
185
- output[item.label].append(item)
186
-
187
- return output
188
-
189
- def split_dataset_by_domain(self, data_source):
190
- output = defaultdict(list)
191
-
192
- for item in data_source:
193
- output[item.domain].append(item)
194
-
195
- return output
196
-
197
-
198
- class DatasetWrapper(TorchDataset):
199
-
200
- def __init__(self, data_source, transform=None, is_train=False):
201
- self.data_source = data_source
202
- self.transform = transform
203
- self.is_train = is_train
204
- self._h5_cache = {}
205
- self._h5_info_printed = set()
206
-
207
- def __len__(self):
208
- return len(self.data_source)
209
-
210
- def __getitem__(self, idx):
211
- item = self.data_source[idx]
212
-
213
- impath = item.impath
214
- if isinstance(impath, str):
215
- img0 = Image.open(impath).convert("RGB")
216
- elif isinstance(impath, tuple) and len(impath) == 4 and impath[0] == 'h5':
217
- _, fpath, key, index = impath
218
- f = self._h5_cache.get(fpath)
219
- if f is None:
220
- f = h5py.File(fpath, 'r')
221
- self._h5_cache[fpath] = f
222
- # Print file info once
223
- try:
224
- keys = list(f.keys())
225
- print(f"H5 open: {os.path.basename(fpath)} keys={keys[:5]}{'...' if len(keys) > 5 else ''}")
226
- except Exception as e:
227
- print(f"H5 open (keys) failed for {fpath}: {e}")
228
- # Print dataset info per file the first time we see this path
229
- if fpath not in self._h5_info_printed:
230
- try:
231
- ds = f[key]
232
- print(f"H5 dataset: {os.path.basename(fpath)}[{key}] shape={getattr(ds, 'shape', '?')} dtype={getattr(ds, 'dtype', '?')}")
233
- except Exception as e:
234
- print(f"H5 dataset info failed for {fpath}[{key}]: {e}")
235
- self._h5_info_printed.add(fpath)
236
- arr = f[key][int(index)]
237
- arr = np.asarray(arr)
238
- # Convert CHW -> HWC if needed
239
- if arr.ndim == 3 and arr.shape[0] in (1, 3) and arr.shape[-1] not in (1, 3):
240
- arr = np.transpose(arr, (1, 2, 0))
241
- # Ensure HWC and uint8
242
- if arr.ndim == 3 and arr.shape[-1] in (1, 3):
243
- pass
244
- else:
245
- raise ValueError(f"Unexpected H5 image shape: {arr.shape}")
246
- if arr.dtype != np.uint8:
247
- arr = arr.astype(np.uint8)
248
- if arr.shape[-1] == 1:
249
- img0 = Image.fromarray(arr.squeeze(-1), mode='L').convert('RGB')
250
- else:
251
- img0 = Image.fromarray(arr, mode='RGB')
252
- else:
253
- # Fallback: if already PIL Image
254
- if isinstance(impath, Image.Image):
255
- img0 = impath
256
- else:
257
- raise ValueError("Unsupported impath type in DatasetWrapper")
258
-
259
- if self.transform:
260
- img = self.transform(img0)
261
- else:
262
- img = img0
263
-
264
- return img, item.label
265
-
266
-
267
- def check_isfile(fpath):
268
- isfile = osp.isfile(fpath)
269
- if not isfile:
270
- warnings.warn('No file found at "{}"'.format(fpath))
271
- return isfile
272
-
273
-
274
- def mkdir_if_missing(dirname):
275
- if not osp.exists(dirname):
276
- try:
277
- os.makedirs(dirname)
278
- except OSError as e:
279
- if e.errno != errno.EEXIST:
280
- raise
281
-
282
-
283
- def listdir_nohidden(path, sort=False):
284
- items = [f for f in os.listdir(path) if not f.startswith(".")]
285
- if sort:
286
- items.sort()
287
- return items
288
-
289
-
290
- def read_json(fpath):
291
- with open(fpath, "r") as f:
292
- obj = json.load(f)
293
- return obj
294
-
295
-
296
- def write_json(obj, fpath):
297
- mkdir_if_missing(osp.dirname(fpath))
298
- with open(fpath, "w") as f:
299
- json.dump(obj, f, indent=4, separators=(",", ": "))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/MTIL_datasets/voc2007.py DELETED
@@ -1,173 +0,0 @@
1
- import os
2
- import os.path as osp
3
- from typing import List, Dict, Tuple
4
- from collections import defaultdict
5
-
6
- from .utils import Datum, DatasetBase, listdir_nohidden
7
-
8
-
9
- VOC2007_CLASSES = [
10
- 'aeroplane', 'bicycle', 'bird', 'boat', 'bottle',
11
- 'bus', 'car', 'cat', 'chair', 'cow',
12
- 'dog', 'horse', 'motorbike', 'person', 'sheep',
13
- 'sofa', 'diningtable', 'pottedplant', 'train', 'tvmonitor',
14
- ]
15
-
16
- VOC2007_TEMPLATES = [
17
- 'a photo of a {}.',
18
- ]
19
-
20
-
21
- class VOC2007(DatasetBase):
22
- """
23
- VOC2007 multi-label classification dataset adapter for MTIL evaluation.
24
-
25
- Expects directory structure:
26
- data/VOC2007/
27
- JPEGImages/
28
- Main/
29
- <class>_trainval.txt
30
- <class>_test.txt
31
-
32
- Each *_split.txt has lines: "<image_id> <label>", where label in {1, 0, -1}.
33
- We treat 1 as positive and others as negative for that class.
34
- """
35
-
36
- dataset_dir = 'VOC2007'
37
-
38
- def __init__(self, root: str, seed: int = 32, single_label: bool = False):
39
- self.root = os.path.expanduser(root)
40
- self.dataset_dir = osp.join(self.root, self.dataset_dir)
41
- self.images_dir = osp.join(self.dataset_dir, 'JPEGImages')
42
- self.main_dir = osp.join(self.dataset_dir, 'Main')
43
-
44
- classes = VOC2007_CLASSES
45
- class_to_label = {c: i for i, c in enumerate(classes)}
46
-
47
- if not single_label:
48
- # Multi-label mode (for Zero-Shot evaluation)
49
- test_list = self._read_split('test', classes)
50
- test = self._build_data_list(test_list, class_to_label)
51
-
52
- # Create a meta train set with one sample per class (integer labels) for proper initialization
53
- # Try to point to a real image containing that class as positive; fallback to any available image or empty path
54
- meta_train: List[Datum] = []
55
- for ci, cname in enumerate(classes):
56
- impath = None
57
- for d in test:
58
- try:
59
- if isinstance(d.label, (list, tuple)) and ci < len(d.label) and int(d.label[ci]) == 1:
60
- impath = d.impath
61
- break
62
- except Exception:
63
- continue
64
- if impath is None:
65
- impath = test[0].impath if len(test) > 0 else ''
66
- meta_train.append(Datum(impath=impath, label=ci, classname=cname))
67
-
68
- train_x: List[Datum] = meta_train
69
- val: List[Datum] = []
70
- self.templates = VOC2007_TEMPLATES
71
- super().__init__(train_x=train_x, val=val, test=test)
72
- else:
73
- # Single-label mode (for CIL downstream training/evaluation)
74
- # Build trainval and test splits as single-label datasets by selecting the first positive class per image
75
- trainval_multi = self._read_split('trainval', classes)
76
- test_multi = self._read_split('test', classes)
77
- train_x = self._build_single_label_list(trainval_multi, classes)
78
- val = []
79
- test = self._build_single_label_list(test_multi, classes)
80
- self.templates = VOC2007_TEMPLATES
81
- super().__init__(train_x=train_x, val=val, test=test)
82
-
83
- def _read_split(self, split: str, classes: List[str]) -> Dict[str, List[int]]:
84
- """Return mapping: image_id -> multi-hot list for the given split."""
85
- # Collect image ids present in this split
86
- img_ids = set()
87
- by_class_labels: Dict[str, Dict[str, int]] = {}
88
- for cname in classes:
89
- split_file = osp.join(self.main_dir, f"{cname}_{split}.txt")
90
- if not osp.isfile(split_file):
91
- # Try alternate common path name
92
- split_file = osp.join(self.dataset_dir, 'ImageSets', 'Main', f"{cname}_{split}.txt")
93
- if not osp.isfile(split_file):
94
- # If missing, skip this class
95
- continue
96
- class_map: Dict[str, int] = {}
97
- with open(split_file, 'r') as f:
98
- for line in f:
99
- parts = line.strip().split()
100
- if len(parts) < 2:
101
- continue
102
- img_id, label_str = parts[0], parts[1]
103
- try:
104
- label = int(label_str)
105
- except Exception:
106
- continue
107
- class_map[img_id] = 1 if label == 1 else 0
108
- img_ids.add(img_id)
109
- by_class_labels[cname] = class_map
110
-
111
- # Aggregate into multi-hot vectors
112
- result: Dict[str, List[int]] = {}
113
- for img_id in img_ids:
114
- vec = [0] * len(classes)
115
- for ci, cname in enumerate(classes):
116
- cmap = by_class_labels.get(cname, {})
117
- vec[ci] = int(cmap.get(img_id, 0))
118
- result[img_id] = vec
119
- return result
120
-
121
- def _build_data_list(self, id_to_vec: Dict[str, List[int]], class_to_label: Dict[str, int]) -> List[Datum]:
122
- data: List[Datum] = []
123
- for img_id, vec in id_to_vec.items():
124
- # Try common image extensions
125
- impath = None
126
- for ext in ['.jpg', '.jpeg', '.png']:
127
- p = osp.join(self.images_dir, img_id + ext)
128
- if osp.isfile(p):
129
- impath = p
130
- break
131
- if impath is None:
132
- # As a fallback, if there's exactly one file starting with img_id
133
- try:
134
- candidates = [f for f in listdir_nohidden(self.images_dir) if f.startswith(img_id + '.')]
135
- if candidates:
136
- impath = osp.join(self.images_dir, candidates[0])
137
- except Exception:
138
- pass
139
- if impath is None:
140
- # Skip missing images
141
- continue
142
- # For multi-label, we store the full vector as the label
143
- data.append(Datum(impath=impath, label=vec, classname=''))
144
- return data
145
-
146
- def _build_single_label_list(self, id_to_vec: Dict[str, List[int]], classes: List[str]) -> List[Datum]:
147
- data: List[Datum] = []
148
- for img_id, vec in id_to_vec.items():
149
- # choose the first positive class; skip if none
150
- try:
151
- cls_idx = next((i for i, v in enumerate(vec) if int(v) == 1), None)
152
- except Exception:
153
- cls_idx = None
154
- if cls_idx is None:
155
- continue
156
- # image path resolution
157
- impath = None
158
- for ext in ['.jpg', '.jpeg', '.png']:
159
- p = osp.join(self.images_dir, img_id + ext)
160
- if osp.isfile(p):
161
- impath = p
162
- break
163
- if impath is None:
164
- try:
165
- candidates = [f for f in listdir_nohidden(self.images_dir) if f.startswith(img_id + '.')]
166
- if candidates:
167
- impath = osp.join(self.images_dir, candidates[0])
168
- except Exception:
169
- pass
170
- if impath is None:
171
- continue
172
- data.append(Datum(impath=impath, label=int(cls_idx), classname=classes[cls_idx]))
173
- return data
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/README.md DELETED
@@ -1,227 +0,0 @@
1
- # DFA-MoE: Tackling Dual Forgetting in Vision-Language Continual Learning
2
-
3
- [![Python 3.8+](https://img.shields.io/badge/python-3.8+-blue.svg)](https://www.python.org/)
4
- [![PyTorch](https://img.shields.io/badge/PyTorch-red.svg)](https://pytorch.org/)
5
-
6
- Official PyTorch implementation of the paper "Don't Forget Why You Started: Tackling Dual Forgetting in Vision-Language Continual Learning" ICML2026.
7
-
8
- ## Abstract
9
-
10
- Vision-Language Models (VLMs) are strong continual learners, but standard class-incremental learning often damages the pre-trained vision-language alignment that underpins zero-shot generalization. This repository studies that dual-forgetting problem from two perspectives: Incremental Knowledge Forgetting (IKF), which harms previously learned classes, and Pre-trained Knowledge Forgetting (PKF), which erodes the original zero-shot capabilities of the VLM. To address this issue, we implement the Dual-Forgetting-Aware Class-Incremental Learning (DFA-CIL) framework, the Similarity-Calibrated Retention (SCR) metric, and DFA-MoE, a functionally heterogeneous PEFT method that decouples alignment preservation from task adaptation.
11
-
12
- ![Introduction](docs/intro.png)
13
-
14
- ## Key Contributions
15
-
16
- - Dual-forgetting evaluation for VLM continual learning through the DFA-CIL protocol.
17
- - Similarity-Calibrated Retention (SCR) utilizing similarity based weight to disentangle genuine foundational retention from the confounding effects of positive transfer.
18
- - DFA-MoE with two functional pathways:
19
- - Alignment Pathway: task-agnostic contrastive expert for PKF mitigation.
20
- - Plasticity Pathway: task-specific experts with classification and auxiliary contrastive learning for IKF mitigation.
21
- - Hierarchical routing with an inner router over task-specific experts and an outer router balancing alignment and plasticity outputs.
22
-
23
- ## Installation
24
-
25
- ### Setup Environment
26
-
27
- ```bash
28
- git clone https://github.com/RL-MIND/DFA-MoE
29
- cd DFA_MoE
30
- pip install -r requirements.txt
31
- ```
32
-
33
- ## Supported Downstream Datasets
34
-
35
- The current `configs/class` directory provides downstream class-incremental configs for the following dataset indices. These indices are the values used in `+train_dataset=[id]`.
36
-
37
- - `0`: `FGVCAircraft`
38
- - `1`: `Caltech101`
39
- - `2`: `CIFAR100`
40
- - `3`: `DescribableTextures`
41
- - `4`: `EuroSAT`
42
- - `5`: `OxfordFlowers`
43
- - `6`: `Food101`
44
- - `7`: `MNIST`
45
- - `8`: `OxfordPets`
46
- - `9`: `StanfordCars`
47
- - `10`: `SUN397`
48
- - `11`: `Country211`
49
- - `14`: `GTSRB`
50
- - `15`: `RESISC45`
51
- - `16`: `FER2013`
52
- - `17`: `UCF101`
53
- - `18`: `CIFAR10`
54
- - `19`: `STL10`
55
- - `20`: `VOC2007`
56
- - `21`: `ImageNetR`
57
- - `22`: `KittiDistance`
58
- - `24`: `CLEVRCount`
59
-
60
- The datasets pool also contains:
61
-
62
- - `12`: `SST2`
63
- - `13`: `HatefulMemes`
64
- - `23`: `PCam`
65
-
66
- These datasets are not supported as `--downstream-dataset` for `calculate_sim.py` because they are binary classification datasets.
67
-
68
- ## Data Preparation
69
-
70
- We will soon release all the datasets used in this work at [google drive](https://drive.google.com/drive/folders/1DxD6MixpyTcsLSv8Gah5V0Cp3yMupYZO?usp=sharing).
71
-
72
- 1. Download or prepare the datasets under a common `dataset_root`.
73
- 2. Update `dataset_root` in the command line when launching experiments.
74
- 3. Choose the dataset-specific config from `configs/class/`.
75
-
76
- Example dataset config files:
77
-
78
- - `configs/class/eurosat.yaml`
79
- - `configs/class/flower.yaml`
80
- - `configs/class/cifar100.yaml`
81
- - `configs/class/aircraft.yaml`
82
-
83
- ## Quick Start
84
-
85
- ### Train a DFA-CIL Run
86
-
87
- Example: EuroSAT with 5 class-incremental splits.
88
-
89
- ```bash
90
- python main.py \
91
- --config-path ./configs/class \
92
- --config-name eurosat.yaml \
93
- dataset_root="/path/to/data" \
94
- +train_dataset=[4] \
95
- +cil_splits=[5]
96
- ```
97
-
98
- Example: OxfordFlowers with 17 splits.
99
-
100
- ```bash
101
- python main.py \
102
- --config-path ./configs/class \
103
- --config-name flower.yaml \
104
- dataset_root="/path/to/data" \
105
- +train_dataset=[5] \
106
- +cil_splits=[17]
107
- ```
108
-
109
-
110
- ## Important Configuration Options
111
-
112
- This project uses Hydra-based configuration. Key parameters include:
113
-
114
- ```yaml
115
- model_name: "ViT-B/16"
116
- prompt_template: "a bad photo of a {}."
117
-
118
- batch_size: 128
119
- weight_decay: 0.0
120
- ls: 0.0
121
-
122
- epochs_a: 1
123
- epochs_b: 1
124
-
125
- lr_e1: 1.0e-5
126
- lr_e2: 5.0e-4
127
- text_lr_e2: 1.0e-5
128
- lr_e2_router: 1.0e-3
129
- lr_top_router: 1.0e-5
130
-
131
- tau_con: 0.15
132
- tau_b_con: 0.17
133
- lambda_b_con: 0.001
134
-
135
- num_task_experts: 2
136
- e2_top_k: 2
137
- moco_queue_size: 128
138
- ```
139
-
140
- ## Pre-task Zero-shot Baseline
141
-
142
- To compute SCR correctly, the metric log must contain the original CLIP zero-shot baseline `A_k^0`. Enable this by setting:
143
-
144
- ```yaml
145
- pre_task_zero_shot_eval: true
146
- zero_shot_eval: true
147
- ```
148
-
149
- or from the command line:
150
-
151
- ```bash
152
- python main.py \
153
- --config-path ./configs/class \
154
- --config-name eurosat.yaml \
155
- dataset_root="/path/to/data" \
156
- +train_dataset=[4] \
157
- +cil_splits=[5] \
158
- pre_task_zero_shot_eval=true \
159
- zero_shot_eval=true
160
- ```
161
-
162
- This writes a `task: -1` entry with `zs_pre` into `metrics.json`, which is required by `calculate_SCR.py`.
163
-
164
- ## SCR Evaluation Workflow
165
-
166
- ### Step 1: Run continual learning and save `metrics.json`
167
-
168
- Run `main.py` with `pre_task_zero_shot_eval=true` and `zero_shot_eval=true`.
169
-
170
- ### Step 2: Generate the similarity matrix
171
-
172
- Example: EuroSAT with 5 splits.
173
-
174
- ```bash
175
- python calculate_sim.py \
176
- --dataset-root "/path/to/data" \
177
- --downstream-dataset EuroSAT \
178
- --cil-split 5 \
179
- --output eurosat_similarity.json
180
- ```
181
-
182
- This script:
183
-
184
- - uses the original frozen CLIP model,
185
- - produces a task-to-upstream similarity matrix for SCR.
186
-
187
- ### Step 3: Compute SCR
188
-
189
- ```bash
190
- python calculate_SCR.py \
191
- --metric-path /path/to/metrics.json \
192
- --sim-json /path/to/eurosat_similarity_sim.json
193
- ```
194
-
195
- `calculate_SCR.py` now strictly requires:
196
-
197
- - a `task: -1` row with non-empty `zs_pre`,
198
- - per-task `zs` results in `metrics.json`,
199
- - a similarity JSON generated by `calculate_sim.py`.
200
-
201
- ## Reference Script
202
-
203
- The repository also contains `run.sh` as a reference batch script.
204
-
205
- ## Notes
206
-
207
- - `main.py`, `calculate_sim.py`, and `calculate_SCR.py` are the recommended entry points for experiments and evaluation.
208
-
209
- ## Citation
210
-
211
- If you find this repository useful in your research, please cite the paper:
212
-
213
- ```bibtex
214
- @inproceedings{kang2026dont,
215
- title={Don't Forget Why You Started: Tackling Dual Forgetting in Vision-Language Continual Learning},
216
- author={Kang, Borui and Gu, Jinrui and Feng, Tao and Fan, Qi and Shi, Yinghuan and Wang, Lei and Li, Wenbin and Gao, Yang},
217
- booktitle={Proceedings of the 43rd International Conference on Machine Learning},
218
- year={2026}
219
- }
220
- ```
221
-
222
- ## Acknowledgement
223
-
224
- This repository is built upon and modified from [MoE-Adapters4CL](https://github.com/JiazuoYu/MoE-Adapters4CL). We thank the original authors for making their code publicly available.
225
-
226
- The dataset processing pipeline in this repository is also based on [DIKI](https://github.com/lloongx/DIKI). We thank the authors for releasing their implementation.
227
-
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/calculate_SCR.py DELETED
@@ -1,266 +0,0 @@
1
- from __future__ import annotations
2
-
3
- import argparse
4
- import json
5
- from typing import List, Dict, Any, Tuple
6
-
7
-
8
- DATASET_NAME_ALIASES = {
9
- "Aircraft": "FGVCAircraft",
10
- "FGVCAircraft": "FGVCAircraft",
11
- "DTD": "DescribableTextures",
12
- "DescribableTextures": "DescribableTextures",
13
- }
14
-
15
-
16
- def canonicalize_dataset_name(name: str) -> str:
17
- return DATASET_NAME_ALIASES.get(str(name), str(name))
18
-
19
-
20
- def format_matrix(mat: List[List[float]], decimals: int = 3) -> str:
21
- if not mat:
22
- return "[]"
23
- fmt = f"{{:.{decimals}f}}"
24
- lines = []
25
- for row in mat:
26
- row_str = ", ".join(fmt.format(v) for v in row)
27
- lines.append(f"[{row_str}]")
28
- return "[" + "\n" + ",\n".join(lines) + "\n]"
29
-
30
-
31
- def load_lines(path: str) -> List[Dict[str, Any]]:
32
- rows = []
33
- with open(path, 'r', encoding='utf-8') as f:
34
- for line in f:
35
- line = line.strip()
36
- if not line:
37
- continue
38
- try:
39
- rows.append(json.loads(line))
40
- except Exception as e:
41
- print(f"[WARN] skip invalid JSON line: {e}")
42
- rows.sort(key=lambda r: int(r.get('task', 0)))
43
- return rows
44
-
45
-
46
- def build_zs_matrix(rows: List[Dict[str, Any]]) -> Tuple[List[List[float]], List[str]]:
47
- if not rows:
48
- return [], []
49
-
50
- baseline_row = None
51
- task_rows = []
52
-
53
- for r in rows:
54
- task_id = int(r.get('task', 0))
55
- zs_pre = r.get('zs_pre')
56
- zs_task = r.get('zs')
57
- zs_task_alt = r.get('zs_mtil')
58
-
59
- if task_id == -1 and isinstance(zs_pre, dict) and len(zs_pre) > 0:
60
- baseline_row = zs_pre
61
- continue
62
- if task_id >= 0:
63
- if isinstance(zs_task, dict) and len(zs_task) > 0:
64
- task_rows.append((task_id, zs_task))
65
- elif isinstance(zs_task_alt, dict) and len(zs_task_alt) > 0:
66
- task_rows.append((task_id, zs_task_alt))
67
-
68
- if baseline_row is None:
69
- raise ValueError(
70
- "metric.json must contain a pre-task baseline row with `task: -1` and non-empty `zs_pre`."
71
- )
72
- if not task_rows:
73
- return [], []
74
-
75
- names = list(baseline_row.keys())
76
- matrix: List[List[float]] = []
77
- matrix.append([float(baseline_row.get(name, 0.0)) for name in names])
78
- for _task_id, zs in sorted(task_rows, key=lambda item: item[0]):
79
- row = [float(zs.get(name, 0.0)) for name in names]
80
- matrix.append(row)
81
- return matrix, names
82
-
83
-
84
- def load_similarity_json(path: str) -> Tuple[List[List[float]], List[str]]:
85
- with open(path, 'r', encoding='utf-8') as f:
86
- payload = json.load(f)
87
- sim_mat_raw = payload.get("similarity_matrix", [])
88
- sim_names_raw = payload.get("upstream_names", [])
89
- sim_mat = [[float(x) for x in row] for row in sim_mat_raw]
90
- sim_names = [canonicalize_dataset_name(x) for x in sim_names_raw]
91
- return sim_mat, sim_names
92
-
93
-
94
- def compute_scores_with_matrix(
95
- zs_mat: List[List[float]],
96
- zs_names: List[str],
97
- sim_mat: List[List[float]],
98
- sim_names: List[str],
99
- ) -> List[float]:
100
- if not zs_mat or not sim_mat:
101
- return []
102
-
103
- zs_names = [canonicalize_dataset_name(nm) for nm in zs_names]
104
- zs_name_to_idx = {nm: i for i, nm in enumerate(zs_names)}
105
- sim_name_to_idx = {nm: i for i, nm in enumerate(sim_names)}
106
- common = [nm for nm in zs_names if nm in sim_name_to_idx]
107
- if len(common) < len(zs_names):
108
- missing = [nm for nm in zs_names if nm not in sim_name_to_idx]
109
- print(f"[WARN] Missing similarity columns for: {missing}")
110
- zs_cols = [zs_name_to_idx[nm] for nm in common]
111
- sim_cols = [sim_name_to_idx[nm] for nm in common]
112
-
113
- base = zs_mat[0]
114
- scores: List[float] = []
115
- T = min(len(sim_mat), max(0, len(zs_mat) - 1))
116
- if T < (len(zs_mat) - 1):
117
- print(f"[WARN] sim-matrix rows ({len(sim_mat)}) < tasks ({len(zs_mat) - 1}); truncating to {T}")
118
- for t in range(T):
119
- row = zs_mat[t + 1]
120
- total = 0.0
121
- sum_w = 0.0
122
- sims = [float(sim_mat[t][sim_cols[k]]) for k in range(len(common))]
123
- risks = [max(0.0, 1-s) for s in sims]
124
- norm = sum(risks) if risks else 0.0
125
- default_weight = 1.0 / max(1, len(common))
126
- for k in range(len(common)):
127
- i = zs_cols[k]
128
- diff = float(row[i]) - float(base[i])
129
- w = (risks[k] / norm) if norm > 0.0 else default_weight
130
- total += diff * w
131
- sum_w += w
132
- avg = (total / sum_w) if sum_w > 0.0 else 0.0
133
- scores.append(round(avg, 2))
134
- return scores
135
-
136
-
137
- def compute_scores_grouped(
138
- zs_mat: List[List[float]],
139
- zs_names: List[str],
140
- sim_mat: List[List[float]],
141
- sim_names: List[str],
142
- ) -> Dict[str, Any]:
143
- if not zs_mat or not sim_mat:
144
- return {
145
- "weighted": {"low": [], "mid": [], "high": []},
146
- }
147
-
148
- zs_names = [canonicalize_dataset_name(nm) for nm in zs_names]
149
- zs_name_to_idx = {nm: i for i, nm in enumerate(zs_names)}
150
- sim_name_to_idx = {nm: i for i, nm in enumerate(sim_names)}
151
- common = [nm for nm in zs_names if nm in sim_name_to_idx]
152
- if len(common) < len(zs_names):
153
- missing = [nm for nm in zs_names if nm not in sim_name_to_idx]
154
- print(f"[WARN] Missing similarity columns for grouped score: {missing}")
155
- zs_cols = [zs_name_to_idx[nm] for nm in common]
156
- sim_cols = [sim_name_to_idx[nm] for nm in common]
157
-
158
- base = zs_mat[0]
159
- T = min(len(sim_mat), max(0, len(zs_mat) - 1))
160
- if T < (len(zs_mat) - 1):
161
- print(f"[WARN] sim-matrix rows ({len(sim_mat)}) < tasks ({len(zs_mat) - 1}); truncating to {T}")
162
-
163
- def calc_weighted(items):
164
- total = sum(item["diff"] * item["weight"] for item in items)
165
- sum_w = sum(item["weight"] for item in items)
166
- return round((total / sum_w) if sum_w > 0.0 else 0.0, 2)
167
-
168
- w_low, w_mid, w_high = [], [], []
169
-
170
- for t in range(T):
171
- row = zs_mat[t + 1]
172
- sims = [float(sim_mat[t][sim_cols[k]]) for k in range(len(common))]
173
- risks = [max(0.0, 1.0 - s) for s in sims]
174
- norm = sum(risks) if risks else 0.0
175
- default_weight = 1.0 / max(1, len(common))
176
- triplets = []
177
- for k in range(len(common)):
178
- i = zs_cols[k]
179
- sim_val = sims[k]
180
- diff = float(row[i]) - float(base[i])
181
- w = (risks[k] / norm) if norm > 0.0 else default_weight
182
- triplets.append({
183
- "sim": sim_val,
184
- "diff": diff,
185
- "weight": w,
186
- "index": i + 1,
187
- })
188
- triplets.sort(key=lambda x: x["sim"])
189
- n = len(triplets)
190
- q = n // 3
191
- low = triplets[:q]
192
- mid = triplets[q:2 * q]
193
- high = triplets[2 * q:]
194
-
195
- w_low.append(calc_weighted(low))
196
- w_mid.append(calc_weighted(mid))
197
- w_high.append(calc_weighted(high))
198
-
199
- return {
200
- "weighted": {"low": w_low, "mid": w_mid, "high": w_high},
201
- }
202
-
203
-
204
- def main() -> None:
205
- parser = argparse.ArgumentParser(
206
- description="Compute SCR metrics from metric.json and calculate_sim similarity JSON.",
207
- )
208
- parser.add_argument("--metric-path", type=str, default="metric.json", help="Path to the metric.json file")
209
- parser.add_argument("--sim-json", type=str, required=True, help="Path to the similarity JSON generated by calculate_sim.py")
210
- parser.add_argument("--zs-decimals", type=int, default=2, help="Decimals when printing ZS matrix")
211
- args = parser.parse_args()
212
-
213
- rows = load_lines(args.metric_path)
214
- try:
215
- zs_mat, zs_names = build_zs_matrix(rows)
216
- except ValueError as e:
217
- print(f"[ERROR] {e}")
218
- return
219
- sim_mat, sim_names = load_similarity_json(args.sim_json)
220
-
221
- if not zs_mat:
222
- print("[ERROR] No zero-shot matrix could be extracted from metric.json.")
223
- return
224
- if not sim_mat:
225
- print("[ERROR] No similarity matrix could be extracted from the calculate_sim JSON.")
226
- return
227
-
228
- num_cols = len(zs_mat[0]) if zs_mat else 0
229
- if any(len(row) != num_cols for row in zs_mat):
230
- print("[ERROR] Extracted zero-shot matrix rows must all have the same length.")
231
- return
232
-
233
- sim_cols = len(sim_mat[0]) if sim_mat else 0
234
- if any(len(row) != sim_cols for row in sim_mat):
235
- print("[ERROR] similarity_matrix rows in the calculate_sim JSON must all have the same length.")
236
- return
237
- if sim_names and len(sim_names) != sim_cols:
238
- print("[ERROR] upstream_names length must match similarity_matrix columns.")
239
- return
240
- if not sim_names:
241
- sim_names = zs_names
242
-
243
- print("1. Zero-shot matrix (baseline + tasks):")
244
- print(format_matrix(zs_mat, decimals=args.zs_decimals))
245
-
246
- scores_weighted = compute_scores_with_matrix(zs_mat, zs_names, sim_mat, sim_names)
247
- avg_weighted = round(sum(scores_weighted) / len(scores_weighted), 2) if scores_weighted else 0.0
248
- print("\nSCR:")
249
- print(avg_weighted)
250
-
251
- grouped = compute_scores_grouped(zs_mat, zs_names, sim_mat, sim_names)
252
-
253
- def mean_or_zero(arr: list) -> float:
254
- return round(sum(arr) / len(arr), 2) if arr else 0.0
255
-
256
- weighted = grouped["weighted"]
257
- low_w_mean = mean_or_zero(weighted["low"])
258
- mid_w_mean = mean_or_zero(weighted["mid"])
259
- high_w_mean = mean_or_zero(weighted["high"])
260
-
261
- print("\nLow-Similarity SCR:", low_w_mean)
262
- print("Mid-Similarity SCR:", mid_w_mean)
263
- print("High-Similarity SCR:", high_w_mean)
264
-
265
- if __name__ == "__main__":
266
- main()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/calculate_sim.py DELETED
@@ -1,456 +0,0 @@
1
- import argparse
2
- import os
3
- from typing import List, Dict, Tuple
4
-
5
- import numpy as np
6
- import torch
7
- from omegaconf import OmegaConf
8
-
9
- # Project-local imports
10
- from mtil_datasets import get_dataset as get_mtil_dataset
11
- from continual_clip.mtil_cil import build_mtil_cil_scenarios
12
- from continual_clip.clip_original import load as load_orig_clip, tokenize as tokenize_orig
13
-
14
-
15
- DEFAULT_NAMES_ORDER1 = [
16
- "FGVCAircraft", "Caltech101", "CIFAR100", "DescribableTextures", "EuroSAT",
17
- "OxfordFlowers", "Food101", "MNIST", "OxfordPets", "StanfordCars",
18
- "SUN397", "Country211", "SST2", "HatefulMemes", "GTSRB",
19
- "RESISC45", "FER2013", "UCF101", "CIFAR10", "STL10",
20
- "VOC2007", "ImageNetR", "KittiDistance", "PCam", "CLEVRCount",
21
- ]
22
-
23
-
24
- DATASET_NAME_TO_CONFIG_STEM = {
25
- "FGVCAircraft": "aircraft",
26
- "Aircraft": "aircraft",
27
- "Caltech101": "caltech",
28
- "CIFAR100": "cifar100",
29
- "DescribableTextures": "dtd",
30
- "DTD": "dtd",
31
- "EuroSAT": "eurosat",
32
- "OxfordFlowers": "flower",
33
- "Food101": "food",
34
- "MNIST": "mnist",
35
- "OxfordPets": "pet",
36
- "StanfordCars": "car",
37
- "SUN397": "sun",
38
- "Country211": "conuntry",
39
- "GTSRB": "gtsrb",
40
- "RESISC45": "resisc45",
41
- "FER2013": "fer2013",
42
- "UCF101": "ucf",
43
- "CIFAR10": "cifar10",
44
- "STL10": "stl",
45
- "VOC2007": "voc",
46
- "ImageNetR": "image-r",
47
- "KittiDistance": "kitti",
48
- "CLEVRCount": "clevr",
49
- }
50
-
51
-
52
- UNSUPPORTED_DOWNSTREAM_DATASETS = {
53
- "SST2",
54
- "HatefulMemes",
55
- "PCam",
56
- }
57
-
58
-
59
- def resolve_downstream_seed(repo_root: str, downstream_name: str) -> int:
60
- if downstream_name in UNSUPPORTED_DOWNSTREAM_DATASETS:
61
- raise ValueError(
62
- f"Downstream dataset '{downstream_name}' is currently unsupported as `--downstream-dataset` because no matching `configs/class/*.yaml` file is available."
63
- )
64
- config_stem = DATASET_NAME_TO_CONFIG_STEM.get(downstream_name)
65
- if not config_stem:
66
- raise ValueError(
67
- f"No `configs/class/*.yaml` mapping is defined for downstream dataset '{downstream_name}'."
68
- )
69
- config_path = os.path.join(repo_root, "configs", "class", f"{config_stem}.yaml")
70
- if not os.path.exists(config_path):
71
- raise FileNotFoundError(
72
- f"Expected downstream config file '{config_path}' for dataset '{downstream_name}'."
73
- )
74
- cfg = OmegaConf.load(config_path)
75
- seed = OmegaConf.select(cfg, "seed")
76
- if seed is None:
77
- raise ValueError(
78
- f"Config file '{config_path}' does not define a `seed` entry."
79
- )
80
- return int(seed)
81
-
82
-
83
- def encode_texts(model, device, sentences: List[str], batch_size: int = 256) -> torch.Tensor:
84
- feats_all = []
85
- with torch.no_grad():
86
- for i in range(0, len(sentences), batch_size):
87
- chunk = sentences[i:i+batch_size]
88
- tokens = tokenize_orig(chunk).to(device)
89
- feats = model.encode_text(tokens)
90
- feats = feats / (feats.norm(dim=-1, keepdim=True) + 1e-12)
91
- feats_all.append(feats)
92
- if not feats_all:
93
- return torch.zeros((0, model.text_projection.shape[1]), device=device)
94
- return torch.cat(feats_all, dim=0)
95
-
96
-
97
- def build_text_class_prototypes(model, device, classnames: List[str], templates, batch_size: int = 256) -> torch.Tensor:
98
- D = model.text_projection.shape[1]
99
- class_vecs: List[torch.Tensor] = []
100
- for cname in classnames:
101
- sents: List[str] = []
102
- if templates and len(templates) > 0:
103
- for t in templates:
104
- try:
105
- sents.append(t(cname) if callable(t) else str(t).format(cname))
106
- except Exception:
107
- continue
108
- else:
109
- sents = [f"a photo of a {cname}."]
110
- feats = encode_texts(model, device, sents, batch_size=batch_size)
111
- if feats.numel() == 0:
112
- v = torch.zeros(D, device=device)
113
- else:
114
- v = feats.mean(dim=0)
115
- v = v / (v.norm() + 1e-12)
116
- class_vecs.append(v)
117
- if not class_vecs:
118
- return torch.zeros((0, D), device=device)
119
- return torch.stack(class_vecs, dim=0)
120
-
121
-
122
- def normalize_labels_to_list(lab) -> List[int]:
123
- if isinstance(lab, (int, np.integer)):
124
- return [int(lab)]
125
- import torch as _torch
126
- if _torch.is_tensor(lab):
127
- arr = lab.detach().cpu().numpy()
128
- if arr.ndim == 0:
129
- return [int(arr)]
130
- if arr.ndim == 1 and arr.size > 1 and set(np.unique(arr)).issubset({0,1}):
131
- return [int(x) for x in np.where(arr > 0.5)[0].tolist()]
132
- return [int(x) for x in arr.flatten().tolist()]
133
- if isinstance(lab, (list, tuple, np.ndarray)):
134
- arr = np.asarray(lab)
135
- if arr.ndim == 0:
136
- return [int(arr)]
137
- if arr.ndim == 1 and arr.size > 1 and set(np.unique(arr)).issubset({0,1}):
138
- return [int(x) for x in np.where(arr > 0.5)[0].tolist()]
139
- if arr.ndim == 1:
140
- return [int(x) for x in arr.tolist()]
141
- return [int(x) for x in np.where(arr.flatten() > 0.5)[0].tolist()]
142
- return []
143
-
144
-
145
- def build_visual_class_prototypes(model, device, ds_wrapper, num_classes: int, max_per_class: int, batch_size: int, preprocess_eval) -> torch.Tensor:
146
- D = None
147
- sums: Dict[int, torch.Tensor] = {}
148
- counts: Dict[int, int] = {i: 0 for i in range(num_classes)}
149
- imgs_batch: List[torch.Tensor] = []
150
- labels_batch_multi: List[List[int]] = []
151
- with torch.no_grad():
152
- for i in range(len(ds_wrapper)):
153
- img, lab = ds_wrapper[i]
154
- lab_ids = [lid for lid in normalize_labels_to_list(lab) if 0 <= lid < num_classes and counts[lid] < max_per_class]
155
- if not lab_ids:
156
- continue
157
- if isinstance(img, torch.Tensor):
158
- tensor_img = img
159
- else:
160
- tensor_img = preprocess_eval(img)
161
- imgs_batch.append(tensor_img.unsqueeze(0))
162
- labels_batch_multi.append(lab_ids)
163
- if len(imgs_batch) >= max(1, batch_size):
164
- batch = torch.cat(imgs_batch, dim=0).to(device)
165
- feats = model.encode_image(batch)
166
- feats = feats / (feats.norm(dim=-1, keepdim=True) + 1e-12)
167
- if D is None:
168
- D = int(feats.shape[1])
169
- for f, ls in zip(feats, labels_batch_multi):
170
- for l in ls:
171
- if counts[l] >= max_per_class:
172
- continue
173
- if l not in sums:
174
- sums[l] = f.detach().clone()
175
- else:
176
- sums[l] = sums[l] + f.detach()
177
- counts[l] += 1
178
- imgs_batch.clear()
179
- labels_batch_multi.clear()
180
- if all(counts[l] >= max_per_class for l in range(num_classes)):
181
- break
182
- if imgs_batch:
183
- batch = torch.cat(imgs_batch, dim=0).to(device)
184
- feats = model.encode_image(batch)
185
- feats = feats / (feats.norm(dim=-1, keepdim=True) + 1e-12)
186
- if D is None:
187
- D = int(feats.shape[1])
188
- for f, ls in zip(feats, labels_batch_multi):
189
- for l in ls:
190
- if counts[l] >= max_per_class:
191
- continue
192
- if l not in sums:
193
- sums[l] = f.detach().clone()
194
- else:
195
- sums[l] = sums[l] + f.detach()
196
- counts[l] += 1
197
- imgs_batch.clear()
198
- labels_batch_multi.clear()
199
- class_vecs: List[torch.Tensor] = []
200
- for lid in range(num_classes):
201
- c = counts.get(lid, 0)
202
- if c <= 0:
203
- class_vecs.append(torch.zeros(int(D or 0), device=device))
204
- else:
205
- m = sums[lid] / float(c)
206
- m = m / (m.norm() + 1e-12)
207
- class_vecs.append(m)
208
- if not class_vecs:
209
- return torch.zeros((0, int(D or 0)), device=device)
210
- return torch.stack(class_vecs, dim=0)
211
-
212
-
213
- def build_visual_class_prototypes_subset(model, device, ds_wrapper, label_ids: List[int], max_per_class: int, batch_size: int, preprocess_eval) -> torch.Tensor:
214
- D = None
215
- sums: Dict[int, torch.Tensor] = {}
216
- counts: Dict[int, int] = {lid: 0 for lid in label_ids}
217
- label_set = set(label_ids)
218
- imgs_batch: List[torch.Tensor] = []
219
- labels_batch_multi: List[List[int]] = []
220
- with torch.no_grad():
221
- for i in range(len(ds_wrapper)):
222
- img, lab = ds_wrapper[i]
223
- lab_all = [lid for lid in normalize_labels_to_list(lab) if lid in label_set and counts.get(lid, 0) < max_per_class]
224
- if not lab_all:
225
- continue
226
- tensor_img = img if isinstance(img, torch.Tensor) else preprocess_eval(img)
227
- imgs_batch.append(tensor_img.unsqueeze(0))
228
- labels_batch_multi.append(lab_all)
229
- if len(imgs_batch) >= max(1, batch_size):
230
- batch = torch.cat(imgs_batch, dim=0).to(device)
231
- feats = model.encode_image(batch)
232
- feats = feats / (feats.norm(dim=-1, keepdim=True) + 1e-12)
233
- if D is None:
234
- D = int(feats.shape[1])
235
- for f, ls in zip(feats, labels_batch_multi):
236
- for l in ls:
237
- if counts[l] >= max_per_class:
238
- continue
239
- if l not in sums:
240
- sums[l] = f.detach().clone()
241
- else:
242
- sums[l] = sums[l] + f.detach()
243
- counts[l] += 1
244
- imgs_batch.clear()
245
- labels_batch_multi.clear()
246
- if all(counts[lid] >= max_per_class for lid in label_ids):
247
- break
248
- if imgs_batch:
249
- batch = torch.cat(imgs_batch, dim=0).to(device)
250
- feats = model.encode_image(batch)
251
- feats = feats / (feats.norm(dim=-1, keepdim=True) + 1e-12)
252
- if D is None:
253
- D = int(feats.shape[1])
254
- for f, ls in zip(feats, labels_batch_multi):
255
- for l in ls:
256
- if counts[l] >= max_per_class:
257
- continue
258
- if l not in sums:
259
- sums[l] = f.detach().clone()
260
- else:
261
- sums[l] = sums[l] + f.detach()
262
- counts[l] += 1
263
- imgs_batch.clear()
264
- labels_batch_multi.clear()
265
- class_vecs: List[torch.Tensor] = []
266
- for lid in label_ids:
267
- c = counts.get(lid, 0)
268
- if c <= 0:
269
- class_vecs.append(torch.zeros(int(D or 0), device=device))
270
- else:
271
- m = sums[lid] / float(c)
272
- m = m / (m.norm() + 1e-12)
273
- class_vecs.append(m)
274
- if not class_vecs:
275
- return torch.zeros((0, int(D or 0)), device=device)
276
- return torch.stack(class_vecs, dim=0)
277
-
278
-
279
- def cosine_distance_matrix(A: torch.Tensor, B: torch.Tensor) -> torch.Tensor:
280
- S = (A @ B.t()).clamp(-1.0, 1.0)
281
- return 1.0 - S
282
-
283
-
284
- def directed_covering_distance(T_src: torch.Tensor, V_src: torch.Tensor,
285
- T_tgt: torch.Tensor, V_tgt: torch.Tensor) -> float:
286
- if V_src.size(0) == 0 or V_tgt.size(0) == 0:
287
- return 0.0
288
- F_src = torch.cat([V_src, T_src], dim=1)
289
- F_tgt = torch.cat([V_tgt, T_tgt], dim=1)
290
- F_src = F_src / (F_src.norm(dim=-1, keepdim=True) + 1e-12)
291
- F_tgt = F_tgt / (F_tgt.norm(dim=-1, keepdim=True) + 1e-12)
292
- C = cosine_distance_matrix(F_src, F_tgt)
293
- d_row_min = C.min(dim=1).values
294
- return float(d_row_min.mean().item())
295
-
296
-
297
- def tasks_to_upstream_similarity_matrix(vision_model, device,
298
- upstream_names: List[str], name_to_idx: Dict[str, int],
299
- dataset_list, classes_names_list, templates_list,
300
- downstream_name: str, cil_splits: int,
301
- preprocess_eval,
302
- max_images_per_class: int, vision_batch_size: int,
303
- dataset_root: str,
304
- beta: float,
305
- clip_text_model=None,
306
- seed: int = 32) -> Tuple[np.ndarray, List[str]]:
307
- T_up_list: List[torch.Tensor] = []
308
- V_up_list: List[torch.Tensor] = []
309
- for nm in upstream_names:
310
- i = name_to_idx[nm]
311
- classnames = classes_names_list[i]
312
- templates = templates_list[i]
313
- ds_wrapper = dataset_list[i]
314
- K = len(classnames)
315
- T_k = build_text_class_prototypes(clip_text_model, device, classnames, templates, batch_size=256)
316
- V_k = build_visual_class_prototypes(vision_model, device, ds_wrapper, K, max_per_class=int(max_images_per_class), batch_size=int(vision_batch_size), preprocess_eval=preprocess_eval)
317
- T_up_list.append(T_k)
318
- V_up_list.append(V_k)
319
-
320
- ds_idx = name_to_idx[downstream_name]
321
- cfg_ds = type("Cfg", (), {})()
322
- cfg_ds.dataset = "MTIL"
323
- cfg_ds.dataset_root = dataset_root
324
- cfg_ds.MTIL_order_2 = False
325
- cfg_ds.train_one_dataset = ds_idx
326
- cfg_ds.seed = int(seed)
327
- cfg_ds.use_validation = False
328
- train_list, train_classes_names, train_templates, _ = get_mtil_dataset(
329
- cfg_ds, 'train', transforms=preprocess_eval
330
- )
331
- test_list, _, _, _ = get_mtil_dataset(
332
- cfg_ds, 'test', transforms=preprocess_eval
333
- )
334
- assert len(train_list) == 1 and len(test_list) == 1, "Expected single selected dataset for downstream"
335
- classnames_single = train_classes_names[0]
336
- templates_single = train_templates[0]
337
- _, _, _, class_ids_per_task, _ = build_mtil_cil_scenarios(
338
- train_list[0], test_list[0], classnames_single, cil_splits, seed=int(seed)
339
- )
340
-
341
- distances = np.zeros((cil_splits, len(upstream_names)), dtype=float)
342
- for t, cls_ids in enumerate(class_ids_per_task):
343
- cls_names_t = [classnames_single[c] for c in cls_ids]
344
- T_t = build_text_class_prototypes(clip_text_model, device, cls_names_t, templates_single, batch_size=256)
345
- ds_down = train_list[0]
346
- V_t = build_visual_class_prototypes_subset(vision_model, device, ds_down, list(cls_ids), max_per_class=int(max_images_per_class), batch_size=int(vision_batch_size), preprocess_eval=preprocess_eval)
347
- for j, nm in enumerate(upstream_names):
348
- distances[t, j] = directed_covering_distance(T_t, V_t, T_up_list[j], V_up_list[j])
349
-
350
- similarity = np.exp(-float(beta) * distances)
351
- similarity = np.clip(similarity, 0.0, 1.0)
352
- return similarity, upstream_names
353
-
354
-
355
- def main():
356
- parser = argparse.ArgumentParser(description="Dataset similarity based on CLIP text and visual prototypes")
357
- parser.add_argument("--dataset-root", type=str, default=os.environ.get("DATASET_ROOT", "data"))
358
- parser.add_argument("--model-name", type=str, default="ViT-B/16")
359
- parser.add_argument("--vision-batch-size", type=int, default=64)
360
- parser.add_argument("--max-images-per-class", type=int, default=64, help="Maximum number of samples to use for each class when building visual prototypes. For efficiency, Chamfer distance is not computed using all samples in a dataset; instead, each class is sampled with at most this many examples.")
361
- parser.add_argument("--beta", type=float, default=1.0, help="Similarity mapping exp(-beta * dist)")
362
- parser.add_argument("--output", type=str, default="")
363
- parser.add_argument("--downstream-dataset", type=str, default="", help="Name of downstream dataset (e.g., CIFAR100, PCam, FGVCAircraft, DescribableTextures)")
364
- parser.add_argument("--cil-split", type=int, default=0, help="Number of CIL splits for downstream dataset; if >0, output [cil_split x 24] matrix")
365
- args = parser.parse_args()
366
-
367
- names = DEFAULT_NAMES_ORDER1
368
-
369
- device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
370
- clip_model, _, preprocess_eval_clip = load_orig_clip(args.model_name, device=device, jit=False)
371
- clip_model.eval()
372
-
373
- vision_model = clip_model
374
- preprocess_eval = preprocess_eval_clip
375
- clip_text_model = clip_model
376
-
377
- cfg = type("Cfg", (), {})()
378
- cfg.dataset = "MTIL"
379
- cfg.dataset_root = args.dataset_root
380
- cfg.MTIL_order_2 = False
381
- cfg.train_one_dataset = -1
382
- cfg.seed = 32
383
- cfg.use_validation = False
384
-
385
- dataset_list, classes_names_list, templates_list, _ = get_mtil_dataset(
386
- cfg, 'test', transforms=preprocess_eval
387
- )
388
-
389
- n = min(len(names), len(classes_names_list))
390
- names = names[:n]
391
- classes_names_list = classes_names_list[:n]
392
- templates_list = templates_list[:n] if templates_list is not None else [None] * n
393
-
394
- if args.downstream_dataset and int(args.cil_split) > 0:
395
- repo_root = os.path.dirname(os.path.abspath(__file__))
396
- alias_in = (args.downstream_dataset or '').strip()
397
- alias_map = {
398
- "Aircraft": "FGVCAircraft",
399
- "FGVCAircraft": "FGVCAircraft",
400
- "DTD": "DescribableTextures",
401
- "DescribableTextures": "DescribableTextures",
402
- }
403
- ds_internal = alias_map.get(alias_in, alias_in) if alias_in else None
404
- downstream_seed = resolve_downstream_seed(repo_root, ds_internal)
405
- name_to_idx = {nm: idx for idx, nm in enumerate(names)}
406
- if ds_internal not in name_to_idx:
407
- raise ValueError(f"Downstream dataset '{args.downstream_dataset}' (mapped to '{ds_internal}') not found in MTIL names.")
408
- cfg.seed = downstream_seed
409
- dataset_list, classes_names_list, templates_list, _ = get_mtil_dataset(
410
- cfg, 'test', transforms=preprocess_eval
411
- )
412
- n = min(len(names), len(classes_names_list))
413
- names = names[:n]
414
- classes_names_list = classes_names_list[:n]
415
- templates_list = templates_list[:n] if templates_list is not None else [None] * n
416
- upstream_names = [nm for nm in names if nm != ds_internal]
417
- similarity_matrix, upstream_names = tasks_to_upstream_similarity_matrix(
418
- vision_model, device,
419
- upstream_names, name_to_idx,
420
- dataset_list, classes_names_list, templates_list,
421
- ds_internal, int(args.cil_split),
422
- preprocess_eval,
423
- int(args.max_images_per_class), int(args.vision_batch_size),
424
- args.dataset_root, float(args.beta), clip_text_model=clip_text_model,
425
- seed=downstream_seed,
426
- )
427
- print("DEFAULT_SIM_MATRIX = [")
428
- fmt = "{:.3f}"
429
- for i in range(similarity_matrix.shape[0]):
430
- row_str = ", ".join(fmt.format(float(x)) for x in similarity_matrix[i])
431
- print(f" [{row_str}],")
432
- print("]")
433
- names_py = ", ".join([f'"{n}"' for n in upstream_names])
434
- print(f"DEFAULT_SIM_UPSTREAM_NAMES = [{names_py}]")
435
- if args.output:
436
- import json
437
- out = {
438
- "downstream": args.downstream_dataset,
439
- "cil_split": int(args.cil_split),
440
- "upstream_names": upstream_names,
441
- "similarity_matrix": similarity_matrix.tolist(),
442
- "beta": float(args.beta),
443
- "seed": int(downstream_seed),
444
- "max_images_per_class": int(args.max_images_per_class),
445
- "vision_encoder": "clip",
446
- "text_encoder": "clip",
447
- }
448
- base, ext = os.path.splitext(args.output)
449
- out2 = base + "_sim" + ext
450
- with open(out2, 'w', encoding='utf-8') as f:
451
- json.dump(out, f, ensure_ascii=False, indent=2)
452
- print(f"Saved similarity JSON to {out2}")
453
- return
454
-
455
- if __name__ == "__main__":
456
- main()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/class_orders/cifar100.yaml DELETED
@@ -1 +0,0 @@
1
- class_order: [87, 0, 52, 58, 44, 91, 68, 97, 51, 15, 94, 92, 10, 72, 49, 78, 61, 14, 8, 86, 84, 96, 18, 24, 32, 45, 88, 11, 4, 67, 69, 66, 77, 47, 79, 93, 29, 50, 57, 83, 17, 81, 41, 12, 37, 59, 25, 20, 80, 73, 1, 28, 6, 46, 62, 82, 53, 9, 31, 75, 38, 63, 33, 74, 27, 22, 36, 3, 16, 21, 60, 19, 70, 90, 89, 43, 5, 42, 65, 76, 40, 30, 23, 85, 2, 95, 56, 48, 71, 64, 98, 13, 99, 7, 34, 55, 54, 26, 35, 39]
 
 
DFA-MoE/class_orders/imagenet100.yaml DELETED
@@ -1,11 +0,0 @@
1
- class_order: [
2
- 68, 56, 78, 8, 23, 84, 90, 65, 74, 76, 40,
3
- 89, 3, 92, 55, 9, 26, 80, 43, 38, 58, 70,
4
- 77, 1, 85, 19, 17, 50, 28, 53, 13, 81, 45,
5
- 82, 6, 59, 83, 16, 15, 44, 91, 41, 72, 60,
6
- 79, 52, 20, 10, 31, 54, 37, 95, 14, 71, 96,
7
- 98, 97, 2, 64, 66, 42, 22, 35, 86, 24, 34,
8
- 87, 21, 99, 0, 88, 27, 18, 94, 11, 12, 47,
9
- 25, 30, 46, 62, 69, 36, 61, 7, 63, 75, 5, 32,
10
- 4, 51, 48, 73, 93, 39, 67, 29, 49, 57, 33
11
- ]
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/class_orders/imagenet1000.yaml DELETED
@@ -1,48 +0,0 @@
1
- class_order: [
2
- 54, 7, 894, 512, 126, 337, 988, 11, 284, 493, 133, 783, 192, 979, 622, 215, 240, 548, 238, 419, 274, 108,
3
- 928, 856, 494, 836, 473, 650, 85, 262, 508, 590, 390, 174, 637, 288, 658, 219, 912, 142, 852, 160, 704, 289,
4
- 123, 323, 600, 542, 999, 634, 391, 761, 490, 842, 127, 850, 665, 990, 597, 722, 748, 14, 77, 437, 394, 859,
5
- 279, 539, 75, 466, 886, 312, 303, 62, 966, 413, 959, 782, 509, 400, 471, 632, 275, 730, 105, 523, 224, 186,
6
- 478, 507, 470, 906, 699, 989, 324, 812, 260, 911, 446, 44, 765, 759, 67, 36, 5, 30, 184, 797, 159, 741, 954,
7
- 465, 533, 585, 150, 101, 897, 363, 818, 620, 824, 154, 956, 176, 588, 986, 172, 223, 461, 94, 141, 621, 659,
8
- 360, 136, 578, 163, 427, 70, 226, 925, 596, 336, 412, 731, 755, 381, 810, 69, 898, 310, 120, 752, 93, 39,
9
- 326, 537, 905, 448, 347, 51, 615, 601, 229, 947, 348, 220, 949, 972, 73, 913, 522, 193, 753, 921, 257, 957,
10
- 691, 155, 820, 584, 948, 92, 582, 89, 379, 392, 64, 904, 169, 216, 694, 103, 410, 374, 515, 484, 624, 409,
11
- 156, 455, 846, 344, 371, 468, 844, 276, 740, 562, 503, 831, 516, 663, 630, 763, 456, 179, 996, 936, 248,
12
- 333, 941, 63, 738, 802, 372, 828, 74, 540, 299, 750, 335, 177, 822, 643, 593, 800, 459, 580, 933, 306, 378,
13
- 76, 227, 426, 403, 322, 321, 808, 393, 27, 200, 764, 651, 244, 479, 3, 415, 23, 964, 671, 195, 569, 917,
14
- 611, 644, 707, 355, 855, 8, 534, 657, 571, 811, 681, 543, 313, 129, 978, 592, 573, 128, 243, 520, 887, 892,
15
- 696, 26, 551, 168, 71, 398, 778, 529, 526, 792, 868, 266, 443, 24, 57, 15, 871, 678, 745, 845, 208, 188,
16
- 674, 175, 406, 421, 833, 106, 994, 815, 581, 676, 49, 619, 217, 631, 934, 932, 568, 353, 863, 827, 425, 420,
17
- 99, 823, 113, 974, 438, 874, 343, 118, 340, 472, 552, 937, 0, 10, 675, 316, 879, 561, 387, 726, 255, 407,
18
- 56, 927, 655, 809, 839, 640, 297, 34, 497, 210, 606, 971, 589, 138, 263, 587, 993, 973, 382, 572, 735, 535,
19
- 139, 524, 314, 463, 895, 376, 939, 157, 858, 457, 935, 183, 114, 903, 767, 666, 22, 525, 902, 233, 250, 825,
20
- 79, 843, 221, 214, 205, 166, 431, 860, 292, 976, 739, 899, 475, 242, 961, 531, 110, 769, 55, 701, 532, 586,
21
- 729, 253, 486, 787, 774, 165, 627, 32, 291, 962, 922, 222, 705, 454, 356, 445, 746, 776, 404, 950, 241, 452,
22
- 245, 487, 706, 2, 137, 6, 98, 647, 50, 91, 202, 556, 38, 68, 649, 258, 345, 361, 464, 514, 958, 504, 826,
23
- 668, 880, 28, 920, 918, 339, 315, 320, 768, 201, 733, 575, 781, 864, 617, 171, 795, 132, 145, 368, 147, 327,
24
- 713, 688, 848, 690, 975, 354, 853, 148, 648, 300, 436, 780, 693, 682, 246, 449, 492, 162, 97, 59, 357, 198,
25
- 519, 90, 236, 375, 359, 230, 476, 784, 117, 940, 396, 849, 102, 122, 282, 181, 130, 467, 88, 271, 793, 151,
26
- 847, 914, 42, 834, 521, 121, 29, 806, 607, 510, 837, 301, 669, 78, 256, 474, 840, 52, 505, 547, 641, 987,
27
- 801, 629, 491, 605, 112, 429, 401, 742, 528, 87, 442, 910, 638, 785, 264, 711, 369, 428, 805, 744, 380, 725,
28
- 480, 318, 997, 153, 384, 252, 985, 538, 654, 388, 100, 432, 832, 565, 908, 367, 591, 294, 272, 231, 213,
29
- 196, 743, 817, 433, 328, 970, 969, 4, 613, 182, 685, 724, 915, 311, 931, 865, 86, 119, 203, 268, 718, 317,
30
- 926, 269, 161, 209, 807, 645, 513, 261, 518, 305, 758, 872, 58, 65, 146, 395, 481, 747, 41, 283, 204, 564,
31
- 185, 777, 33, 500, 609, 286, 567, 80, 228, 683, 757, 942, 134, 673, 616, 960, 450, 350, 544, 830, 736, 170,
32
- 679, 838, 819, 485, 430, 190, 566, 511, 482, 232, 527, 411, 560, 281, 342, 614, 662, 47, 771, 861, 692, 686,
33
- 277, 373, 16, 946, 265, 35, 9, 884, 909, 610, 358, 18, 737, 977, 677, 803, 595, 135, 458, 12, 46, 418, 599,
34
- 187, 107, 992, 770, 298, 104, 351, 893, 698, 929, 502, 273, 20, 96, 791, 636, 708, 267, 867, 772, 604, 618,
35
- 346, 330, 554, 816, 664, 716, 189, 31, 721, 712, 397, 43, 943, 804, 296, 109, 576, 869, 955, 17, 506, 963,
36
- 786, 720, 628, 779, 982, 633, 891, 734, 980, 386, 365, 794, 325, 841, 878, 370, 695, 293, 951, 66, 594, 717,
37
- 116, 488, 796, 983, 646, 499, 53, 1, 603, 45, 424, 875, 254, 237, 199, 414, 307, 362, 557, 866, 341, 19,
38
- 965, 143, 555, 687, 235, 790, 125, 173, 364, 882, 727, 728, 563, 495, 21, 558, 709, 719, 877, 352, 83, 998,
39
- 991, 469, 967, 760, 498, 814, 612, 715, 290, 72, 131, 259, 441, 924, 773, 48, 625, 501, 440, 82, 684, 862,
40
- 574, 309, 408, 680, 623, 439, 180, 652, 968, 889, 334, 61, 766, 399, 598, 798, 653, 930, 149, 249, 890, 308,
41
- 881, 40, 835, 577, 422, 703, 813, 857, 995, 602, 583, 167, 670, 212, 751, 496, 608, 84, 639, 579, 178, 489,
42
- 37, 197, 789, 530, 111, 876, 570, 700, 444, 287, 366, 883, 385, 536, 460, 851, 81, 144, 60, 251, 13, 953,
43
- 270, 944, 319, 885, 710, 952, 517, 278, 656, 919, 377, 550, 207, 660, 984, 447, 553, 338, 234, 383, 749,
44
- 916, 626, 462, 788, 434, 714, 799, 821, 477, 549, 661, 206, 667, 541, 642, 689, 194, 152, 981, 938, 854,
45
- 483, 332, 280, 546, 389, 405, 545, 239, 896, 672, 923, 402, 423, 907, 888, 140, 870, 559, 756, 25, 211, 158,
46
- 723, 635, 302, 702, 453, 218, 164, 829, 247, 775, 191, 732, 115, 331, 901, 416, 873, 754, 900, 435, 762,
47
- 124, 304, 329, 349, 295, 95, 451, 285, 225, 945, 697, 417
48
- ]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/class_orders/tinyimagenet.yaml DELETED
@@ -1,17 +0,0 @@
1
- class_order: [
2
- 131, 181, 22, 172, 144, 92, 97, 187, 58, 93, 6, 70, 106, 68,
3
- 153, 168, 179, 199, 29, 46, 9, 142, 134, 88, 193, 110, 26,
4
- 32, 117, 112, 17, 39, 166, 13, 94, 138, 109, 147, 51, 101,
5
- 59, 188, 116, 5, 170, 99, 100, 167, 180, 146, 65, 1, 104,
6
- 43, 38, 184, 123, 171, 137, 162, 71, 44, 95, 174, 12, 7,
7
- 54, 152, 21, 47, 28, 176, 34, 2, 132, 118, 42, 189, 150,
8
- 14, 165, 41, 192, 45, 82, 128, 63, 57, 197, 160, 53, 75,
9
- 108, 135, 121, 159, 183, 67, 169, 50, 87, 69, 89, 196,
10
- 115, 19, 148, 96, 86, 11, 8, 60, 33, 173, 78, 4, 119, 105,
11
- 182, 127, 177, 30, 186, 40, 49, 178, 76, 157, 161, 73, 164,
12
- 151, 31, 74, 191, 27, 125, 198, 81, 20, 155, 114, 139, 36,
13
- 61, 56, 145, 48, 16, 83, 62, 85, 126, 0, 102, 23, 3, 140,
14
- 15, 195, 133, 113, 190, 141, 52, 163, 156, 80, 111, 90, 175,
15
- 143, 120, 84, 18, 25, 79, 37, 154, 136, 64, 158, 24, 185,
16
- 72, 35, 129, 55, 149, 91, 122, 77, 103, 124, 130, 66, 10, 107, 194, 98
17
- ]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/clip/README.md DELETED
@@ -1 +0,0 @@
1
- This folder is a lightly modified version of https://github.com/openai/CLIP.
 
 
DFA-MoE/clip/__init__.py DELETED
@@ -1 +0,0 @@
1
- from .clip import *
 
 
DFA-MoE/clip/adapter.py DELETED
@@ -1,69 +0,0 @@
1
- # --------------------------------------------------------
2
- # References:
3
- # https://github.com/jxhe/unify-parameter-efficient-tuning
4
- # --------------------------------------------------------
5
-
6
- import math
7
- import torch
8
- import torch.nn as nn
9
-
10
-
11
- class Adapter(nn.Module):
12
- def __init__(self,
13
- d_model=None,
14
- bottleneck=None,
15
- dropout=0.0,
16
- init_option="lora",
17
- adapter_scalar="1.0",
18
- adapter_layernorm_option="in"):
19
- super().__init__()
20
- self.n_embd = d_model if d_model is None else d_model
21
- self.down_size = bottleneck
22
-
23
- #_before
24
- self.adapter_layernorm_option = adapter_layernorm_option
25
-
26
- self.adapter_layer_norm_before = None
27
- if adapter_layernorm_option == "in" or adapter_layernorm_option == "out":
28
- self.adapter_layer_norm_before = nn.LayerNorm(self.n_embd)
29
-
30
- if adapter_scalar == "learnable_scalar":
31
- self.scale = nn.Parameter(torch.ones(1))
32
- else:
33
- self.scale = float(adapter_scalar)
34
-
35
- self.down_proj = nn.Linear(self.n_embd, 64)
36
- self.non_linear_func = nn.ReLU()
37
- self.up_proj = nn.Linear(self.down_size, self.n_embd)
38
-
39
- self.dropout = dropout
40
- if init_option == "bert":
41
- raise NotImplementedError
42
- elif init_option == "lora":
43
- with torch.no_grad():
44
- nn.init.kaiming_uniform_(self.down_proj.weight, a=math.sqrt(5))
45
- nn.init.zeros_(self.up_proj.weight)
46
- nn.init.zeros_(self.down_proj.bias)
47
- nn.init.zeros_(self.up_proj.bias)
48
-
49
- def forward(self, x, add_residual=True, residual=None):
50
-
51
- residual = x if residual is None else residual
52
- if self.adapter_layernorm_option == 'in': # none
53
- x = self.adapter_layer_norm_before(x)
54
-
55
- down = self.down_proj(x)
56
- down = self.non_linear_func(down)
57
- down = nn.functional.dropout(down, p=self.dropout, training=self.training)
58
- up = self.up_proj(down)
59
-
60
- up = up * self.scale
61
-
62
- if self.adapter_layernorm_option == 'out': # none
63
- up = self.adapter_layer_norm_before(up)
64
-
65
- if add_residual:
66
- output = up + residual
67
- else:
68
- output = up
69
- return output
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/clip/bpe_simple_vocab_16e6.txt.gz DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:924691ac288e54409236115652ad4aa250f48203de50a9e4722a6ecd48d6804a
3
- size 1356917
 
 
 
 
DFA-MoE/clip/clip.py DELETED
@@ -1,310 +0,0 @@
1
- # Code ported from https://github.com/openai/CLIP
2
-
3
- import hashlib
4
- import os
5
- import urllib
6
- import warnings
7
- from typing import Union, List
8
-
9
- import torch
10
- from torchvision.transforms import Compose, Resize, CenterCrop, ToTensor, Normalize, RandomResizedCrop, InterpolationMode
11
- from tqdm import tqdm
12
-
13
- from clip.model import build_model
14
- from clip.tokenizer import SimpleTokenizer as _Tokenizer
15
-
16
- __all__ = ["available_models", "load", "tokenize"]
17
- _tokenizer = _Tokenizer()
18
-
19
- _MODELS = {
20
- "RN50": "https://openaipublic.azureedge.net/clip/models/afeb0e10f9e5a86da6080e35cf09123aca3b358a0c3e3b6c78a7b63bc04b6762/RN50.pt",
21
- "RN101": "https://openaipublic.azureedge.net/clip/models/8fa8567bab74a42d41c5915025a8e4538c3bdbe8804a470a72f30b0d94fab599/RN101.pt",
22
- "RN50x4": "https://openaipublic.azureedge.net/clip/models/7e526bd135e493cef0776de27d5f42653e6b4c8bf9e0f653bb11773263205fdd/RN50x4.pt",
23
- "RN50x16": "https://openaipublic.azureedge.net/clip/models/52378b407f34354e150460fe41077663dd5b39c54cd0bfd2b27167a4a06ec9aa/RN50x16.pt",
24
- "ViT-B/32": "https://openaipublic.azureedge.net/clip/models/40d365715913c9da98579312b702a82c18be219cc2a73407c4526f58eba950af/ViT-B-32.pt",
25
- "ViT-B/16": "https://openaipublic.azureedge.net/clip/models/5806e77cd80f8b59890b7e101eabd078d9fb84e6937f9e85e4ecb61988df416f/ViT-B-16.pt",
26
- }
27
-
28
-
29
- def _download(url: str, root: str = os.path.expanduser("~/.cache/clip")):
30
- os.makedirs(root, exist_ok=True)
31
- filename = os.path.basename(url)
32
-
33
- expected_sha256 = url.split("/")[-2]
34
- download_target = os.path.join(root, filename)
35
-
36
- if os.path.exists(download_target) and not os.path.isfile(download_target):
37
- raise RuntimeError(f"{download_target} exists and is not a regular file")
38
-
39
- if os.path.isfile(download_target):
40
- if hashlib.sha256(open(download_target, "rb").read()).hexdigest() == expected_sha256:
41
- return download_target
42
- else:
43
- warnings.warn(f"{download_target} exists, but the SHA256 checksum does not match; re-downloading the file")
44
-
45
- with urllib.request.urlopen(url) as source, open(download_target, "wb") as output:
46
- with tqdm(total=int(source.info().get("Content-Length")), ncols=80, unit='iB', unit_scale=True) as loop:
47
- while True:
48
- buffer = source.read(8192)
49
- if not buffer:
50
- break
51
-
52
- output.write(buffer)
53
- loop.update(len(buffer))
54
-
55
- if hashlib.sha256(open(download_target, "rb").read()).hexdigest() != expected_sha256:
56
- raise RuntimeError(f"Model has been downloaded but the SHA256 checksum does not not match")
57
-
58
- return download_target
59
-
60
- def _convert_to_rgb(image):
61
- return image.convert('RGB')
62
-
63
- def _transform(n_px: int, is_train: bool):
64
- normalize = Normalize((0.48145466, 0.4578275, 0.40821073), (0.26862954, 0.26130258, 0.27577711))
65
- if is_train:
66
- return Compose([
67
- RandomResizedCrop(n_px, scale=(0.9, 1.0), interpolation=InterpolationMode.BICUBIC),
68
- _convert_to_rgb,
69
- ToTensor(),
70
- normalize,
71
- ])
72
- else:
73
- return Compose([
74
- Resize(n_px, interpolation=InterpolationMode.BICUBIC),
75
- CenterCrop(n_px),
76
- _convert_to_rgb,
77
- ToTensor(),
78
- normalize,
79
- ])
80
-
81
-
82
-
83
- def available_models() -> List[str]:
84
- """Returns the names of available CLIP models"""
85
- return list(_MODELS.keys())
86
-
87
- # def load(name: str, device: Union[str, torch.device] = "cuda" if torch.cuda.is_available() else "cpu", jit: bool = False, download_root: str = None):
88
- # """Load a CLIP model
89
-
90
- # Parameters
91
- # ----------
92
- # name : str
93
- # A model name listed by `clip.available_models()`, or the path to a model checkpoint containing the state_dict
94
-
95
- # device : Union[str, torch.device]
96
- # The device to put the loaded model
97
-
98
- # jit : bool
99
- # Whether to load the optimized JIT model or more hackable non-JIT model (default).
100
-
101
- # download_root: str
102
- # path to download the model files; by default, it uses "~/.cache/clip"
103
-
104
- # Returns
105
- # -------
106
- # model : torch.nn.Module
107
- # The CLIP model
108
-
109
- # preprocess : Callable[[PIL.Image], torch.Tensor]
110
- # A torchvision transform that converts a PIL image into a tensor that the returned model can take as its input
111
- # """
112
- # if name in _MODELS:
113
- # model_path = _download(_MODELS[name], download_root or os.path.expanduser("~/.cache/clip"))
114
- # elif os.path.isfile(name):
115
- # model_path = name
116
- # else:
117
- # raise RuntimeError(f"Model {name} not found; available models = {available_models()}")
118
-
119
- # try:
120
- # # loading JIT archive
121
- # model = torch.jit.load(model_path, map_location=device if jit else "cpu").eval()
122
- # state_dict = None
123
- # except RuntimeError:
124
- # # loading saved state dict
125
- # if jit:
126
- # warnings.warn(f"File {model_path} is not a JIT archive. Loading as a state dict instead")
127
- # jit = False
128
- # state_dict = torch.load(model_path, map_location="cpu")
129
-
130
- # if not jit:
131
- # model = build_model(state_dict or model.state_dict()).to(device)
132
- # if str(device) == "cpu":
133
- # model.float()
134
- # return model, _transform(model.visual.input_resolution)
135
-
136
- # # patch the device names
137
- # device_holder = torch.jit.trace(lambda: torch.ones([]).to(torch.device(device)), example_inputs=[])
138
- # device_node = [n for n in device_holder.graph.findAllNodes("prim::Constant") if "Device" in repr(n)][-1]
139
-
140
- # def patch_device(module):
141
- # try:
142
- # graphs = [module.graph] if hasattr(module, "graph") else []
143
- # except RuntimeError:
144
- # graphs = []
145
-
146
- # if hasattr(module, "forward1"):
147
- # graphs.append(module.forward1.graph)
148
-
149
- # for graph in graphs:
150
- # for node in graph.findAllNodes("prim::Constant"):
151
- # if "value" in node.attributeNames() and str(node["value"]).startswith("cuda"):
152
- # node.copyAttributes(device_node)
153
-
154
- # model.apply(patch_device)
155
- # patch_device(model.encode_image)
156
- # patch_device(model.encode_text)
157
-
158
- # # patch dtype to float32 on CPU
159
- # if str(device) == "cpu":
160
- # float_holder = torch.jit.trace(lambda: torch.ones([]).float(), example_inputs=[])
161
- # float_input = list(float_holder.graph.findNode("aten::to").inputs())[1]
162
- # float_node = float_input.node()
163
-
164
- # def patch_float(module):
165
- # try:
166
- # graphs = [module.graph] if hasattr(module, "graph") else []
167
- # except RuntimeError:
168
- # graphs = []
169
-
170
- # if hasattr(module, "forward1"):
171
- # graphs.append(module.forward1.graph)
172
-
173
- # for graph in graphs:
174
- # for node in graph.findAllNodes("aten::to"):
175
- # inputs = list(node.inputs())
176
- # for i in [1, 2]: # dtype can be the second or third argument to aten::to()
177
- # if inputs[i].node()["value"] == 5:
178
- # inputs[i].node().copyAttributes(float_node)
179
-
180
- # model.apply(patch_float)
181
- # patch_float(model.encode_image)
182
- # patch_float(model.encode_text)
183
-
184
- # model.float()
185
-
186
- # return model, _transform(model.input_resolution.item())
187
-
188
-
189
- def load(name: str, device: Union[str, torch.device] = "cuda" if torch.cuda.is_available() else "cpu", jit=True, is_train=False, pretrained=True):
190
- """Load a CLIP model
191
- Parameters
192
- ----------
193
- name : str
194
- A model name listed by `clip.available_models()`, or the path to a model checkpoint containing the state_dict
195
- device : Union[str, torch.device]
196
- The device to put the loaded model
197
- jit : bool
198
- Whether to load the optimized JIT model (default) or more hackable non-JIT model.
199
- Returns
200
- -------
201
- model : torch.nn.Module
202
- The CLIP model
203
- preprocess : Callable[[PIL.Image], torch.Tensor]
204
- A torchvision transform that converts a PIL image into a tensor that the returned model can take as its input
205
- """
206
- if name in _MODELS:
207
- model_path = _download(_MODELS[name])
208
- elif os.path.isfile(name):
209
- model_path = name
210
- else:
211
- raise RuntimeError(f"Model {name} not found; available models = {available_models()}")
212
-
213
- try:
214
- # loading JIT archive
215
- model = torch.jit.load(model_path, map_location=device if jit else "cpu").eval()
216
- state_dict = None
217
- except RuntimeError:
218
- # loading saved state dict
219
- if jit:
220
- warnings.warn(f"File {model_path} is not a JIT archive. Loading as a state dict instead")
221
- jit = False
222
- state_dict = torch.load(model_path, map_location="cpu")
223
-
224
- if not jit:
225
- try:
226
- model = build_model(state_dict or model.state_dict()).to(device)
227
- except KeyError:
228
- sd = {k[7:]: v for k,v in state_dict["state_dict"].items()}
229
- model = build_model(sd).to(device)
230
-
231
- if str(device) == "cpu":
232
- model.float()
233
- return model, \
234
- _transform(model.visual.input_resolution, is_train=True), \
235
- _transform(model.visual.input_resolution, is_train=False)
236
-
237
- # patch the device names
238
- device_holder = torch.jit.trace(lambda: torch.ones([]).to(torch.device(device)), example_inputs=[])
239
- device_node = [n for n in device_holder.graph.findAllNodes("prim::Constant") if "Device" in repr(n)][-1]
240
-
241
- def patch_device(module):
242
- graphs = [module.graph] if hasattr(module, "graph") else []
243
- if hasattr(module, "forward1"):
244
- graphs.append(module.forward1.graph)
245
-
246
- for graph in graphs:
247
- for node in graph.findAllNodes("prim::Constant"):
248
- if "value" in node.attributeNames() and str(node["value"]).startswith("cuda"):
249
- node.copyAttributes(device_node)
250
-
251
- model.apply(patch_device)
252
- patch_device(model.encode_image)
253
- patch_device(model.encode_text)
254
-
255
- # patch dtype to float32 on CPU
256
- if str(device) == "cpu":
257
- float_holder = torch.jit.trace(lambda: torch.ones([]).float(), example_inputs=[])
258
- float_input = list(float_holder.graph.findNode("aten::to").inputs())[1]
259
- float_node = float_input.node()
260
-
261
- def patch_float(module):
262
- graphs = [module.graph] if hasattr(module, "graph") else []
263
- if hasattr(module, "forward1"):
264
- graphs.append(module.forward1.graph)
265
-
266
- for graph in graphs:
267
- for node in graph.findAllNodes("aten::to"):
268
- inputs = list(node.inputs())
269
- for i in [1, 2]: # dtype can be the second or third argument to aten::to()
270
- if inputs[i].node()["value"] == 5:
271
- inputs[i].node().copyAttributes(float_node)
272
-
273
- model.apply(patch_float)
274
- patch_float(model.encode_image)
275
- patch_float(model.encode_text)
276
-
277
- model.float()
278
-
279
- return model, \
280
- _transform(model.input_resolution.item(), is_train=True), \
281
- _transform(model.input_resolution.item(), is_train=False)
282
-
283
-
284
- def tokenize(texts: Union[str, List[str]], context_length: int = 77) -> torch.LongTensor:
285
- """
286
- Returns the tokenized representation of given input string(s)
287
- Parameters
288
- ----------
289
- texts : Union[str, List[str]]
290
- An input string or a list of input strings to tokenize
291
- context_length : int
292
- The context length to use; all CLIP models use 77 as the context length
293
- Returns
294
- -------
295
- A two-dimensional tensor containing the resulting tokens, shape = [number of input strings, context_length]
296
- """
297
- if isinstance(texts, str):
298
- texts = [texts]
299
-
300
- sot_token = _tokenizer.encoder["<start_of_text>"]
301
- eot_token = _tokenizer.encoder["<end_of_text>"]
302
- all_tokens = [[sot_token] + _tokenizer.encode(text) + [eot_token] for text in texts]
303
- result = torch.zeros(len(all_tokens), context_length, dtype=torch.long)
304
-
305
- for i, tokens in enumerate(all_tokens):
306
- if len(tokens) > context_length: # Truncate
307
- tokens = tokens[:context_length]
308
- result[i, :len(tokens)] = torch.tensor(tokens)
309
-
310
- return result
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/clip/model.py DELETED
@@ -1,713 +0,0 @@
1
- from collections import OrderedDict
2
- from typing import Tuple, Union
3
-
4
- import os
5
- import json
6
- import numpy as np
7
- import torch
8
- import torch.nn.functional as F
9
- from torch import nn
10
- from .adapter import Adapter
11
- from torch.distributions.normal import Normal
12
- from collections import Counter
13
-
14
- global_taskid = 0
15
- global_is_train=True
16
- class SparseDispatcher(object):
17
- """Helper for implementing a mixture of experts.
18
- The purpose of this class is to create input minibatches for the
19
- experts and to combine the results of the experts to form a unified
20
- output tensor.
21
- There are two functions:
22
- dispatch - take an input Tensor and create input Tensors for each expert.
23
- combine - take output Tensors from each expert and form a combined output
24
- Tensor. Outputs from different experts for the same batch element are
25
- summed together, weighted by the provided "gates".
26
- The class is initialized with a "gates" Tensor, which specifies which
27
- batch elements go to which experts, and the weights to use when combining
28
- the outputs. Batch element b is sent to expert e iff gates[b, e] != 0.
29
- The inputs and outputs are all two-dimensional [batch, depth].
30
- Caller is responsible for collapsing additional dimensions prior to
31
- calling this class and reshaping the output to the original shape.
32
- See common_layers.reshape_like().
33
- Example use:
34
- gates: a float32 `Tensor` with shape `[batch_size, num_experts]`
35
- inputs: a float32 `Tensor` with shape `[batch_size, input_size]`
36
- experts: a list of length `num_experts` containing sub-networks.
37
- dispatcher = SparseDispatcher(num_experts, gates)
38
- expert_inputs = dispatcher.dispatch(inputs)
39
- expert_outputs = [experts[i](expert_inputs[i]) for i in range(num_experts)]
40
- outputs = dispatcher.combine(expert_outputs)
41
- The preceding code sets the output for a particular example b to:
42
- output[b] = Sum_i(gates[b, i] * experts[i](inputs[b]))
43
- This class takes advantage of sparsity in the gate matrix by including in the
44
- `Tensor`s for expert i only the batch elements for which `gates[b, i] > 0`.
45
- """
46
-
47
- def __init__(self, num_experts, gates):
48
- """Create a SparseDispatcher."""
49
-
50
- self._gates = gates
51
- self._num_experts = num_experts
52
-
53
- sorted_experts, index_sorted_experts = torch.nonzero(gates).sort(0)
54
-
55
- # drop indices
56
- _, self._expert_index = sorted_experts.split(1, dim=1)
57
- # get according batch index for each expert
58
- self._batch_index = torch.nonzero(gates)[index_sorted_experts[:, 1], 0]
59
- # calculate num samples that each expert gets
60
- self._part_sizes = (gates > 0).sum(0).tolist()
61
- # expand gates to match with self._batch_index
62
- gates_exp = gates[self._batch_index.flatten()]
63
- self._nonzero_gates = torch.gather(gates_exp, 1, self._expert_index)
64
-
65
- def dispatch(self, inp):
66
- """Create one input Tensor for each expert.
67
- The `Tensor` for a expert `i` contains the slices of `inp` corresponding
68
- to the batch elements `b` where `gates[b, i] > 0`.
69
- Args:
70
- inp: a `Tensor` of shape "[batch_size, <extra_input_dims>]`
71
- Returns:
72
- a list of `num_experts` `Tensor`s with shapes
73
- `[expert_batch_size_i, <extra_input_dims>]`.
74
- """
75
-
76
- # assigns samples to experts whose gate is nonzero
77
-
78
- inp_exp = inp[self._batch_index].squeeze(1)
79
- return torch.split(inp_exp, self._part_sizes, dim=0)
80
-
81
- def combine(self, expert_out, multiply_by_gates=True):
82
- """Sum together the expert output, weighted by the gates.
83
- The slice corresponding to a particular batch element `b` is computed
84
- as the sum over all experts `i` of the expert output, weighted by the
85
- corresponding gate values. If `multiply_by_gates` is set to False, the
86
- gate values are ignored.
87
- Args:
88
- expert_out: a list of `num_experts` `Tensor`s, each with shape
89
- `[expert_batch_size_i, <extra_output_dims>]`.
90
- multiply_by_gates: a boolean
91
- Returns:
92
- a `Tensor` with shape `[batch_size, <extra_output_dims>]`.
93
- """
94
- # apply exp to expert outputs, so we are not longer in log space
95
-
96
- stitched = torch.cat(expert_out, 0)
97
- if multiply_by_gates:
98
- stitched = stitched.mul(self._nonzero_gates) # 加权
99
-
100
- zeros = torch.zeros(self._gates.size(0), expert_out[-1].size(1), device=stitched.device)
101
- # combine samples that have been processed by the same k experts
102
-
103
- combined = zeros.index_add(0, self._batch_index, stitched.float())
104
- # add eps to all zero values in order to avoid nans when going back to log space
105
- # back to log space
106
- return combined
107
-
108
- def expert_to_gates(self):
109
- """Gate values corresponding to the examples in the per-expert `Tensor`s.
110
- Returns:
111
- a list of `num_experts` one-dimensional `Tensor`s with type `tf.float32`
112
- and shapes `[expert_batch_size_i]`
113
- """
114
- # split nonzero gates for each expert
115
- return torch.split(self._nonzero_gates, self._part_sizes, dim=0)
116
-
117
- class Bottleneck(nn.Module):
118
- expansion = 4
119
-
120
- def __init__(self, inplanes, planes, stride=1):
121
- super().__init__()
122
-
123
- # all conv layers have stride 1. an avgpool is performed after the second convolution when stride > 1
124
- self.conv1 = nn.Conv2d(inplanes, planes, 1, bias=False)
125
- self.bn1 = nn.BatchNorm2d(planes)
126
-
127
- self.conv2 = nn.Conv2d(planes, planes, 3, padding=1, bias=False)
128
- self.bn2 = nn.BatchNorm2d(planes)
129
-
130
- self.avgpool = nn.AvgPool2d(stride) if stride > 1 else nn.Identity()
131
-
132
- self.conv3 = nn.Conv2d(planes, planes * self.expansion, 1, bias=False)
133
- self.bn3 = nn.BatchNorm2d(planes * self.expansion)
134
-
135
- self.relu = nn.ReLU(inplace=True)
136
- self.downsample = None
137
- self.stride = stride
138
-
139
- if stride > 1 or inplanes != planes * Bottleneck.expansion:
140
- # downsampling layer is prepended with an avgpool, and the subsequent convolution has stride 1
141
- self.downsample = nn.Sequential(OrderedDict([
142
- ("-1", nn.AvgPool2d(stride)),
143
- ("0", nn.Conv2d(inplanes, planes * self.expansion, 1, stride=1, bias=False)),
144
- ("1", nn.BatchNorm2d(planes * self.expansion))
145
- ]))
146
-
147
- def forward(self, x: torch.Tensor):
148
- identity = x
149
-
150
- out = self.relu(self.bn1(self.conv1(x)))
151
- out = self.relu(self.bn2(self.conv2(out)))
152
- out = self.avgpool(out)
153
- out = self.bn3(self.conv3(out))
154
-
155
- if self.downsample is not None:
156
- identity = self.downsample(x)
157
-
158
- out += identity
159
- out = self.relu(out)
160
- return out
161
-
162
-
163
- class AttentionPool2d(nn.Module):
164
- def __init__(self, spacial_dim: int, embed_dim: int, num_heads: int, output_dim: int = None):
165
- super().__init__()
166
- self.positional_embedding = nn.Parameter(torch.randn(spacial_dim ** 2 + 1, embed_dim) / embed_dim ** 0.5)
167
- self.k_proj = nn.Linear(embed_dim, embed_dim)
168
- self.q_proj = nn.Linear(embed_dim, embed_dim)
169
- self.v_proj = nn.Linear(embed_dim, embed_dim)
170
- self.c_proj = nn.Linear(embed_dim, output_dim or embed_dim)
171
- self.num_heads = num_heads
172
-
173
- def forward(self, x):
174
- x = x.reshape(x.shape[0], x.shape[1], x.shape[2] * x.shape[3]).permute(2, 0, 1) # NCHW -> (HW)NC
175
- x = torch.cat([x.mean(dim=0, keepdim=True), x], dim=0) # (HW+1)NC
176
- x = x + self.positional_embedding[:, None, :].to(x.dtype) # (HW+1)NC
177
- x, _ = F.multi_head_attention_forward(
178
- query=x, key=x, value=x,
179
- embed_dim_to_check=x.shape[-1],
180
- num_heads=self.num_heads,
181
- q_proj_weight=self.q_proj.weight,
182
- k_proj_weight=self.k_proj.weight,
183
- v_proj_weight=self.v_proj.weight,
184
- in_proj_weight=None,
185
- in_proj_bias=torch.cat([self.q_proj.bias, self.k_proj.bias, self.v_proj.bias]),
186
- bias_k=None,
187
- bias_v=None,
188
- add_zero_attn=False,
189
- dropout_p=0,
190
- out_proj_weight=self.c_proj.weight,
191
- out_proj_bias=self.c_proj.bias,
192
- use_separate_proj_weight=True,
193
- training=self.training,
194
- need_weights=False
195
- )
196
-
197
- return x[0]
198
-
199
-
200
- class ModifiedResNet(nn.Module):
201
- """
202
- A ResNet class that is similar to torchvision's but contains the following changes:
203
- - There are now 3 "stem" convolutions as opposed to 1, with an average pool instead of a max pool.
204
- - Performs anti-aliasing strided convolutions, where an avgpool is prepended to convolutions with stride > 1
205
- - The final pooling layer is a QKV attention instead of an average pool
206
- """
207
-
208
- def __init__(self, layers, output_dim, heads, input_resolution=224, width=64):
209
- super().__init__()
210
- self.output_dim = output_dim
211
- self.input_resolution = input_resolution
212
-
213
- # the 3-layer stem
214
- self.conv1 = nn.Conv2d(3, width // 2, kernel_size=3, stride=2, padding=1, bias=False)
215
- self.bn1 = nn.BatchNorm2d(width // 2)
216
- self.conv2 = nn.Conv2d(width // 2, width // 2, kernel_size=3, padding=1, bias=False)
217
- self.bn2 = nn.BatchNorm2d(width // 2)
218
- self.conv3 = nn.Conv2d(width // 2, width, kernel_size=3, padding=1, bias=False)
219
- self.bn3 = nn.BatchNorm2d(width)
220
- self.avgpool = nn.AvgPool2d(2)
221
- self.relu = nn.ReLU(inplace=True)
222
-
223
- # residual layers
224
- self._inplanes = width # this is a *mutable* variable used during construction
225
- self.layer1 = self._make_layer(width, layers[0])
226
- self.layer2 = self._make_layer(width * 2, layers[1], stride=2)
227
- self.layer3 = self._make_layer(width * 4, layers[2], stride=2)
228
- self.layer4 = self._make_layer(width * 8, layers[3], stride=2)
229
-
230
- embed_dim = width * 32 # the ResNet feature dimension
231
- self.attnpool = AttentionPool2d(input_resolution // 32, embed_dim, heads, output_dim)
232
-
233
- def _make_layer(self, planes, blocks, stride=1):
234
- layers = [Bottleneck(self._inplanes, planes, stride)]
235
-
236
- self._inplanes = planes * Bottleneck.expansion
237
- for _ in range(1, blocks):
238
- layers.append(Bottleneck(self._inplanes, planes))
239
-
240
- return nn.Sequential(*layers)
241
-
242
- def forward(self, x):
243
- def stem(x):
244
- for conv, bn in [(self.conv1, self.bn1), (self.conv2, self.bn2), (self.conv3, self.bn3)]:
245
- x = self.relu(bn(conv(x)))
246
- x = self.avgpool(x)
247
- return x
248
-
249
- x = x.type(self.conv1.weight.dtype)
250
- x = stem(x)
251
- x = self.layer1(x)
252
- x = self.layer2(x)
253
- x = self.layer3(x)
254
- x = self.layer4(x)
255
- x = self.attnpool(x)
256
-
257
- return x
258
-
259
-
260
- class LayerNorm(nn.LayerNorm):
261
- """Subclass torch's LayerNorm to handle fp16."""
262
-
263
- def forward(self, x: torch.Tensor):
264
- orig_type = x.dtype
265
- ret = super().forward(x.type(torch.float32))
266
- return ret.type(orig_type)
267
-
268
-
269
- class QuickGELU(nn.Module):
270
- def forward(self, x: torch.Tensor):
271
- return x * torch.sigmoid(1.702 * x)
272
-
273
-
274
- class ResidualAttentionBlock(nn.Module):
275
- def __init__(self, d_model: int, n_head: int, attn_mask: torch.Tensor = None, text_or_image=None):
276
- super().__init__()
277
- self.register_buffer("mean", torch.tensor([0.0]))
278
- self.register_buffer("std", torch.tensor([1.0]))
279
- self.attn = nn.MultiheadAttention(d_model, n_head)
280
- self.ln_1 = LayerNorm(d_model)
281
- self.mlp = nn.Sequential(OrderedDict([
282
- ("c_fc", nn.Linear(d_model, d_model * 4)),
283
- ("gelu", QuickGELU()),
284
- ("c_proj", nn.Linear(d_model * 4, d_model))
285
- ]))
286
- self.ln_2 = LayerNorm(d_model)
287
- self.attn_mask = attn_mask
288
- self.is_train = global_is_train
289
- self.step = 1
290
- self.top_k = 2
291
- self.ffn_num = 64
292
- self.experts_num = 2
293
- self.softmax = nn.Softmax(1)
294
- self.softplus = nn.Softplus()
295
- self.noisy_gating = True
296
- self.adaptmlp_list = nn.ModuleList()
297
- self.text_or_image = text_or_image
298
- if text_or_image == 'text':
299
- # print('text transformer')
300
- self.choose_map_text = torch.zeros([ self.experts_num])
301
- else:
302
- # print('image transformer')
303
- self.choose_map_image = torch.zeros([ self.experts_num])
304
- self.router_list = nn.ParameterList()
305
- self.w_noise_list = nn.ParameterList()
306
- for i in range(self.step):
307
- self.router_list.append(nn.Parameter(torch.zeros(d_model, self.experts_num), requires_grad=True))
308
- self.w_noise_list.append(nn.Parameter(torch.zeros(d_model, self.experts_num), requires_grad=True))
309
- for i in range(self.experts_num): #
310
- self.adaptmlp = Adapter(d_model=d_model, dropout=0.1, bottleneck=self.ffn_num,
311
- init_option='lora',
312
- adapter_scalar=0.1,
313
- adapter_layernorm_option='none',
314
- )
315
- self.adaptmlp_list.append(self.adaptmlp)
316
-
317
- # self.taskid = None
318
- def attention(self, x: torch.Tensor):
319
- self.attn_mask = self.attn_mask.to(dtype=x.dtype, device=x.device) if self.attn_mask is not None else None
320
- return self.attn(x, x, x, need_weights=False, attn_mask=self.attn_mask)[0]
321
-
322
- def cv_squared(self, x):
323
- """The squared coefficient of variation of a sample.
324
- Useful as a loss to encourage a positive distribution to be more uniform.
325
- Epsilons added for numerical stability.
326
- Returns 0 for an empty Tensor.
327
- Args:
328
- x: a `Tensor`.
329
- Returns:
330
- a `Scalar`.
331
- """
332
- eps = 1e-10
333
- # if only num_experts = 1
334
-
335
- if x.shape[0] == 1:
336
- return torch.tensor([0], device=x.device, dtype=x.dtype)
337
- return x.float().var() / (x.float().mean()**2 + eps)
338
-
339
- def _gates_to_load(self, gates):
340
- """Compute the true load per expert, given the gates.
341
- The load is the number of examples for which the corresponding gate is >0.
342
- Args:
343
- gates: a `Tensor` of shape [batch_size, n]
344
- Returns:
345
- a float32 `Tensor` of shape [n]
346
- """
347
- return (gates > 0).sum(0)
348
-
349
- def _prob_in_top_k(self, clean_values, noisy_values, noise_stddev, noisy_top_values):
350
- """Helper function to NoisyTopKGating.
351
- Computes the probability that value is in top k, given different random noise.
352
- This gives us a way of backpropagating from a loss that balances the number
353
- of times each expert is in the top k experts per example.
354
- In the case of no noise, pass in None for noise_stddev, and the result will
355
- not be differentiable.
356
- Args:
357
- clean_values: a `Tensor` of shape [batch, n].
358
- noisy_values: a `Tensor` of shape [batch, n]. Equal to clean values plus
359
- normally distributed noise with standard deviation noise_stddev.
360
- noise_stddev: a `Tensor` of shape [batch, n], or None
361
- noisy_top_values: a `Tensor` of shape [batch, m].
362
- "values" Output of tf.top_k(noisy_top_values, m). m >= k+1
363
- Returns:
364
- a `Tensor` of shape [batch, n].
365
- """
366
- # print('1231',clean_values) # 全nan
367
- batch = clean_values.size(0)
368
- m = noisy_top_values.size(1)
369
- top_values_flat = noisy_top_values.flatten()
370
-
371
- threshold_positions_if_in = torch.arange(batch, device=clean_values.device) * m + self.top_k
372
- threshold_if_in = torch.unsqueeze(torch.gather(top_values_flat, 0, threshold_positions_if_in), 1)
373
- is_in = torch.gt(noisy_values, threshold_if_in)
374
- threshold_positions_if_out = threshold_positions_if_in - 1
375
- threshold_if_out = torch.unsqueeze(torch.gather(top_values_flat, 0, threshold_positions_if_out), 1)
376
- # is each value currently in the top k.
377
- normal = Normal(self.mean, self.std)
378
- #
379
-
380
- prob_if_in = normal.cdf((clean_values - threshold_if_in)/noise_stddev)
381
- prob_if_out = normal.cdf((clean_values - threshold_if_out)/noise_stddev)
382
- prob = torch.where(is_in, prob_if_in, prob_if_out)
383
- return prob
384
-
385
- def noisy_top_k_gating(self, x, train, w_gate, w_noise, noise_epsilon=1e-2):
386
- """Noisy top-k gating.
387
- See paper: https://arxiv.org/abs/1701.06538.
388
- Args:
389
- x: input Tensor with shape [batch_size, input_size]
390
- train: a boolean - we only add noise at training time.
391
- noise_epsilon: a float
392
- Returns:
393
- gates: a Tensor with shape [batch_size, num_experts]
394
- load: a Tensor with shape [num_experts]
395
- """
396
-
397
- clean_logits = x @ w_gate.to(x)
398
- if self.noisy_gating and train:
399
- raw_noise_stddev = x @ w_noise.to(x)
400
- noise_stddev = ((self.softplus(raw_noise_stddev) + noise_epsilon))
401
- noisy_logits = clean_logits + (torch.randn_like(clean_logits) * noise_stddev)
402
- logits = noisy_logits
403
- else:
404
- logits = clean_logits
405
- # calculate topk + 1 that will be needed for the noisy gates
406
- top_logits, top_indices = logits.topk(min(self.top_k + 1, self.experts_num), dim=1)
407
- top_k_logits = top_logits[:, :self.top_k]
408
- top_k_indices = top_indices[:, :self.top_k]
409
- top_k_gates = self.softmax(top_k_logits)
410
- zeros = torch.zeros_like(logits)
411
- gates = zeros.scatter(1, top_k_indices, top_k_gates)
412
- if self.noisy_gating and self.top_k < self.experts_num and train: # 目前未用上
413
- load = (self._prob_in_top_k(clean_logits, noisy_logits, noise_stddev, top_logits)).sum(0)
414
- else:
415
- load = self._gates_to_load(gates)
416
- return gates, load
417
-
418
- def forward(self, x: torch.Tensor):
419
- x = x + self.attention(self.ln_1(x))
420
- if global_taskid is not None:
421
- x_re = x.permute(1, 0, 2)[:, 0, :]
422
- gates, load = self.noisy_top_k_gating(x_re, self.is_train, self.router_list[global_taskid],
423
- self.w_noise_list[global_taskid])
424
- importance = gates.sum(0)
425
-
426
- nonzero_indices = torch.nonzero(gates)
427
- counter = Counter(nonzero_indices[:, 1].tolist())
428
- for number, count in counter.items():
429
- if self.text_or_image == 'text':
430
- self.choose_map_text[number] = self.choose_map_text[number] + count
431
- else:
432
- self.choose_map_image[number] = self.choose_map_image[number] + count
433
- dispatcher = SparseDispatcher(self.experts_num, gates)
434
- expert_inputs = dispatcher.dispatch(x.permute(1, 0, 2).view(x.shape[1], -1))
435
- expert_outputs = [self.adaptmlp_list[i](expert_inputs[i].view(expert_inputs[i].shape[0],
436
- x.shape[0], x.shape[2]).to(x), add_residual=False)
437
- for i in range(self.experts_num)]
438
-
439
- i = 0
440
- while i < len(expert_outputs):
441
- if expert_outputs[i].shape[0] == 0:
442
- expert_outputs.pop(i)
443
- else:
444
- expert_outputs[i] = expert_outputs[i].view(expert_outputs[i].shape[0], -1)
445
- i += 1
446
-
447
- y = dispatcher.combine(expert_outputs)
448
- y = y.view(x.shape[1], x.shape[0], x.shape[2])
449
- x = x + self.mlp(self.ln_2(x)) + y.permute(1, 0, 2)
450
- else:
451
- x = x + self.mlp(self.ln_2(x))
452
- return x
453
-
454
-
455
- class Transformer(nn.Module):
456
- def __init__(self, width: int, layers: int, heads: int, attn_mask: torch.Tensor = None, text_or_image=None):
457
- super().__init__()
458
- self.width = width
459
- self.layers = layers
460
- self.resblocks = nn.Sequential(*[ResidualAttentionBlock(width, heads, attn_mask, text_or_image) for _ in range(layers)])
461
-
462
- def forward(self, x: torch.Tensor):
463
- return self.resblocks(x)
464
-
465
-
466
- class VisualTransformer(nn.Module):
467
- def __init__(self, input_resolution: int, patch_size: int, width: int, layers: int, heads: int, output_dim: int, text_or_image=None):
468
- super().__init__()
469
- self.input_resolution = input_resolution
470
- self.output_dim = output_dim
471
- # Added so this info is available. should not change anything.
472
- self.patch_size = patch_size
473
- self.width = width
474
- self.layers = layers
475
- self.heads = heads
476
-
477
- self.conv1 = nn.Conv2d(in_channels=3, out_channels=width, kernel_size=patch_size, stride=patch_size, bias=False)
478
-
479
- scale = width ** -0.5
480
- self.class_embedding = nn.Parameter(scale * torch.randn(width))
481
- self.positional_embedding = nn.Parameter(scale * torch.randn((input_resolution // patch_size) ** 2 + 1, width))
482
- self.ln_pre = LayerNorm(width)
483
-
484
- self.transformer = Transformer(width, layers, heads, text_or_image=text_or_image)
485
-
486
- self.ln_post = LayerNorm(width)
487
- self.proj = nn.Parameter(scale * torch.randn(width, output_dim))
488
-
489
- def forward(self, x: torch.Tensor):
490
- x = self.conv1(x)
491
- x = x.reshape(x.shape[0], x.shape[1], -1)
492
- x = x.permute(0, 2, 1)
493
- x = torch.cat([self.class_embedding.to(x.dtype) + torch.zeros(x.shape[0], 1, x.shape[-1], dtype=x.dtype, device=x.device), x], dim=1) # shape = [*, grid ** 2 + 1, width]
494
- x = x + self.positional_embedding.to(x.dtype)
495
- x = self.ln_pre(x)
496
-
497
- x = x.permute(1, 0, 2) # NLD -> LND
498
- x = self.transformer(x)
499
- x = x.permute(1, 0, 2) # LND -> NLD
500
-
501
- x = self.ln_post(x[:, 0, :])
502
-
503
- if self.proj is not None:
504
- x = x @ self.proj
505
-
506
- return x
507
-
508
-
509
- class CLIP(nn.Module):
510
- def __init__(self,
511
- embed_dim: int,
512
- # vision
513
- image_resolution: int,
514
- vision_layers: Union[Tuple[int, int, int, int], int],
515
- vision_width: int,
516
- vision_patch_size: int,
517
- # text
518
- context_length: int,
519
- vocab_size: int,
520
- transformer_width: int,
521
- transformer_heads: int,
522
- transformer_layers: int,
523
- baseline = False
524
- ):
525
- super().__init__()
526
- self.baseline = baseline
527
-
528
- self.context_length = context_length
529
-
530
- if isinstance(vision_layers, (tuple, list)):
531
- vision_heads = vision_width * 32 // 64
532
- self.visual = ModifiedResNet(
533
- layers=vision_layers,
534
- output_dim=embed_dim,
535
- heads=vision_heads,
536
- input_resolution=image_resolution,
537
- width=vision_width
538
- )
539
- else:
540
- vision_heads = vision_width // 64
541
- self.visual = VisualTransformer(
542
- input_resolution=image_resolution,
543
- patch_size=vision_patch_size,
544
- width=vision_width,
545
- layers=vision_layers,
546
- heads=vision_heads,
547
- output_dim=embed_dim,
548
- text_or_image='image'
549
- )
550
-
551
- self.transformer = Transformer(
552
- width=transformer_width,
553
- layers=transformer_layers,
554
- heads=transformer_heads,
555
- attn_mask=self.build_attention_mask(),
556
- text_or_image='text'
557
- )
558
-
559
- self.vocab_size = vocab_size
560
- self.token_embedding = nn.Embedding(vocab_size, transformer_width)
561
- self.positional_embedding = nn.Parameter(torch.empty(self.context_length, transformer_width))
562
- self.ln_final = LayerNorm(transformer_width)
563
-
564
- self.text_projection = nn.Parameter(torch.empty(transformer_width, embed_dim))
565
- self.logit_scale = nn.Parameter(torch.ones([]) * np.log(1 / 0.07))
566
-
567
- self.initialize_parameters()
568
-
569
- def initialize_parameters(self):
570
- nn.init.normal_(self.token_embedding.weight, std=0.02)
571
- nn.init.normal_(self.positional_embedding, std=0.01)
572
- self.logit_scale = nn.Parameter(torch.ones([]) * np.log(1 / 0.07))
573
-
574
- if isinstance(self.visual, ModifiedResNet):
575
- if self.visual.attnpool is not None:
576
- std = self.visual.attnpool.c_proj.in_features ** -0.5
577
- nn.init.normal_(self.visual.attnpool.q_proj.weight, std=std)
578
- nn.init.normal_(self.visual.attnpool.k_proj.weight, std=std)
579
- nn.init.normal_(self.visual.attnpool.v_proj.weight, std=std)
580
- nn.init.normal_(self.visual.attnpool.c_proj.weight, std=std)
581
-
582
- for resnet_block in [self.visual.layer1, self.visual.layer2, self.visual.layer3, self.visual.layer4]:
583
- for name, param in resnet_block.named_parameters():
584
- if name.endswith("bn3.weight"):
585
- nn.init.zeros_(param)
586
-
587
- proj_std = (self.transformer.width ** -0.5) * ((2 * self.transformer.layers) ** -0.5)
588
- attn_std = self.transformer.width ** -0.5
589
- fc_std = (2 * self.transformer.width) ** -0.5
590
- for block in self.transformer.resblocks:
591
- nn.init.normal_(block.attn.in_proj_weight, std=attn_std)
592
- nn.init.normal_(block.attn.out_proj.weight, std=proj_std)
593
- nn.init.normal_(block.mlp.c_fc.weight, std=fc_std)
594
- nn.init.normal_(block.mlp.c_proj.weight, std=proj_std)
595
-
596
- if self.text_projection is not None:
597
- nn.init.normal_(self.text_projection, std=self.transformer.width ** -0.5)
598
-
599
- def build_attention_mask(self):
600
- # lazily create causal attention mask, with full attention between the vision tokens
601
- # pytorch uses additive attention mask; fill with -inf
602
- mask = torch.empty(self.context_length, self.context_length)
603
- mask.fill_(float("-inf"))
604
- mask.triu_(1) # zero out the lower diagonal
605
- return mask
606
-
607
- @property
608
- def dtype(self):
609
- return self.visual.conv1.weight.dtype
610
-
611
- def encode_image(self, image):
612
- return self.visual(image.type(self.dtype))
613
-
614
- def encode_text(self, text):
615
-
616
- x = self.token_embedding(text).type(self.dtype) # [batch_size, n_ctx, d_model]
617
-
618
- x = x + self.positional_embedding.type(self.dtype)
619
- x = x.permute(1, 0, 2) # NLD -> LND
620
- x = self.transformer(x)
621
- x = x.permute(1, 0, 2) # LND -> NLD
622
- x = self.ln_final(x).type(self.dtype)
623
-
624
- # take features from the eot embedding (eot_token is the highest number in each sequence)
625
- x = x[torch.arange(x.shape[0]), text.argmax(dim=-1)] @ self.text_projection
626
-
627
- return x
628
-
629
- def forward(self, image, text, taskid, is_train):
630
- global global_taskid, global_is_train
631
- global_taskid = taskid
632
- global_is_train = is_train
633
- if image is None:
634
- return self.encode_text(text)
635
- elif text is None:
636
- return self.encode_image(image)
637
- image_features = self.encode_image(image)
638
- text_features = self.encode_text(text)
639
-
640
- image_features = image_features / image_features.norm(dim=-1, keepdim=True)
641
- text_features = text_features / text_features.norm(dim=-1, keepdim=True)
642
-
643
- # if self.baseline:
644
- logit_scale = self.logit_scale.exp()
645
- logits_per_image = logit_scale * image_features @ text_features.t()
646
- logits_per_text = logits_per_image.t()
647
- return logits_per_image, logits_per_text
648
-
649
-
650
-
651
- def convert_weights(model: nn.Module):
652
- """Convert applicable model parameters to fp16"""
653
-
654
- def _convert_weights_to_fp16(l):
655
- if isinstance(l, (nn.Conv1d, nn.Conv2d, nn.Linear)):
656
- l.weight.data = l.weight.data.half()
657
- if l.bias is not None:
658
- l.bias.data = l.bias.data.half()
659
-
660
- if isinstance(l, nn.MultiheadAttention):
661
- for attr in [*[f"{s}_proj_weight" for s in ["in", "q", "k", "v"]], "in_proj_bias", "bias_k", "bias_v"]:
662
- tensor = getattr(l, attr)
663
- if tensor is not None:
664
- tensor.data = tensor.data.half()
665
-
666
- for name in ["text_projection", "proj"]:
667
- if hasattr(l, name):
668
- attr = getattr(l, name)
669
- if attr is not None:
670
- attr.data = attr.data.half()
671
-
672
- model.apply(_convert_weights_to_fp16)
673
-
674
-
675
- def build_model(state_dict: dict):
676
- vit = "visual.proj" in state_dict
677
-
678
- if vit:
679
- vision_width = state_dict["visual.conv1.weight"].shape[0]
680
- vision_layers = len([k for k in state_dict.keys() if k.startswith("visual.") and k.endswith(".attn.in_proj_weight")])
681
- vision_patch_size = state_dict["visual.conv1.weight"].shape[-1]
682
- grid_size = round((state_dict["visual.positional_embedding"].shape[0] - 1) ** 0.5)
683
- image_resolution = vision_patch_size * grid_size
684
- else:
685
- counts: list = [len(set(k.split(".")[2] for k in state_dict if k.startswith(f"visual.layer{b}"))) for b in [1, 2, 3, 4]]
686
- vision_layers = tuple(counts)
687
- vision_width = state_dict["visual.layer1.0.conv1.weight"].shape[0]
688
- output_width = round((state_dict["visual.attnpool.positional_embedding"].shape[0] - 1) ** 0.5)
689
- vision_patch_size = None
690
- assert output_width ** 2 + 1 == state_dict["visual.attnpool.positional_embedding"].shape[0]
691
- image_resolution = output_width * 32
692
-
693
- embed_dim = state_dict["text_projection"].shape[1]
694
- context_length = state_dict["positional_embedding"].shape[0]
695
- vocab_size = state_dict["token_embedding.weight"].shape[0]
696
- transformer_width = state_dict["ln_final.weight"].shape[0]
697
- transformer_heads = transformer_width // 64
698
- transformer_layers = len(set(k.split(".")[2] for k in state_dict if k.startswith(f"transformer.resblocks")))
699
-
700
- model = CLIP(
701
- embed_dim,
702
- image_resolution, vision_layers, vision_width, vision_patch_size,
703
- context_length, vocab_size, transformer_width, transformer_heads, transformer_layers
704
- )
705
-
706
- for key in ["input_resolution", "context_length", "vocab_size"]:
707
- if key in state_dict:
708
- del state_dict[key]
709
-
710
- model.load_state_dict(state_dict, strict=False)
711
- for p in model.parameters():
712
- p.data = p.data.float()
713
- return model.eval()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/clip/tokenizer.py DELETED
@@ -1,140 +0,0 @@
1
- import gzip
2
- import html
3
- import os
4
- from functools import lru_cache
5
-
6
- import ftfy
7
- import regex as re
8
-
9
-
10
- @lru_cache()
11
- def default_bpe():
12
- return os.path.join(os.path.dirname(os.path.abspath(__file__)), "bpe_simple_vocab_16e6.txt.gz")
13
-
14
-
15
- @lru_cache()
16
- def bytes_to_unicode():
17
- """
18
- Returns list of utf-8 byte and a corresponding list of unicode strings.
19
- The reversible bpe codes work on unicode strings.
20
- This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
21
- When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
22
- This is a signficant percentage of your normal, say, 32K bpe vocab.
23
- To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
24
- And avoids mapping to whitespace/control characters the bpe code barfs on.
25
- """
26
- bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
27
- cs = bs[:]
28
- n = 0
29
- for b in range(2**8):
30
- if b not in bs:
31
- bs.append(b)
32
- cs.append(2**8+n)
33
- n += 1
34
- cs = [chr(n) for n in cs]
35
- return dict(zip(bs, cs))
36
-
37
-
38
- def get_pairs(word):
39
- """Return set of symbol pairs in a word.
40
- Word is represented as tuple of symbols (symbols being variable-length strings).
41
- """
42
- pairs = set()
43
- prev_char = word[0]
44
- for char in word[1:]:
45
- pairs.add((prev_char, char))
46
- prev_char = char
47
- return pairs
48
-
49
-
50
- def basic_clean(text):
51
- text = ftfy.fix_text(text)
52
- text = html.unescape(html.unescape(text))
53
- return text.strip()
54
-
55
-
56
- def whitespace_clean(text):
57
- text = re.sub(r'\s+', ' ', text)
58
- text = text.strip()
59
- return text
60
-
61
-
62
- class SimpleTokenizer(object):
63
- def __init__(self, bpe_path: str = default_bpe(), special_tokens=None):
64
- self.byte_encoder = bytes_to_unicode()
65
- self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
66
- merges = gzip.open(bpe_path).read().decode("utf-8").split('\n')
67
- merges = merges[1:49152-256-2+1]
68
- merges = [tuple(merge.split()) for merge in merges]
69
- vocab = list(bytes_to_unicode().values())
70
- vocab = vocab + [v+'</w>' for v in vocab]
71
- for merge in merges:
72
- vocab.append(''.join(merge))
73
- if not special_tokens:
74
- special_tokens = ['<start_of_text>', '<end_of_text>']
75
- else:
76
- special_tokens = ['<start_of_text>', '<end_of_text>'] + special_tokens
77
- vocab.extend(special_tokens)
78
- self.encoder = dict(zip(vocab, range(len(vocab))))
79
- self.decoder = {v: k for k, v in self.encoder.items()}
80
- self.bpe_ranks = dict(zip(merges, range(len(merges))))
81
- self.cache = {t:t for t in special_tokens}
82
- special = "|".join(special_tokens)
83
- self.pat = re.compile(special + r"""|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""", re.IGNORECASE)
84
-
85
- self.vocab_size = len(self.encoder)
86
- self.all_special_ids = [self.encoder[t] for t in special_tokens]
87
-
88
- def bpe(self, token):
89
- if token in self.cache:
90
- return self.cache[token]
91
- word = tuple(token[:-1]) + ( token[-1] + '</w>',)
92
- pairs = get_pairs(word)
93
-
94
- if not pairs:
95
- return token+'</w>'
96
-
97
- while True:
98
- bigram = min(pairs, key = lambda pair: self.bpe_ranks.get(pair, float('inf')))
99
- if bigram not in self.bpe_ranks:
100
- break
101
- first, second = bigram
102
- new_word = []
103
- i = 0
104
- while i < len(word):
105
- try:
106
- j = word.index(first, i)
107
- new_word.extend(word[i:j])
108
- i = j
109
- except:
110
- new_word.extend(word[i:])
111
- break
112
-
113
- if word[i] == first and i < len(word)-1 and word[i+1] == second:
114
- new_word.append(first+second)
115
- i += 2
116
- else:
117
- new_word.append(word[i])
118
- i += 1
119
- new_word = tuple(new_word)
120
- word = new_word
121
- if len(word) == 1:
122
- break
123
- else:
124
- pairs = get_pairs(word)
125
- word = ' '.join(word)
126
- self.cache[token] = word
127
- return word
128
-
129
- def encode(self, text):
130
- bpe_tokens = []
131
- text = whitespace_clean(basic_clean(text)).lower()
132
- for token in re.findall(self.pat, text):
133
- token = ''.join(self.byte_encoder[b] for b in token.encode('utf-8'))
134
- bpe_tokens.extend(self.encoder[bpe_token] for bpe_token in self.bpe(token).split(' '))
135
- return bpe_tokens
136
-
137
- def decode(self, tokens):
138
- text = ''.join([self.decoder[token] for token in tokens])
139
- text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors="replace").replace('</w>', ' ')
140
- return text
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/configs/class/aircraft.yaml DELETED
@@ -1,76 +0,0 @@
1
- hydra:
2
- run:
3
- dir: ./experiments/Aircraft
4
- job:
5
- chdir: true
6
-
7
- job_logging:
8
- version: 1
9
- formatters:
10
- simple:
11
- format: '%(message)s'
12
-
13
- class_order: ""
14
- dataset_root: ""
15
- workdir: ""
16
- log_path: "metrics.json"
17
- model_name: "ViT-B/16"
18
- prompt_template: "a bad photo of a {}."
19
- zero_shot_eval: true
20
- pre_task_zero_shot_eval: False
21
- # zs_mtil_indices: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24] # indices of MTIL datasets (0..24) to evaluate in zero-shot; excludes any selected as downstream train_dataset
22
- zs_mtil_indices: [1] # indices of MTIL datasets (0..24) to evaluate in zero-shot; excludes any selected as downstream train_dataset
23
-
24
-
25
- batch_size: 128
26
- increment: ${initial_increment}
27
- initial_increment: 10
28
- scenario: "class"
29
- dataset: "cifar100"
30
-
31
- # method: "lwf"
32
- # lr: 7.1e-6
33
- # ls: 0.2
34
- # we:
35
- # avg_freq:
36
- # ref_dataset:
37
- # ref_sentences:
38
-
39
- weight_decay: 0.0
40
- l2: 0
41
- ce_method: 0
42
-
43
- method: "MoE-Adapters"
44
- lr: 1e-3
45
- ls: 0.0
46
- #we:
47
- #avg_freq:
48
- #ref_dataset:
49
- #ref_sentences: random
50
-
51
- # =====================
52
- # DFA (two-expert + router) hyperparameters
53
- # =====================
54
-
55
-
56
- # Training schedule
57
- epochs_a: 1
58
- epochs_b: 1
59
- e2_top_k: 2 # top-k for E2 router
60
-
61
- # Stage A (InfoNCE)
62
- lr_e1: 1.0e-5
63
- tau_con: 0.15 # contrastive temperature for InfoNCE
64
- use_moco_queue: true # enable momentum queue for more negatives
65
- moco_queue_size: 128 # queue size (default 4096, increases negative samples significantly)
66
-
67
- # Stage B (Task expert + Router)
68
- lr_e2: 5.0e-4
69
-
70
- text_lr_e2: 1.0e-5
71
- lr_e2_router: 1.0e-3
72
- lr_top_router: 1.0e-5
73
- lambda_b_con: 0.001
74
- tau_b_con: 0.19
75
- num_task_experts: 2
76
- seed: 43
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/configs/class/caltech.yaml DELETED
@@ -1,79 +0,0 @@
1
- hydra:
2
- run:
3
- dir: ./experiments/Caltech101
4
- job:
5
- chdir: true
6
-
7
- job_logging:
8
- version: 1
9
- formatters:
10
- simple:
11
- format: '%(message)s'
12
-
13
- class_order: ""
14
- dataset_root: ""
15
- workdir: ""
16
- log_path: "metrics.json"
17
- model_name: "ViT-B/16"
18
- prompt_template: "a bad photo of a {}."
19
- zero_shot_eval: true
20
- pre_task_zero_shot_eval: False
21
- zs_mtil_indices: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24] # indices of MTIL datasets (0..24) to evaluate in zero-shot; excludes any selected as downstream train_dataset
22
-
23
-
24
-
25
- batch_size: 128
26
- increment: ${initial_increment}
27
- initial_increment: 10
28
- scenario: "class"
29
- dataset: "cifar100"
30
-
31
- # method: "lwf"
32
- # lr: 7.1e-6
33
- # ls: 0.2
34
- # we:
35
- # avg_freq:
36
- # ref_dataset:
37
- # ref_sentences:
38
-
39
- weight_decay: 0.0
40
- l2: 0
41
- ce_method: 0
42
-
43
- method: "MoE-Adapters"
44
- lr: 1e-3
45
- ls: 0.0
46
- #we:
47
- #avg_freq:
48
- #ref_dataset:
49
- #ref_sentences: random
50
-
51
- # =====================
52
- # DFA (two-expert + router) hyperparameters
53
- # =====================
54
-
55
-
56
- # Training schedule
57
- epochs_a: 1
58
- epochs_b: 1
59
- e2_top_k: 2 # top-k for E2 router
60
-
61
- # Stage A (InfoNCE)
62
- lr_e1: 1.0e-5
63
- tau_con: 0.15 # contrastive temperature for InfoNCE
64
- use_moco_queue: true # enable momentum queue for more negatives
65
- moco_queue_size: 128 # queue size (default 4096, increases negative samples significantly)
66
-
67
-
68
-
69
-
70
- # Stage B (Task expert + Router)
71
- lr_e2: 5.0e-4
72
-
73
- text_lr_e2: 1.0e-5
74
- lr_e2_router: 1.0e-3
75
- lr_top_router: 1.0e-5
76
- lambda_b_con: 0.001
77
- tau_b_con: 0.19
78
- num_task_experts: 2
79
- seed: 43
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/configs/class/car.yaml DELETED
@@ -1,79 +0,0 @@
1
- hydra:
2
- run:
3
- dir: ./experiments/StanfordCars
4
- job:
5
- chdir: true
6
-
7
- job_logging:
8
- version: 1
9
- formatters:
10
- simple:
11
- format: '%(message)s'
12
-
13
- class_order: ""
14
- dataset_root: ""
15
- workdir: ""
16
- log_path: "metrics.json"
17
- model_name: "ViT-B/16"
18
- prompt_template: "a bad photo of a {}."
19
- zero_shot_eval: true
20
- pre_task_zero_shot_eval: False
21
- zs_mtil_indices: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24] # indices of MTIL datasets (0..24) to evaluate in zero-shot; excludes any selected as downstream train_dataset
22
-
23
-
24
-
25
- batch_size: 128
26
- increment: ${initial_increment}
27
- initial_increment: 10
28
- scenario: "class"
29
- dataset: "cifar100"
30
-
31
- # method: "lwf"
32
- # lr: 7.1e-6
33
- # ls: 0.2
34
- # we:
35
- # avg_freq:
36
- # ref_dataset:
37
- # ref_sentences:
38
-
39
- weight_decay: 0.0
40
- l2: 0
41
- ce_method: 0
42
-
43
- method: "MoE-Adapters"
44
- lr: 1e-3
45
- ls: 0.0
46
- #we:
47
- #avg_freq:
48
- #ref_dataset:
49
- #ref_sentences: random
50
-
51
- # =====================
52
- # DFA (two-expert + router) hyperparameters
53
- # =====================
54
-
55
-
56
- # Training schedule
57
- epochs_a: 1
58
- epochs_b: 1
59
- e2_top_k: 2 # top-k for E2 router
60
-
61
- # Stage A (InfoNCE)
62
- lr_e1: 1.0e-5
63
- tau_con: 0.15 # contrastive temperature for InfoNCE
64
- use_moco_queue: true # enable momentum queue for more negatives
65
- moco_queue_size: 128 # queue size (default 4096, increases negative samples significantly)
66
-
67
-
68
-
69
-
70
- # Stage B (Task expert + Router)
71
- lr_e2: 5.0e-4
72
-
73
- text_lr_e2: 1.0e-5
74
- lr_e2_router: 1.0e-3
75
- lr_top_router: 1.0e-5
76
- lambda_b_con: 0.001
77
- tau_b_con: 0.17
78
- num_task_experts: 2
79
- seed: 43
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/configs/class/cifar10.yaml DELETED
@@ -1,79 +0,0 @@
1
- hydra:
2
- run:
3
- dir: ./experiments/Cifar10
4
- job:
5
- chdir: true
6
-
7
- job_logging:
8
- version: 1
9
- formatters:
10
- simple:
11
- format: '%(message)s'
12
-
13
- class_order: ""
14
- dataset_root: ""
15
- workdir: ""
16
- log_path: "metrics.json"
17
- model_name: "ViT-B/16"
18
- prompt_template: "a bad photo of a {}."
19
- zero_shot_eval: true
20
- pre_task_zero_shot_eval: False
21
- zs_mtil_indices: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24] # indices of MTIL datasets (0..24) to evaluate in zero-shot; excludes any selected as downstream train_dataset
22
-
23
-
24
-
25
- batch_size: 128
26
- increment: ${initial_increment}
27
- initial_increment: 10
28
- scenario: "class"
29
- dataset: "cifar100"
30
-
31
- # method: "lwf"
32
- # lr: 7.1e-6
33
- # ls: 0.2
34
- # we:
35
- # avg_freq:
36
- # ref_dataset:
37
- # ref_sentences:
38
-
39
- weight_decay: 0.0
40
- l2: 0
41
- ce_method: 0
42
-
43
- method: "MoE-Adapters"
44
- lr: 1e-3
45
- ls: 0.0
46
- #we:
47
- #avg_freq:
48
- #ref_dataset:
49
- #ref_sentences: random
50
-
51
- # =====================
52
- # DFA (two-expert + router) hyperparameters
53
- # =====================
54
-
55
-
56
- # Training schedule
57
- epochs_a: 1
58
- epochs_b: 1
59
- e2_top_k: 2 # top-k for E2 router
60
-
61
- # Stage A (InfoNCE)
62
- lr_e1: 1.0e-5
63
- tau_con: 0.15 # contrastive temperature for InfoNCE
64
- use_moco_queue: true # enable momentum queue for more negatives
65
- moco_queue_size: 128 # queue size (default 4096, increases negative samples significantly)
66
-
67
-
68
-
69
-
70
- # Stage B (Task expert + Router)
71
- lr_e2: 5.0e-4
72
-
73
- text_lr_e2: 1.0e-5
74
- lr_e2_router: 1.0e-3
75
- lr_top_router: 1.0e-5
76
- lambda_b_con: 0.001
77
- tau_b_con: 0.19
78
- num_task_experts: 2
79
- seed: 43
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/configs/class/cifar100.yaml DELETED
@@ -1,79 +0,0 @@
1
- hydra:
2
- run:
3
- dir: ./experiments/CIFAR100
4
- job:
5
- chdir: true
6
-
7
- job_logging:
8
- version: 1
9
- formatters:
10
- simple:
11
- format: '%(message)s'
12
-
13
- class_order: ""
14
- dataset_root: ""
15
- workdir: ""
16
- log_path: "metrics.json"
17
- model_name: "ViT-B/16"
18
- prompt_template: "a bad photo of a {}."
19
- zero_shot_eval: true
20
- pre_task_zero_shot_eval: False
21
- zs_mtil_indices: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24] # indices of MTIL datasets (0..24) to evaluate in zero-shot; excludes any selected as downstream train_dataset
22
-
23
-
24
-
25
- batch_size: 64
26
- increment: ${initial_increment}
27
- initial_increment: 10
28
- scenario: "class"
29
- dataset: "cifar100"
30
-
31
- # method: "lwf"
32
- # lr: 7.1e-6
33
- # ls: 0.2
34
- # we:
35
- # avg_freq:
36
- # ref_dataset:
37
- # ref_sentences:
38
-
39
- weight_decay: 0.0
40
- l2: 0
41
- ce_method: 0
42
-
43
- method: "MoE-Adapters"
44
- lr: 1e-3
45
- ls: 0.0
46
- #we:
47
- #avg_freq:
48
- #ref_dataset:
49
- #ref_sentences: random
50
-
51
- # =====================
52
- # DFA (two-expert + router) hyperparameters
53
- # =====================
54
-
55
-
56
- # Training schedule
57
- epochs_a: 1
58
- epochs_b: 1
59
- e2_top_k: 2 # top-k for E2 router
60
-
61
- # Stage A (InfoNCE)
62
- lr_e1: 1.0e-5
63
- tau_con: 0.15 # contrastive temperature for InfoNCE
64
- use_moco_queue: true # enable momentum queue for more negatives
65
- moco_queue_size: 128 # queue size (default 4096, increases negative samples significantly)
66
-
67
-
68
-
69
-
70
- # Stage B (Task expert + Router)
71
- lr_e2: 5.0e-4
72
-
73
- text_lr_e2: 1.0e-5
74
- lr_e2_router: 1.0e-3
75
- lr_top_router: 1.0e-5
76
- lambda_b_con: 0.001
77
- tau_b_con: 0.17
78
- num_task_experts: 2
79
- seed: 43
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/configs/class/clevr.yaml DELETED
@@ -1,79 +0,0 @@
1
- hydra:
2
- run:
3
- dir: ./experiments/CLEVRCount
4
- job:
5
- chdir: true
6
-
7
- job_logging:
8
- version: 1
9
- formatters:
10
- simple:
11
- format: '%(message)s'
12
-
13
- class_order: ""
14
- dataset_root: ""
15
- workdir: ""
16
- log_path: "metrics.json"
17
- model_name: "ViT-B/16"
18
- prompt_template: "a bad photo of a {}."
19
- zero_shot_eval: true
20
- pre_task_zero_shot_eval: False
21
- zs_mtil_indices: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24] # indices of MTIL datasets (0..24) to evaluate in zero-shot; excludes any selected as downstream train_dataset
22
-
23
-
24
-
25
- batch_size: 128
26
- increment: ${initial_increment}
27
- initial_increment: 10
28
- scenario: "class"
29
- dataset: "cifar100"
30
-
31
- # method: "lwf"
32
- # lr: 7.1e-6
33
- # ls: 0.2
34
- # we:
35
- # avg_freq:
36
- # ref_dataset:
37
- # ref_sentences:
38
-
39
- weight_decay: 0.0
40
- l2: 0
41
- ce_method: 0
42
-
43
- method: "MoE-Adapters"
44
- lr: 1e-3
45
- ls: 0.0
46
- #we:
47
- #avg_freq:
48
- #ref_dataset:
49
- #ref_sentences: random
50
-
51
- # =====================
52
- # DFA (two-expert + router) hyperparameters
53
- # =====================
54
-
55
-
56
- # Training schedule
57
- epochs_a: 1
58
- epochs_b: 1
59
- e2_top_k: 2 # top-k for E2 router
60
-
61
- # Stage A (InfoNCE)
62
- lr_e1: 1.0e-5
63
- tau_con: 0.15 # contrastive temperature for InfoNCE
64
- use_moco_queue: true # enable momentum queue for more negatives
65
- moco_queue_size: 128 # queue size (default 4096, increases negative samples significantly)
66
-
67
-
68
-
69
-
70
- # Stage B (Task expert + Router)
71
- lr_e2: 5.0e-4
72
-
73
- text_lr_e2: 1.0e-5
74
- lr_e2_router: 1.0e-3
75
- lr_top_router: 1.0e-5
76
- lambda_b_con: 0.001
77
- tau_b_con: 0.19
78
- num_task_experts: 2
79
- seed: 43
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/configs/class/conuntry.yaml DELETED
@@ -1,79 +0,0 @@
1
- hydra:
2
- run:
3
- dir: ./experiments/Country211
4
- job:
5
- chdir: true
6
-
7
- job_logging:
8
- version: 1
9
- formatters:
10
- simple:
11
- format: '%(message)s'
12
-
13
- class_order: ""
14
- dataset_root: ""
15
- workdir: ""
16
- log_path: "metrics.json"
17
- model_name: "ViT-B/16"
18
- prompt_template: "a bad photo of a {}."
19
- zero_shot_eval: true
20
- pre_task_zero_shot_eval: False
21
- zs_mtil_indices: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24] # indices of MTIL datasets (0..24) to evaluate in zero-shot; excludes any selected as downstream train_dataset
22
-
23
-
24
-
25
- batch_size: 128
26
- increment: ${initial_increment}
27
- initial_increment: 10
28
- scenario: "class"
29
- dataset: "cifar100"
30
-
31
- # method: "lwf"
32
- # lr: 7.1e-6
33
- # ls: 0.2
34
- # we:
35
- # avg_freq:
36
- # ref_dataset:
37
- # ref_sentences:
38
-
39
- weight_decay: 0.0
40
- l2: 0
41
- ce_method: 0
42
-
43
- method: "MoE-Adapters"
44
- lr: 1e-3
45
- ls: 0.0
46
- #we:
47
- #avg_freq:
48
- #ref_dataset:
49
- #ref_sentences: random
50
-
51
- # =====================
52
- # DFA (two-expert + router) hyperparameters
53
- # =====================
54
-
55
-
56
- # Training schedule
57
- epochs_a: 1
58
- epochs_b: 1
59
- e2_top_k: 2 # top-k for E2 router
60
-
61
- # Stage A (InfoNCE)
62
- lr_e1: 1.0e-5
63
- tau_con: 0.15 # contrastive temperature for InfoNCE
64
- use_moco_queue: true # enable momentum queue for more negatives
65
- moco_queue_size: 128 # queue size (default 4096, increases negative samples significantly)
66
-
67
-
68
-
69
-
70
- # Stage B (Task expert + Router)
71
- lr_e2: 5.0e-4
72
-
73
- text_lr_e2: 1.0e-5
74
- lr_e2_router: 1.0e-3
75
- lr_top_router: 1.0e-5
76
- lambda_b_con: 0.001
77
- tau_b_con: 0.19
78
- num_task_experts: 2
79
- seed: 43
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/configs/class/dtd.yaml DELETED
@@ -1,79 +0,0 @@
1
- hydra:
2
- run:
3
- dir: ./experiments/DTD
4
- job:
5
- chdir: true
6
-
7
- job_logging:
8
- version: 1
9
- formatters:
10
- simple:
11
- format: '%(message)s'
12
-
13
- class_order: ""
14
- dataset_root: ""
15
- workdir: ""
16
- log_path: "metrics.json"
17
- model_name: "ViT-B/16"
18
- prompt_template: "a bad photo of a {}."
19
- zero_shot_eval: true
20
- pre_task_zero_shot_eval: False
21
- zs_mtil_indices: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24] # indices of MTIL datasets (0..24) to evaluate in zero-shot; excludes any selected as downstream train_dataset
22
-
23
-
24
-
25
- batch_size: 128
26
- increment: ${initial_increment}
27
- initial_increment: 10
28
- scenario: "class"
29
- dataset: "cifar100"
30
-
31
- # method: "lwf"
32
- # lr: 7.1e-6
33
- # ls: 0.2
34
- # we:
35
- # avg_freq:
36
- # ref_dataset:
37
- # ref_sentences:
38
-
39
- weight_decay: 0.0
40
- l2: 0
41
- ce_method: 0
42
-
43
- method: "MoE-Adapters"
44
- lr: 1e-3
45
- ls: 0.0
46
- #we:
47
- #avg_freq:
48
- #ref_dataset:
49
- #ref_sentences: random
50
-
51
- # =====================
52
- # DFA (two-expert + router) hyperparameters
53
- # =====================
54
-
55
-
56
- # Training schedule
57
- epochs_a: 1
58
- epochs_b: 1
59
- e2_top_k: 2 # top-k for E2 router
60
-
61
- # Stage A (InfoNCE)
62
- lr_e1: 1.0e-5
63
- tau_con: 0.15 # contrastive temperature for InfoNCE
64
- use_moco_queue: true # enable momentum queue for more negatives
65
- moco_queue_size: 128 # queue size (default 4096, increases negative samples significantly)
66
-
67
-
68
-
69
-
70
- # Stage B (Task expert + Router)
71
- lr_e2: 5.0e-4
72
-
73
- text_lr_e2: 1.0e-5
74
- lr_e2_router: 1.0e-3
75
- lr_top_router: 1.0e-5
76
- lambda_b_con: 0.001
77
- tau_b_con: 0.19
78
- num_task_experts: 2
79
- seed: 43
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
DFA-MoE/configs/class/eurosat.yaml DELETED
@@ -1,79 +0,0 @@
1
- hydra:
2
- run:
3
- dir: ./experiments/Eurosat
4
- job:
5
- chdir: true
6
-
7
- job_logging:
8
- version: 1
9
- formatters:
10
- simple:
11
- format: '%(message)s'
12
-
13
- class_order: ""
14
- dataset_root: ""
15
- workdir: ""
16
- log_path: "metrics.json"
17
- model_name: "ViT-B/16"
18
- prompt_template: "a bad photo of a {}."
19
- zero_shot_eval: true
20
- pre_task_zero_shot_eval: False
21
- zs_mtil_indices: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24] # indices of MTIL datasets (0..24) to evaluate in zero-shot; excludes any selected as downstream train_dataset
22
-
23
-
24
- batch_size: 128
25
- increment: ${initial_increment}
26
- initial_increment: 10
27
- scenario: "class"
28
- dataset: "cifar100"
29
-
30
- # method: "lwf"
31
- # lr: 7.1e-6
32
- # ls: 0.2
33
- # we:
34
- # avg_freq:
35
- # ref_dataset:
36
- # ref_sentences:
37
-
38
- weight_decay: 0.0
39
- l2: 0
40
- ce_method: 0
41
-
42
- method: "MoE-Adapters"
43
- lr: 1e-3
44
- ls: 0.0
45
- #we:
46
- #avg_freq:
47
- #ref_dataset:
48
- #ref_sentences: random
49
-
50
- # =====================
51
- # DFA (two-expert + router) hyperparameters
52
- # =====================
53
-
54
-
55
- # Training schedule
56
- epochs_a: 1
57
- epochs_b: 1
58
- e2_top_k: 2 # top-k for E2 router
59
-
60
- # Stage A (InfoNCE)
61
- lr_e1: 1.0e-5
62
- tau_con: 0.15 # contrastive temperature for InfoNCE
63
- use_moco_queue: true # enable momentum queue for more negatives
64
- moco_queue_size: 128 # queue size (default 4096, increases negative samples significantly)
65
-
66
-
67
-
68
-
69
-
70
- # Stage B (Task expert + Router)
71
- lr_e2: 5.0e-4
72
-
73
- text_lr_e2: 1.0e-5
74
- lr_e2_router: 1.0e-3
75
- lr_top_router: 1.0e-5
76
- lambda_b_con: 0.001
77
- tau_b_con: 0.17
78
- num_task_experts: 2
79
- seed: 43