Delete DFA-MoE
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- DFA-MoE/MTIL_datasets/__init__.py +0 -4
- DFA-MoE/MTIL_datasets/caltech101.py +0 -92
- DFA-MoE/MTIL_datasets/cifar10.py +0 -94
- DFA-MoE/MTIL_datasets/cifar100.py +0 -89
- DFA-MoE/MTIL_datasets/clevr_count.py +0 -118
- DFA-MoE/MTIL_datasets/country211.py +0 -386
- DFA-MoE/MTIL_datasets/dtd.py +0 -95
- DFA-MoE/MTIL_datasets/eurosat.py +0 -75
- DFA-MoE/MTIL_datasets/fer2013.py +0 -170
- DFA-MoE/MTIL_datasets/fgvc_aircraft.py +0 -73
- DFA-MoE/MTIL_datasets/food101.py +0 -52
- DFA-MoE/MTIL_datasets/gtsrb.py +0 -202
- DFA-MoE/MTIL_datasets/hatefulmemes.py +0 -117
- DFA-MoE/MTIL_datasets/imagenet_r.py +0 -231
- DFA-MoE/MTIL_datasets/kitti_distance.py +0 -228
- DFA-MoE/MTIL_datasets/mnist.py +0 -132
- DFA-MoE/MTIL_datasets/oxford_flowers.py +0 -89
- DFA-MoE/MTIL_datasets/oxford_pets.py +0 -176
- DFA-MoE/MTIL_datasets/pcam.py +0 -198
- DFA-MoE/MTIL_datasets/resisc.py +0 -199
- DFA-MoE/MTIL_datasets/sst2.py +0 -122
- DFA-MoE/MTIL_datasets/stanford_cars.py +0 -83
- DFA-MoE/MTIL_datasets/stl10.py +0 -143
- DFA-MoE/MTIL_datasets/sun397.py +0 -81
- DFA-MoE/MTIL_datasets/ucf101.py +0 -360
- DFA-MoE/MTIL_datasets/utils.py +0 -299
- DFA-MoE/MTIL_datasets/voc2007.py +0 -173
- DFA-MoE/README.md +0 -227
- DFA-MoE/calculate_SCR.py +0 -266
- DFA-MoE/calculate_sim.py +0 -456
- DFA-MoE/class_orders/cifar100.yaml +0 -1
- DFA-MoE/class_orders/imagenet100.yaml +0 -11
- DFA-MoE/class_orders/imagenet1000.yaml +0 -48
- DFA-MoE/class_orders/tinyimagenet.yaml +0 -17
- DFA-MoE/clip/README.md +0 -1
- DFA-MoE/clip/__init__.py +0 -1
- DFA-MoE/clip/adapter.py +0 -69
- DFA-MoE/clip/bpe_simple_vocab_16e6.txt.gz +0 -3
- DFA-MoE/clip/clip.py +0 -310
- DFA-MoE/clip/model.py +0 -713
- DFA-MoE/clip/tokenizer.py +0 -140
- DFA-MoE/configs/class/aircraft.yaml +0 -76
- DFA-MoE/configs/class/caltech.yaml +0 -79
- DFA-MoE/configs/class/car.yaml +0 -79
- DFA-MoE/configs/class/cifar10.yaml +0 -79
- DFA-MoE/configs/class/cifar100.yaml +0 -79
- DFA-MoE/configs/class/clevr.yaml +0 -79
- DFA-MoE/configs/class/conuntry.yaml +0 -79
- DFA-MoE/configs/class/dtd.yaml +0 -79
- DFA-MoE/configs/class/eurosat.yaml +0 -79
DFA-MoE/MTIL_datasets/__init__.py
DELETED
|
@@ -1,4 +0,0 @@
|
|
| 1 |
-
# This dataset is originally proposed by Zangwei Zheng et al. (https://arxiv.org/abs/2303.06628)
|
| 2 |
-
# Code here is based on CoOp and its following works. (https://github.com/KaiyangZhou/CoOp)
|
| 3 |
-
# Modified by Longxiang Tang (lloong.x@gmail.com) to release the dependence of Dassl lib.
|
| 4 |
-
# To prepare data, please refer to https://github.com/muzairkhattak/PromptSRC/blob/main/docs/DATASETS.md
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/MTIL_datasets/caltech101.py
DELETED
|
@@ -1,92 +0,0 @@
|
|
| 1 |
-
import os
|
| 2 |
-
import pickle
|
| 3 |
-
|
| 4 |
-
from .utils import *
|
| 5 |
-
|
| 6 |
-
from .oxford_pets import OxfordPets
|
| 7 |
-
from .dtd import DescribableTextures as DTD
|
| 8 |
-
|
| 9 |
-
IGNORED = ["BACKGROUND_Google", "Faces_easy"]
|
| 10 |
-
NEW_CNAMES = {
|
| 11 |
-
"airplanes": "airplane",
|
| 12 |
-
"Faces": "face",
|
| 13 |
-
"Leopards": "leopard",
|
| 14 |
-
"Motorbikes": "motorbike",
|
| 15 |
-
}
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
class Caltech101(DatasetBase):
|
| 19 |
-
|
| 20 |
-
dataset_dir = "caltech-101"
|
| 21 |
-
|
| 22 |
-
def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'):
|
| 23 |
-
root = os.path.abspath(os.path.expanduser(root))
|
| 24 |
-
self.dataset_dir = os.path.join(root, self.dataset_dir)
|
| 25 |
-
self.image_dir = os.path.join(self.dataset_dir, "101_ObjectCategories")
|
| 26 |
-
self.split_path = os.path.join(self.dataset_dir, "split_zhou_Caltech101.json")
|
| 27 |
-
self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot")
|
| 28 |
-
mkdir_if_missing(self.split_fewshot_dir)
|
| 29 |
-
|
| 30 |
-
if os.path.exists(self.split_path):
|
| 31 |
-
train, val, test = OxfordPets.read_split(self.split_path, self.image_dir)
|
| 32 |
-
else:
|
| 33 |
-
train, val, test = DTD.read_and_split_data(self.image_dir, ignored=IGNORED, new_cnames=NEW_CNAMES)
|
| 34 |
-
OxfordPets.save_split(train, val, test, self.split_path, self.image_dir)
|
| 35 |
-
|
| 36 |
-
if num_shots >= 1:
|
| 37 |
-
preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
|
| 38 |
-
|
| 39 |
-
if os.path.exists(preprocessed):
|
| 40 |
-
print(f"Loading preprocessed few-shot data from {preprocessed}")
|
| 41 |
-
with open(preprocessed, "rb") as file:
|
| 42 |
-
data = pickle.load(file)
|
| 43 |
-
train, val = data["train"], data["val"]
|
| 44 |
-
else:
|
| 45 |
-
train = self.generate_fewshot_dataset(train, num_shots=num_shots)
|
| 46 |
-
val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
|
| 47 |
-
data = {"train": train, "val": val}
|
| 48 |
-
print(f"Saving preprocessed few-shot data to {preprocessed}")
|
| 49 |
-
with open(preprocessed, "wb") as file:
|
| 50 |
-
pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)
|
| 51 |
-
|
| 52 |
-
subsample = subsample_classes
|
| 53 |
-
train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample)
|
| 54 |
-
|
| 55 |
-
self.templates = [
|
| 56 |
-
lambda c: f"a photo of a {c}.",
|
| 57 |
-
lambda c: f"a painting of a {c}.",
|
| 58 |
-
lambda c: f"a plastic {c}.",
|
| 59 |
-
lambda c: f"a sculpture of a {c}.",
|
| 60 |
-
lambda c: f"a sketch of a {c}.",
|
| 61 |
-
lambda c: f"a tattoo of a {c}.",
|
| 62 |
-
lambda c: f"a toy {c}.",
|
| 63 |
-
lambda c: f"a rendition of a {c}.",
|
| 64 |
-
lambda c: f"a embroidered {c}.",
|
| 65 |
-
lambda c: f"a cartoon {c}.",
|
| 66 |
-
lambda c: f"a {c} in a video game.",
|
| 67 |
-
lambda c: f"a plushie {c}.",
|
| 68 |
-
lambda c: f"a origami {c}.",
|
| 69 |
-
lambda c: f"art of a {c}.",
|
| 70 |
-
lambda c: f"graffiti of a {c}.",
|
| 71 |
-
lambda c: f"a drawing of a {c}.",
|
| 72 |
-
lambda c: f"a doodle of a {c}.",
|
| 73 |
-
lambda c: f"a photo of the {c}.",
|
| 74 |
-
lambda c: f"a painting of the {c}.",
|
| 75 |
-
lambda c: f"the plastic {c}.",
|
| 76 |
-
lambda c: f"a sculpture of the {c}.",
|
| 77 |
-
lambda c: f"a sketch of the {c}.",
|
| 78 |
-
lambda c: f"a tattoo of the {c}.",
|
| 79 |
-
lambda c: f"the toy {c}.",
|
| 80 |
-
lambda c: f"a rendition of the {c}.",
|
| 81 |
-
lambda c: f"the embroidered {c}.",
|
| 82 |
-
lambda c: f"the cartoon {c}.",
|
| 83 |
-
lambda c: f"the {c} in a video game.",
|
| 84 |
-
lambda c: f"the plushie {c}.",
|
| 85 |
-
lambda c: f"the origami {c}.",
|
| 86 |
-
lambda c: f"art of the {c}.",
|
| 87 |
-
lambda c: f"graffiti of the {c}.",
|
| 88 |
-
lambda c: f"a drawing of the {c}.",
|
| 89 |
-
lambda c: f"a doodle of the {c}.",
|
| 90 |
-
]
|
| 91 |
-
|
| 92 |
-
super().__init__(train_x=train, val=val, test=test)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/MTIL_datasets/cifar10.py
DELETED
|
@@ -1,94 +0,0 @@
|
|
| 1 |
-
import os
|
| 2 |
-
from typing import List
|
| 3 |
-
|
| 4 |
-
from .utils import * # Datum, DatasetBase
|
| 5 |
-
from .oxford_pets import OxfordPets
|
| 6 |
-
|
| 7 |
-
try:
|
| 8 |
-
from torchvision.datasets import CIFAR10 as TorchCIFAR10
|
| 9 |
-
except Exception as e:
|
| 10 |
-
TorchCIFAR10 = None
|
| 11 |
-
print(f"Warning: torchvision not available for CIFAR10: {e}")
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
CIFAR10_CLASSES: List[str] = [
|
| 15 |
-
'airplane',
|
| 16 |
-
'automobile',
|
| 17 |
-
'bird',
|
| 18 |
-
'cat',
|
| 19 |
-
'deer',
|
| 20 |
-
'dog',
|
| 21 |
-
'frog',
|
| 22 |
-
'horse',
|
| 23 |
-
'ship',
|
| 24 |
-
'truck',
|
| 25 |
-
]
|
| 26 |
-
|
| 27 |
-
CIFAR10_TEMPLATES: List[str] = [
|
| 28 |
-
'a photo of a {}.',
|
| 29 |
-
'a blurry photo of a {}.',
|
| 30 |
-
'a black and white photo of a {}.',
|
| 31 |
-
'a low contrast photo of a {}.',
|
| 32 |
-
'a high contrast photo of a {}.',
|
| 33 |
-
'a bad photo of a {}.',
|
| 34 |
-
'a good photo of a {}.',
|
| 35 |
-
'a photo of a small {}.',
|
| 36 |
-
'a photo of a big {}.',
|
| 37 |
-
'a photo of the {}.',
|
| 38 |
-
'a blurry photo of the {}.',
|
| 39 |
-
'a black and white photo of the {}.',
|
| 40 |
-
'a low contrast photo of the {}.',
|
| 41 |
-
'a high contrast photo of the {}.',
|
| 42 |
-
'a bad photo of the {}.',
|
| 43 |
-
'a good photo of the {}.',
|
| 44 |
-
'a photo of the small {}.',
|
| 45 |
-
'a photo of the big {}.',
|
| 46 |
-
]
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
class CIFAR10(DatasetBase):
|
| 50 |
-
|
| 51 |
-
dataset_dir = "cifar10"
|
| 52 |
-
|
| 53 |
-
def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'):
|
| 54 |
-
if TorchCIFAR10 is None:
|
| 55 |
-
raise ImportError("torchvision is required for CIFAR10 dataset. Please install torchvision.")
|
| 56 |
-
|
| 57 |
-
root = os.path.abspath(os.path.expanduser(root))
|
| 58 |
-
self.dataset_dir = os.path.join(root, self.dataset_dir)
|
| 59 |
-
|
| 60 |
-
# Use torchvision to download/load data to dataset_dir
|
| 61 |
-
train_ds = TorchCIFAR10(root=self.dataset_dir, train=True, download=True)
|
| 62 |
-
test_ds = TorchCIFAR10(root=self.dataset_dir, train=False, download=True)
|
| 63 |
-
|
| 64 |
-
# Build Datum lists
|
| 65 |
-
trainval = []
|
| 66 |
-
# train_ds.data: numpy array HWC, train_ds.targets: list[int]
|
| 67 |
-
for idx in range(len(train_ds.data)):
|
| 68 |
-
img = Image.fromarray(train_ds.data[idx])
|
| 69 |
-
label = int(train_ds.targets[idx])
|
| 70 |
-
classname = CIFAR10_CLASSES[label]
|
| 71 |
-
trainval.append(Datum(impath=img, label=label, classname=classname))
|
| 72 |
-
|
| 73 |
-
test = []
|
| 74 |
-
for idx in range(len(test_ds.data)):
|
| 75 |
-
img = Image.fromarray(test_ds.data[idx])
|
| 76 |
-
label = int(test_ds.targets[idx])
|
| 77 |
-
classname = CIFAR10_CLASSES[label]
|
| 78 |
-
test.append(Datum(impath=img, label=label, classname=classname))
|
| 79 |
-
|
| 80 |
-
# Split train/val
|
| 81 |
-
train, val = OxfordPets.split_trainval(trainval)
|
| 82 |
-
|
| 83 |
-
# Few-shot sampling if requested
|
| 84 |
-
if num_shots >= 1:
|
| 85 |
-
train = self.generate_fewshot_dataset(train, num_shots=num_shots)
|
| 86 |
-
val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
|
| 87 |
-
|
| 88 |
-
# Optional class subsampling (base/new)
|
| 89 |
-
train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample_classes)
|
| 90 |
-
|
| 91 |
-
# Templates
|
| 92 |
-
self.templates = CIFAR10_TEMPLATES
|
| 93 |
-
|
| 94 |
-
super().__init__(train_x=train, val=val, test=test)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/MTIL_datasets/cifar100.py
DELETED
|
@@ -1,89 +0,0 @@
|
|
| 1 |
-
import os
|
| 2 |
-
import pickle
|
| 3 |
-
import numpy as np
|
| 4 |
-
|
| 5 |
-
from .utils import *
|
| 6 |
-
|
| 7 |
-
from .oxford_pets import OxfordPets
|
| 8 |
-
from .dtd import DescribableTextures as DTD
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
class CIFAR100(DatasetBase):
|
| 12 |
-
|
| 13 |
-
dataset_dir = "cifar100"
|
| 14 |
-
|
| 15 |
-
def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'):
|
| 16 |
-
root = os.path.abspath(os.path.expanduser(root))
|
| 17 |
-
self.dataset_dir = os.path.join(root, self.dataset_dir)
|
| 18 |
-
|
| 19 |
-
file_path = os.path.join(root, self.dataset_dir, 'train')
|
| 20 |
-
with open(file_path, "rb") as f:
|
| 21 |
-
entry = pickle.load(f, encoding="latin1")
|
| 22 |
-
trainval_data = entry["data"]
|
| 23 |
-
if "labels" in entry:
|
| 24 |
-
trainval_targets = entry["labels"]
|
| 25 |
-
else:
|
| 26 |
-
trainval_targets = entry["fine_labels"]
|
| 27 |
-
trainval_data = trainval_data.reshape(-1, 3, 32, 32)
|
| 28 |
-
trainval_data = trainval_data.transpose((0, 2, 3, 1))
|
| 29 |
-
|
| 30 |
-
file_path = os.path.join(root, self.dataset_dir, 'test')
|
| 31 |
-
with open(file_path, "rb") as f:
|
| 32 |
-
entry = pickle.load(f, encoding="latin1")
|
| 33 |
-
test_data = entry["data"]
|
| 34 |
-
if "labels" in entry:
|
| 35 |
-
test_targets = entry["labels"]
|
| 36 |
-
else:
|
| 37 |
-
test_targets = entry["fine_labels"]
|
| 38 |
-
test_data = test_data.reshape(-1, 3, 32, 32)
|
| 39 |
-
test_data = test_data.transpose((0, 2, 3, 1))
|
| 40 |
-
|
| 41 |
-
path = os.path.join(self.dataset_dir, "meta")
|
| 42 |
-
with open(path, "rb") as infile:
|
| 43 |
-
data = pickle.load(infile, encoding="latin1")
|
| 44 |
-
classes = data["fine_label_names"]
|
| 45 |
-
classes = [s.replace("_", " ") for s in classes]
|
| 46 |
-
|
| 47 |
-
trainval = []
|
| 48 |
-
for idx in range(trainval_data.shape[0]):
|
| 49 |
-
item = Datum(impath=Image.fromarray(trainval_data[idx]),
|
| 50 |
-
label=int(trainval_targets[idx]), classname=classes[trainval_targets[idx]])
|
| 51 |
-
trainval.append(item)
|
| 52 |
-
|
| 53 |
-
test = []
|
| 54 |
-
for idx in range(test_data.shape[0]):
|
| 55 |
-
item = Datum(impath=Image.fromarray(test_data[idx]),
|
| 56 |
-
label=int(test_targets[idx]), classname=classes[test_targets[idx]])
|
| 57 |
-
test.append(item)
|
| 58 |
-
|
| 59 |
-
train, val = OxfordPets.split_trainval(trainval)
|
| 60 |
-
|
| 61 |
-
if num_shots >= 1:
|
| 62 |
-
train = self.generate_fewshot_dataset(train, num_shots=num_shots)
|
| 63 |
-
val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
|
| 64 |
-
|
| 65 |
-
subsample = subsample_classes
|
| 66 |
-
train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample)
|
| 67 |
-
|
| 68 |
-
self.templates = [
|
| 69 |
-
lambda c : f'a bad photo of a {c}.',
|
| 70 |
-
lambda c : f'a blurry photo of a {c}.',
|
| 71 |
-
lambda c : f'a black and white photo of a {c}.',
|
| 72 |
-
lambda c : f'a low contrast photo of a {c}.',
|
| 73 |
-
lambda c : f'a high contrast photo of a {c}.',
|
| 74 |
-
lambda c : f'a photo of a {c}.',
|
| 75 |
-
lambda c : f'a good photo of a {c}.',
|
| 76 |
-
lambda c : f'a photo of a small {c}.',
|
| 77 |
-
lambda c : f'a photo of a big {c}.',
|
| 78 |
-
lambda c : f'a photo of the {c}.',
|
| 79 |
-
lambda c : f'a blurry photo of the {c}.',
|
| 80 |
-
lambda c : f'a black and white photo of the {c}.',
|
| 81 |
-
lambda c : f'a low contrast photo of the {c}.',
|
| 82 |
-
lambda c : f'a high contrast photo of the {c}.',
|
| 83 |
-
lambda c : f'a bad photo of the {c}.',
|
| 84 |
-
lambda c : f'a good photo of the {c}.',
|
| 85 |
-
lambda c : f'a photo of the small {c}.',
|
| 86 |
-
lambda c : f'a photo of the big {c}.',
|
| 87 |
-
]
|
| 88 |
-
|
| 89 |
-
super().__init__(train_x=train, val=val, test=test)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/MTIL_datasets/clevr_count.py
DELETED
|
@@ -1,118 +0,0 @@
|
|
| 1 |
-
import os
|
| 2 |
-
import pickle
|
| 3 |
-
from typing import List
|
| 4 |
-
|
| 5 |
-
from .utils import Datum, DatasetBase, mkdir_if_missing
|
| 6 |
-
from .oxford_pets import OxfordPets
|
| 7 |
-
from .utils import read_json, write_json
|
| 8 |
-
|
| 9 |
-
# Class list and templates as specified
|
| 10 |
-
CLEVR_COUNT_CLASSES: List[str] = [
|
| 11 |
-
'10', '3', '4', '5', '6', '7', '8', '9'
|
| 12 |
-
]
|
| 13 |
-
|
| 14 |
-
CLEVR_COUNT_TEMPLATES: List[str] = [
|
| 15 |
-
'a photo of {} objects.',
|
| 16 |
-
]
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
class CLEVRCount(DatasetBase):
|
| 20 |
-
|
| 21 |
-
dataset_dir = 'clevr'
|
| 22 |
-
|
| 23 |
-
def __init__(self, root, num_shots: int = 0, seed: int = 1, subsample_classes: str = 'all'):
|
| 24 |
-
# Root and directories
|
| 25 |
-
root = os.path.abspath(os.path.expanduser(root))
|
| 26 |
-
self.dataset_dir = os.path.join(root, self.dataset_dir)
|
| 27 |
-
self.images_dir = os.path.join(self.dataset_dir, 'images')
|
| 28 |
-
self.scenes_dir = os.path.join(self.dataset_dir, 'scenes')
|
| 29 |
-
self.split_path = os.path.join(self.dataset_dir, 'split_custom_CLEVRCount.json')
|
| 30 |
-
self.split_fewshot_dir = os.path.join(self.dataset_dir, 'split_fewshot')
|
| 31 |
-
mkdir_if_missing(self.split_fewshot_dir)
|
| 32 |
-
|
| 33 |
-
# Required files
|
| 34 |
-
train_scenes = os.path.join(self.scenes_dir, 'CLEVR_train_scenes.json')
|
| 35 |
-
val_scenes = os.path.join(self.scenes_dir, 'CLEVR_val_scenes.json')
|
| 36 |
-
if not os.path.isfile(train_scenes) or not os.path.isfile(val_scenes):
|
| 37 |
-
raise FileNotFoundError(
|
| 38 |
-
f"CLEVRCount expects scenes JSON at {train_scenes} and {val_scenes}"
|
| 39 |
-
)
|
| 40 |
-
|
| 41 |
-
# Load or build split
|
| 42 |
-
if os.path.exists(self.split_path):
|
| 43 |
-
train, val, test = OxfordPets.read_split(self.split_path, self.dataset_dir)
|
| 44 |
-
else:
|
| 45 |
-
trainval = self._read_scenes(train_scenes, split='train')
|
| 46 |
-
test = self._read_scenes(val_scenes, split='val')
|
| 47 |
-
train, val = OxfordPets.split_trainval(trainval)
|
| 48 |
-
OxfordPets.save_split(train, val, test, self.split_path, self.dataset_dir)
|
| 49 |
-
|
| 50 |
-
# Few-shot
|
| 51 |
-
if num_shots >= 1:
|
| 52 |
-
preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
|
| 53 |
-
if os.path.exists(preprocessed):
|
| 54 |
-
print(f"Loading preprocessed few-shot data from {preprocessed}")
|
| 55 |
-
with open(preprocessed, 'rb') as f:
|
| 56 |
-
data = pickle.load(f)
|
| 57 |
-
train, val = data['train'], data['val']
|
| 58 |
-
else:
|
| 59 |
-
train = self.generate_fewshot_dataset(train, num_shots=num_shots)
|
| 60 |
-
val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
|
| 61 |
-
data = {'train': train, 'val': val}
|
| 62 |
-
print(f"Saving preprocessed few-shot data to {preprocessed}")
|
| 63 |
-
with open(preprocessed, 'wb') as f:
|
| 64 |
-
pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL)
|
| 65 |
-
|
| 66 |
-
# Optional class subsampling (base/new)
|
| 67 |
-
train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample_classes)
|
| 68 |
-
|
| 69 |
-
# Templates
|
| 70 |
-
self.templates = CLEVR_COUNT_TEMPLATES
|
| 71 |
-
|
| 72 |
-
# Debug stats
|
| 73 |
-
try:
|
| 74 |
-
def _hist(items: List[Datum]):
|
| 75 |
-
from collections import Counter
|
| 76 |
-
cnt = Counter([it.label for it in items])
|
| 77 |
-
out = {i: int(cnt.get(i, 0)) for i in range(len(CLEVR_COUNT_CLASSES))}
|
| 78 |
-
return out
|
| 79 |
-
except Exception as e:
|
| 80 |
-
print(f"CLEVRCount stats printing failed: {e}")
|
| 81 |
-
|
| 82 |
-
super().__init__(train_x=train, val=val, test=test)
|
| 83 |
-
# Ensure class metadata is stable and matches the provided list
|
| 84 |
-
self._classnames = CLEVR_COUNT_CLASSES
|
| 85 |
-
self._lab2cname = {i: c for i, c in enumerate(CLEVR_COUNT_CLASSES)}
|
| 86 |
-
self._num_classes = len(CLEVR_COUNT_CLASSES)
|
| 87 |
-
|
| 88 |
-
def _read_scenes(self, json_path: str, split: str) -> List[Datum]:
|
| 89 |
-
"""Read CLEVR scenes JSON and construct a list of Datum entries.
|
| 90 |
-
Only images whose object count appears in CLEVR_COUNT_CLASSES are kept.
|
| 91 |
-
"""
|
| 92 |
-
obj = read_json(json_path)
|
| 93 |
-
scenes = obj.get('scenes', [])
|
| 94 |
-
# Map count -> class index
|
| 95 |
-
lab2idx = {int(c): i for i, c in enumerate(CLEVR_COUNT_CLASSES)}
|
| 96 |
-
items: List[Datum] = []
|
| 97 |
-
for sc in scenes:
|
| 98 |
-
# Some files use key 'split', some typos list 'spit'; be robust
|
| 99 |
-
image_filename = sc.get('image_filename', None)
|
| 100 |
-
if not image_filename:
|
| 101 |
-
continue
|
| 102 |
-
n_objects = sc.get('objects', [])
|
| 103 |
-
try:
|
| 104 |
-
num = int(len(n_objects))
|
| 105 |
-
except Exception:
|
| 106 |
-
continue
|
| 107 |
-
if num not in lab2idx:
|
| 108 |
-
# skip counts not in the configured class list
|
| 109 |
-
continue
|
| 110 |
-
label_i = lab2idx[num]
|
| 111 |
-
# Build absolute path to the image based on split
|
| 112 |
-
if split not in ['train', 'val', 'test']:
|
| 113 |
-
split_dir = 'train'
|
| 114 |
-
else:
|
| 115 |
-
split_dir = split
|
| 116 |
-
impath = os.path.join(self.dataset_dir, 'images', split_dir, image_filename)
|
| 117 |
-
items.append(Datum(impath=impath, label=label_i, classname=str(num)))
|
| 118 |
-
return items
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/MTIL_datasets/country211.py
DELETED
|
@@ -1,386 +0,0 @@
|
|
| 1 |
-
import os
|
| 2 |
-
import json
|
| 3 |
-
import pickle
|
| 4 |
-
import re
|
| 5 |
-
|
| 6 |
-
from .utils import *
|
| 7 |
-
from .oxford_pets import OxfordPets
|
| 8 |
-
from .dtd import DescribableTextures as DTD
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
class Country211(DatasetBase):
|
| 12 |
-
"""
|
| 13 |
-
Country211 dataset loader for MTIL.
|
| 14 |
-
Expected structure:
|
| 15 |
-
<root>/country211/
|
| 16 |
-
├─ train/<ISO2>/*.jpg
|
| 17 |
-
├─ valid/<ISO2>/*.jpg
|
| 18 |
-
└─ test/<ISO2>/*.jpg
|
| 19 |
-
|
| 20 |
-
Where <ISO2> are ISO-3166 alpha-2 country codes (e.g., AD, US, CN).
|
| 21 |
-
Class names are mapped from ISO2 codes to provided human-readable names.
|
| 22 |
-
Optionally, a JSON mapping file can override defaults:
|
| 23 |
-
<root>/country211/iso2_to_name.json => {"AD": "Andorra", ...}
|
| 24 |
-
"""
|
| 25 |
-
|
| 26 |
-
dataset_dir = "country211"
|
| 27 |
-
|
| 28 |
-
def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'):
|
| 29 |
-
root = os.path.abspath(os.path.expanduser(root))
|
| 30 |
-
self.dataset_dir = os.path.join(root, self.dataset_dir)
|
| 31 |
-
self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot")
|
| 32 |
-
mkdir_if_missing(self.split_fewshot_dir)
|
| 33 |
-
|
| 34 |
-
# Prefer explicit split directories if they exist
|
| 35 |
-
train_dir = os.path.join(self.dataset_dir, "train")
|
| 36 |
-
valid_dir = os.path.join(self.dataset_dir, "valid")
|
| 37 |
-
test_dir = os.path.join(self.dataset_dir, "test")
|
| 38 |
-
|
| 39 |
-
use_folder_splits = os.path.isdir(train_dir) and os.path.isdir(valid_dir) and os.path.isdir(test_dir)
|
| 40 |
-
|
| 41 |
-
if use_folder_splits:
|
| 42 |
-
# Build ISO2 -> name mapping (allow override via json)
|
| 43 |
-
iso_map = self._load_iso2_to_name()
|
| 44 |
-
# Create unified code set across splits to keep label mapping stable
|
| 45 |
-
all_codes = set()
|
| 46 |
-
for d in [train_dir, valid_dir, test_dir]:
|
| 47 |
-
if os.path.isdir(d):
|
| 48 |
-
for code in listdir_nohidden(d):
|
| 49 |
-
code_path = os.path.join(d, code)
|
| 50 |
-
if os.path.isdir(code_path):
|
| 51 |
-
all_codes.add(code)
|
| 52 |
-
# Validate code format and mapping coverage early
|
| 53 |
-
override_path = os.path.join(self.dataset_dir, 'iso2_to_name.json')
|
| 54 |
-
pat = re.compile(r'^[A-Z]{2}$')
|
| 55 |
-
invalid_codes = sorted([c for c in all_codes if not (pat.match(c) or c == 'XK')])
|
| 56 |
-
if invalid_codes:
|
| 57 |
-
raise ValueError(
|
| 58 |
-
"Country211: Found invalid ISO2 code folder names: {}. "
|
| 59 |
-
"Codes must be two uppercase letters (e.g., 'US', 'CN') or 'XK'. "
|
| 60 |
-
"Please rename these folders accordingly.".format(invalid_codes)
|
| 61 |
-
)
|
| 62 |
-
unknown_codes = sorted([c for c in all_codes if c not in iso_map])
|
| 63 |
-
if unknown_codes:
|
| 64 |
-
raise ValueError(
|
| 65 |
-
"Country211: ISO2 codes missing from mapping: {}. "
|
| 66 |
-
"Add them to {} as a JSON dict, e.g., {\"XX\": \"Country Name\"}.".format(
|
| 67 |
-
unknown_codes, override_path
|
| 68 |
-
)
|
| 69 |
-
)
|
| 70 |
-
# Sort by human name (fallback to code) for stable labels
|
| 71 |
-
def _code_to_name(c):
|
| 72 |
-
return iso_map.get(c, c)
|
| 73 |
-
sorted_codes = sorted(list(all_codes), key=lambda c: _code_to_name(c))
|
| 74 |
-
code_to_label = {c: i for i, c in enumerate(sorted_codes)}
|
| 75 |
-
|
| 76 |
-
train = self._read_split_dir(train_dir, code_to_label, iso_map)
|
| 77 |
-
val = self._read_split_dir(valid_dir, code_to_label, iso_map)
|
| 78 |
-
test = self._read_split_dir(test_dir, code_to_label, iso_map)
|
| 79 |
-
else:
|
| 80 |
-
# Fallbacks: JSON split or naive folder split
|
| 81 |
-
image_dir = os.path.join(self.dataset_dir, "images")
|
| 82 |
-
self.image_dir = image_dir if os.path.isdir(image_dir) else self.dataset_dir
|
| 83 |
-
split_path = os.path.join(self.dataset_dir, "split_zhou_Country211.json")
|
| 84 |
-
if os.path.exists(split_path):
|
| 85 |
-
train, val, test = OxfordPets.read_split(split_path, self.image_dir)
|
| 86 |
-
else:
|
| 87 |
-
train, val, test = DTD.read_and_split_data(self.image_dir)
|
| 88 |
-
OxfordPets.save_split(train, val, test, split_path, self.image_dir)
|
| 89 |
-
|
| 90 |
-
if num_shots >= 1:
|
| 91 |
-
preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
|
| 92 |
-
if os.path.exists(preprocessed):
|
| 93 |
-
print(f"Loading preprocessed few-shot data from {preprocessed}")
|
| 94 |
-
with open(preprocessed, "rb") as file:
|
| 95 |
-
data = pickle.load(file)
|
| 96 |
-
train, val = data["train"], data["val"]
|
| 97 |
-
else:
|
| 98 |
-
train = self.generate_fewshot_dataset(train, num_shots=num_shots)
|
| 99 |
-
val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
|
| 100 |
-
data = {"train": train, "val": val}
|
| 101 |
-
print(f"Saving preprocessed few-shot data to {preprocessed}")
|
| 102 |
-
with open(preprocessed, "wb") as file:
|
| 103 |
-
pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)
|
| 104 |
-
|
| 105 |
-
subsample = subsample_classes
|
| 106 |
-
train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample)
|
| 107 |
-
|
| 108 |
-
# Templates provided by user
|
| 109 |
-
self.templates = [
|
| 110 |
-
lambda c: f'a photo i took in {c}.',
|
| 111 |
-
lambda c: f'a photo i took while visiting {c}.',
|
| 112 |
-
lambda c: f'a photo from my home country of {c}.',
|
| 113 |
-
lambda c: f'a photo from my visit to {c}.',
|
| 114 |
-
lambda c: f'a photo showing the country of {c}.',
|
| 115 |
-
]
|
| 116 |
-
|
| 117 |
-
super().__init__(train_x=train, val=val, test=test)
|
| 118 |
-
|
| 119 |
-
def _read_split_dir(self, split_dir, code_to_label, iso_map):
|
| 120 |
-
items = []
|
| 121 |
-
if not os.path.isdir(split_dir):
|
| 122 |
-
return items
|
| 123 |
-
codes = listdir_nohidden(split_dir)
|
| 124 |
-
for code in codes:
|
| 125 |
-
class_dir = os.path.join(split_dir, code)
|
| 126 |
-
if not os.path.isdir(class_dir):
|
| 127 |
-
continue
|
| 128 |
-
label = code_to_label.get(code)
|
| 129 |
-
if label is None:
|
| 130 |
-
raise ValueError(
|
| 131 |
-
f"Country211: Inconsistent label mapping for code '{code}' in split '{split_dir}'. "
|
| 132 |
-
f"This indicates an internal mismatch between discovered codes and label map."
|
| 133 |
-
)
|
| 134 |
-
try:
|
| 135 |
-
cname = iso_map[code]
|
| 136 |
-
except KeyError:
|
| 137 |
-
raise ValueError(
|
| 138 |
-
f"Country211: Missing ISO mapping for code '{code}'. "
|
| 139 |
-
f"Please add it to '{os.path.join(self.dataset_dir, 'iso2_to_name.json')}'."
|
| 140 |
-
)
|
| 141 |
-
for fname in listdir_nohidden(class_dir):
|
| 142 |
-
impath = os.path.join(class_dir, fname)
|
| 143 |
-
items.append(Datum(impath=impath, label=label, classname=cname))
|
| 144 |
-
return items
|
| 145 |
-
|
| 146 |
-
def _load_iso2_to_name(self):
|
| 147 |
-
"""Load ISO2->country name mapping, allow local JSON override.
|
| 148 |
-
Fallback to built-in mapping; unknown codes map to themselves at usage time.
|
| 149 |
-
"""
|
| 150 |
-
override_path = os.path.join(self.dataset_dir, 'iso2_to_name.json')
|
| 151 |
-
if os.path.exists(override_path):
|
| 152 |
-
try:
|
| 153 |
-
data = read_json(override_path)
|
| 154 |
-
if not isinstance(data, dict):
|
| 155 |
-
raise ValueError(
|
| 156 |
-
f"Country211: Expected a JSON object in '{override_path}', got {type(data)}"
|
| 157 |
-
)
|
| 158 |
-
# Validate keys as ISO2 (two uppercase letters) or 'XK', and values are non-empty strings
|
| 159 |
-
pat = re.compile(r'^[A-Z]{2}$')
|
| 160 |
-
bad_keys = [k for k in data.keys() if not (isinstance(k, str) and (pat.match(k) or k == 'XK'))]
|
| 161 |
-
bad_vals = [k for k, v in data.items() if not (isinstance(v, str) and v.strip())]
|
| 162 |
-
if bad_keys or bad_vals:
|
| 163 |
-
raise ValueError(
|
| 164 |
-
(
|
| 165 |
-
"Country211: Invalid iso2_to_name.json entries. "
|
| 166 |
-
f"Bad keys (must be ISO2 like 'US'): {bad_keys}. "
|
| 167 |
-
f"Bad values (must be non-empty strings) for keys: {bad_vals}."
|
| 168 |
-
)
|
| 169 |
-
)
|
| 170 |
-
return data
|
| 171 |
-
except Exception as e:
|
| 172 |
-
raise ValueError(f"Country211: Failed to load '{override_path}': {e}")
|
| 173 |
-
# Built-in mapping aligned with provided classes
|
| 174 |
-
return {
|
| 175 |
-
'AD': 'Andorra',
|
| 176 |
-
'AE': 'United Arab Emirates',
|
| 177 |
-
'AF': 'Afghanistan',
|
| 178 |
-
'AG': 'Antigua and Barbuda',
|
| 179 |
-
'AI': 'Anguilla',
|
| 180 |
-
'AL': 'Albania',
|
| 181 |
-
'AM': 'Armenia',
|
| 182 |
-
'AO': 'Angola',
|
| 183 |
-
'AQ': 'Antarctica',
|
| 184 |
-
'AR': 'Argentina',
|
| 185 |
-
'AT': 'Austria',
|
| 186 |
-
'AU': 'Australia',
|
| 187 |
-
'AW': 'Aruba',
|
| 188 |
-
'AX': 'Aland Islands',
|
| 189 |
-
'AZ': 'Azerbaijan',
|
| 190 |
-
'BA': 'Bosnia and Herzegovina',
|
| 191 |
-
'BB': 'Barbados',
|
| 192 |
-
'BD': 'Bangladesh',
|
| 193 |
-
'BE': 'Belgium',
|
| 194 |
-
'BF': 'Burkina Faso',
|
| 195 |
-
'BG': 'Bulgaria',
|
| 196 |
-
'BH': 'Bahrain',
|
| 197 |
-
'BJ': 'Benin',
|
| 198 |
-
'BM': 'Bermuda',
|
| 199 |
-
'BN': 'Brunei Darussalam',
|
| 200 |
-
'BO': 'Bolivia',
|
| 201 |
-
'BQ': 'Bonaire, Saint Eustatius and Saba',
|
| 202 |
-
'BR': 'Brazil',
|
| 203 |
-
'BS': 'Bahamas',
|
| 204 |
-
'BT': 'Bhutan',
|
| 205 |
-
'BW': 'Botswana',
|
| 206 |
-
'BY': 'Belarus',
|
| 207 |
-
'BZ': 'Belize',
|
| 208 |
-
'CA': 'Canada',
|
| 209 |
-
'CD': 'DR Congo',
|
| 210 |
-
'CF': 'Central African Republic',
|
| 211 |
-
'CH': 'Switzerland',
|
| 212 |
-
'CI': "Cote d'Ivoire",
|
| 213 |
-
'CK': 'Cook Islands',
|
| 214 |
-
'CL': 'Chile',
|
| 215 |
-
'CM': 'Cameroon',
|
| 216 |
-
'CN': 'China',
|
| 217 |
-
'CO': 'Colombia',
|
| 218 |
-
'CR': 'Costa Rica',
|
| 219 |
-
'CU': 'Cuba',
|
| 220 |
-
'CV': 'Cabo Verde',
|
| 221 |
-
'CW': 'Curacao',
|
| 222 |
-
'CY': 'Cyprus',
|
| 223 |
-
'CZ': 'Czech Republic',
|
| 224 |
-
'DE': 'Germany',
|
| 225 |
-
'DK': 'Denmark',
|
| 226 |
-
'DM': 'Dominica',
|
| 227 |
-
'DO': 'Dominican Republic',
|
| 228 |
-
'DZ': 'Algeria',
|
| 229 |
-
'EC': 'Ecuador',
|
| 230 |
-
'EE': 'Estonia',
|
| 231 |
-
'EG': 'Egypt',
|
| 232 |
-
'ES': 'Spain',
|
| 233 |
-
'ET': 'Ethiopia',
|
| 234 |
-
'FI': 'Finland',
|
| 235 |
-
'FJ': 'Fiji',
|
| 236 |
-
'FK': 'Falkland Islands',
|
| 237 |
-
'FO': 'Faeroe Islands',
|
| 238 |
-
'FR': 'France',
|
| 239 |
-
'GA': 'Gabon',
|
| 240 |
-
'GB': 'United Kingdom',
|
| 241 |
-
'GD': 'Grenada',
|
| 242 |
-
'GE': 'Georgia',
|
| 243 |
-
'GF': 'French Guiana',
|
| 244 |
-
'GG': 'Guernsey',
|
| 245 |
-
'GH': 'Ghana',
|
| 246 |
-
'GI': 'Gibraltar',
|
| 247 |
-
'GL': 'Greenland',
|
| 248 |
-
'GM': 'Gambia',
|
| 249 |
-
'GP': 'Guadeloupe',
|
| 250 |
-
'GR': 'Greece',
|
| 251 |
-
'GS': 'South Georgia and South Sandwich Is.',
|
| 252 |
-
'GT': 'Guatemala',
|
| 253 |
-
'GU': 'Guam',
|
| 254 |
-
'GY': 'Guyana',
|
| 255 |
-
'HK': 'Hong Kong',
|
| 256 |
-
'HN': 'Honduras',
|
| 257 |
-
'HR': 'Croatia',
|
| 258 |
-
'HT': 'Haiti',
|
| 259 |
-
'HU': 'Hungary',
|
| 260 |
-
'ID': 'Indonesia',
|
| 261 |
-
'IE': 'Ireland',
|
| 262 |
-
'IL': 'Israel',
|
| 263 |
-
'IM': 'Isle of Man',
|
| 264 |
-
'IN': 'India',
|
| 265 |
-
'IQ': 'Iraq',
|
| 266 |
-
'IR': 'Iran',
|
| 267 |
-
'IS': 'Iceland',
|
| 268 |
-
'IT': 'Italy',
|
| 269 |
-
'JE': 'Jersey',
|
| 270 |
-
'JM': 'Jamaica',
|
| 271 |
-
'JO': 'Jordan',
|
| 272 |
-
'JP': 'Japan',
|
| 273 |
-
'KE': 'Kenya',
|
| 274 |
-
'KG': 'Kyrgyz Republic',
|
| 275 |
-
'KH': 'Cambodia',
|
| 276 |
-
'KN': 'St. Kitts and Nevis',
|
| 277 |
-
'KP': 'North Korea',
|
| 278 |
-
'KR': 'South Korea',
|
| 279 |
-
'KW': 'Kuwait',
|
| 280 |
-
'KY': 'Cayman Islands',
|
| 281 |
-
'KZ': 'Kazakhstan',
|
| 282 |
-
'LA': 'Laos',
|
| 283 |
-
'LB': 'Lebanon',
|
| 284 |
-
'LC': 'St. Lucia',
|
| 285 |
-
'LI': 'Liechtenstein',
|
| 286 |
-
'LK': 'Sri Lanka',
|
| 287 |
-
'LR': 'Liberia',
|
| 288 |
-
'LT': 'Lithuania',
|
| 289 |
-
'LU': 'Luxembourg',
|
| 290 |
-
'LV': 'Latvia',
|
| 291 |
-
'LY': 'Libya',
|
| 292 |
-
'MA': 'Morocco',
|
| 293 |
-
'MC': 'Monaco',
|
| 294 |
-
'MD': 'Moldova',
|
| 295 |
-
'ME': 'Montenegro',
|
| 296 |
-
'MF': 'Saint-Martin',
|
| 297 |
-
'MG': 'Madagascar',
|
| 298 |
-
'MK': 'Macedonia',
|
| 299 |
-
'ML': 'Mali',
|
| 300 |
-
'MM': 'Myanmar',
|
| 301 |
-
'MN': 'Mongolia',
|
| 302 |
-
'MO': 'Macau',
|
| 303 |
-
'MQ': 'Martinique',
|
| 304 |
-
'MR': 'Mauritania',
|
| 305 |
-
'MT': 'Malta',
|
| 306 |
-
'MU': 'Mauritius',
|
| 307 |
-
'MV': 'Maldives',
|
| 308 |
-
'MW': 'Malawi',
|
| 309 |
-
'MX': 'Mexico',
|
| 310 |
-
'MY': 'Malaysia',
|
| 311 |
-
'MZ': 'Mozambique',
|
| 312 |
-
'NA': 'Namibia',
|
| 313 |
-
'NC': 'New Caledonia',
|
| 314 |
-
'NG': 'Nigeria',
|
| 315 |
-
'NI': 'Nicaragua',
|
| 316 |
-
'NL': 'Netherlands',
|
| 317 |
-
'NO': 'Norway',
|
| 318 |
-
'NP': 'Nepal',
|
| 319 |
-
'NZ': 'New Zealand',
|
| 320 |
-
'OM': 'Oman',
|
| 321 |
-
'PA': 'Panama',
|
| 322 |
-
'PE': 'Peru',
|
| 323 |
-
'PF': 'French Polynesia',
|
| 324 |
-
'PG': 'Papua New Guinea',
|
| 325 |
-
'PH': 'Philippines',
|
| 326 |
-
'PK': 'Pakistan',
|
| 327 |
-
'PL': 'Poland',
|
| 328 |
-
'PR': 'Puerto Rico',
|
| 329 |
-
'PS': 'Palestine',
|
| 330 |
-
'PT': 'Portugal',
|
| 331 |
-
'PW': 'Palau',
|
| 332 |
-
'PY': 'Paraguay',
|
| 333 |
-
'QA': 'Qatar',
|
| 334 |
-
'RE': 'Reunion',
|
| 335 |
-
'RO': 'Romania',
|
| 336 |
-
'RS': 'Serbia',
|
| 337 |
-
'RU': 'Russia',
|
| 338 |
-
'RW': 'Rwanda',
|
| 339 |
-
'SA': 'Saudi Arabia',
|
| 340 |
-
'SB': 'Solomon Islands',
|
| 341 |
-
'SC': 'Seychelles',
|
| 342 |
-
'SD': 'Sudan',
|
| 343 |
-
'SE': 'Sweden',
|
| 344 |
-
'SG': 'Singapore',
|
| 345 |
-
'SH': 'St. Helena',
|
| 346 |
-
'SI': 'Slovenia',
|
| 347 |
-
'SJ': 'Svalbard and Jan Mayen Islands',
|
| 348 |
-
'SK': 'Slovakia',
|
| 349 |
-
'SL': 'Sierra Leone',
|
| 350 |
-
'SM': 'San Marino',
|
| 351 |
-
'SN': 'Senegal',
|
| 352 |
-
'SO': 'Somalia',
|
| 353 |
-
'SS': 'South Sudan',
|
| 354 |
-
'SV': 'El Salvador',
|
| 355 |
-
'SX': 'Sint Maarten',
|
| 356 |
-
'SY': 'Syria',
|
| 357 |
-
'SZ': 'Eswatini',
|
| 358 |
-
'TG': 'Togo',
|
| 359 |
-
'TH': 'Thailand',
|
| 360 |
-
'TJ': 'Tajikistan',
|
| 361 |
-
'TL': 'Timor-Leste',
|
| 362 |
-
'TM': 'Turkmenistan',
|
| 363 |
-
'TN': 'Tunisia',
|
| 364 |
-
'TO': 'Tonga',
|
| 365 |
-
'TR': 'Turkey',
|
| 366 |
-
'TT': 'Trinidad and Tobago',
|
| 367 |
-
'TW': 'Taiwan',
|
| 368 |
-
'TZ': 'Tanzania',
|
| 369 |
-
'UA': 'Ukraine',
|
| 370 |
-
'UG': 'Uganda',
|
| 371 |
-
'US': 'United States',
|
| 372 |
-
'UY': 'Uruguay',
|
| 373 |
-
'UZ': 'Uzbekistan',
|
| 374 |
-
'VA': 'Vatican',
|
| 375 |
-
'VE': 'Venezuela',
|
| 376 |
-
'VG': 'British Virgin Islands',
|
| 377 |
-
'VI': 'United States Virgin Islands',
|
| 378 |
-
'VN': 'Vietnam',
|
| 379 |
-
'VU': 'Vanuatu',
|
| 380 |
-
'WS': 'Samoa',
|
| 381 |
-
'XK': 'Kosovo',
|
| 382 |
-
'YE': 'Yemen',
|
| 383 |
-
'ZA': 'South Africa',
|
| 384 |
-
'ZM': 'Zambia',
|
| 385 |
-
'ZW': 'Zimbabwe',
|
| 386 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/MTIL_datasets/dtd.py
DELETED
|
@@ -1,95 +0,0 @@
|
|
| 1 |
-
import os
|
| 2 |
-
import pickle
|
| 3 |
-
import random
|
| 4 |
-
|
| 5 |
-
from .utils import *
|
| 6 |
-
|
| 7 |
-
from .oxford_pets import OxfordPets
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
class DescribableTextures(DatasetBase):
|
| 11 |
-
|
| 12 |
-
dataset_dir = "dtd"
|
| 13 |
-
|
| 14 |
-
def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'):
|
| 15 |
-
root = os.path.abspath(os.path.expanduser(root))
|
| 16 |
-
self.dataset_dir = os.path.join(root, self.dataset_dir)
|
| 17 |
-
self.image_dir = os.path.join(self.dataset_dir, "images")
|
| 18 |
-
self.split_path = os.path.join(self.dataset_dir, "split_zhou_DescribableTextures.json")
|
| 19 |
-
self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot")
|
| 20 |
-
mkdir_if_missing(self.split_fewshot_dir)
|
| 21 |
-
|
| 22 |
-
if os.path.exists(self.split_path):
|
| 23 |
-
train, val, test = OxfordPets.read_split(self.split_path, self.image_dir)
|
| 24 |
-
else:
|
| 25 |
-
train, val, test = self.read_and_split_data(self.image_dir)
|
| 26 |
-
OxfordPets.save_split(train, val, test, self.split_path, self.image_dir)
|
| 27 |
-
|
| 28 |
-
if num_shots >= 1:
|
| 29 |
-
preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
|
| 30 |
-
|
| 31 |
-
if os.path.exists(preprocessed):
|
| 32 |
-
print(f"Loading preprocessed few-shot data from {preprocessed}")
|
| 33 |
-
with open(preprocessed, "rb") as file:
|
| 34 |
-
data = pickle.load(file)
|
| 35 |
-
train, val = data["train"], data["val"]
|
| 36 |
-
else:
|
| 37 |
-
train = self.generate_fewshot_dataset(train, num_shots=num_shots)
|
| 38 |
-
val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
|
| 39 |
-
data = {"train": train, "val": val}
|
| 40 |
-
print(f"Saving preprocessed few-shot data to {preprocessed}")
|
| 41 |
-
with open(preprocessed, "wb") as file:
|
| 42 |
-
pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)
|
| 43 |
-
|
| 44 |
-
subsample = subsample_classes
|
| 45 |
-
train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample)
|
| 46 |
-
|
| 47 |
-
self.templates = [
|
| 48 |
-
lambda c: f'a photo of a {c} texture.',
|
| 49 |
-
lambda c: f'a photo of a {c} pattern.',
|
| 50 |
-
lambda c: f'a photo of a {c} thing.',
|
| 51 |
-
lambda c: f'a photo of a {c} object.',
|
| 52 |
-
lambda c: f'a photo of the {c} texture.',
|
| 53 |
-
lambda c: f'a photo of the {c} pattern.',
|
| 54 |
-
lambda c: f'a photo of the {c} thing.',
|
| 55 |
-
lambda c: f'a photo of the {c} object.',
|
| 56 |
-
]
|
| 57 |
-
|
| 58 |
-
super().__init__(train_x=train, val=val, test=test)
|
| 59 |
-
|
| 60 |
-
@staticmethod
|
| 61 |
-
def read_and_split_data(image_dir, p_trn=0.5, p_val=0.2, ignored=[], new_cnames=None):
|
| 62 |
-
categories = listdir_nohidden(image_dir)
|
| 63 |
-
categories = [c for c in categories if c not in ignored]
|
| 64 |
-
categories.sort()
|
| 65 |
-
|
| 66 |
-
p_tst = 1 - p_trn - p_val
|
| 67 |
-
print(f"Splitting into {p_trn:.0%} train, {p_val:.0%} val, and {p_tst:.0%} test")
|
| 68 |
-
|
| 69 |
-
def _collate(ims, y, c):
|
| 70 |
-
items = []
|
| 71 |
-
for im in ims:
|
| 72 |
-
item = Datum(impath=im, label=y, classname=c)
|
| 73 |
-
items.append(item)
|
| 74 |
-
return items
|
| 75 |
-
|
| 76 |
-
train, val, test = [], [], []
|
| 77 |
-
for label, category in enumerate(categories):
|
| 78 |
-
category_dir = os.path.join(image_dir, category)
|
| 79 |
-
images = listdir_nohidden(category_dir)
|
| 80 |
-
images = [os.path.join(category_dir, im) for im in images]
|
| 81 |
-
random.shuffle(images)
|
| 82 |
-
n_total = len(images)
|
| 83 |
-
n_train = round(n_total * p_trn)
|
| 84 |
-
n_val = round(n_total * p_val)
|
| 85 |
-
n_test = n_total - n_train - n_val
|
| 86 |
-
assert n_train > 0 and n_val > 0 and n_test > 0
|
| 87 |
-
|
| 88 |
-
if new_cnames is not None and category in new_cnames:
|
| 89 |
-
category = new_cnames[category]
|
| 90 |
-
|
| 91 |
-
train.extend(_collate(images[:n_train], label, category))
|
| 92 |
-
val.extend(_collate(images[n_train : n_train + n_val], label, category))
|
| 93 |
-
test.extend(_collate(images[n_train + n_val :], label, category))
|
| 94 |
-
|
| 95 |
-
return train, val, test
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/MTIL_datasets/eurosat.py
DELETED
|
@@ -1,75 +0,0 @@
|
|
| 1 |
-
import os
|
| 2 |
-
import pickle
|
| 3 |
-
|
| 4 |
-
from .utils import *
|
| 5 |
-
|
| 6 |
-
from .oxford_pets import OxfordPets
|
| 7 |
-
from .dtd import DescribableTextures as DTD
|
| 8 |
-
|
| 9 |
-
NEW_CNAMES = {
|
| 10 |
-
"AnnualCrop": "Annual Crop Land",
|
| 11 |
-
"Forest": "Forest",
|
| 12 |
-
"HerbaceousVegetation": "Herbaceous Vegetation Land",
|
| 13 |
-
"Highway": "Highway or Road",
|
| 14 |
-
"Industrial": "Industrial Buildings",
|
| 15 |
-
"Pasture": "Pasture Land",
|
| 16 |
-
"PermanentCrop": "Permanent Crop Land",
|
| 17 |
-
"Residential": "Residential Buildings",
|
| 18 |
-
"River": "River",
|
| 19 |
-
"SeaLake": "Sea or Lake",
|
| 20 |
-
}
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
class EuroSAT(DatasetBase):
|
| 24 |
-
|
| 25 |
-
dataset_dir = "eurosat"
|
| 26 |
-
|
| 27 |
-
def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'):
|
| 28 |
-
root = os.path.abspath(os.path.expanduser(root))
|
| 29 |
-
self.dataset_dir = os.path.join(root, self.dataset_dir)
|
| 30 |
-
self.image_dir = os.path.join(self.dataset_dir, "2750")
|
| 31 |
-
self.split_path = os.path.join(self.dataset_dir, "split_zhou_EuroSAT.json")
|
| 32 |
-
self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot")
|
| 33 |
-
mkdir_if_missing(self.split_fewshot_dir)
|
| 34 |
-
|
| 35 |
-
if os.path.exists(self.split_path):
|
| 36 |
-
train, val, test = OxfordPets.read_split(self.split_path, self.image_dir)
|
| 37 |
-
else:
|
| 38 |
-
train, val, test = DTD.read_and_split_data(self.image_dir, new_cnames=NEW_CNAMES)
|
| 39 |
-
OxfordPets.save_split(train, val, test, self.split_path, self.image_dir)
|
| 40 |
-
|
| 41 |
-
if num_shots >= 1:
|
| 42 |
-
preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
|
| 43 |
-
|
| 44 |
-
if os.path.exists(preprocessed):
|
| 45 |
-
print(f"Loading preprocessed few-shot data from {preprocessed}")
|
| 46 |
-
with open(preprocessed, "rb") as file:
|
| 47 |
-
data = pickle.load(file)
|
| 48 |
-
train, val = data["train"], data["val"]
|
| 49 |
-
else:
|
| 50 |
-
train = self.generate_fewshot_dataset(train, num_shots=num_shots)
|
| 51 |
-
val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
|
| 52 |
-
data = {"train": train, "val": val}
|
| 53 |
-
print(f"Saving preprocessed few-shot data to {preprocessed}")
|
| 54 |
-
with open(preprocessed, "wb") as file:
|
| 55 |
-
pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)
|
| 56 |
-
|
| 57 |
-
subsample = subsample_classes
|
| 58 |
-
train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample)
|
| 59 |
-
|
| 60 |
-
self.templates = [
|
| 61 |
-
lambda c: f"a centered satellite photo of {c}.",
|
| 62 |
-
lambda c: f"a centered satellite photo of a {c}.",
|
| 63 |
-
lambda c: f"a centered satellite photo of the {c}.",
|
| 64 |
-
]
|
| 65 |
-
|
| 66 |
-
super().__init__(train_x=train, val=val, test=test)
|
| 67 |
-
|
| 68 |
-
def update_classname(self, dataset_old):
|
| 69 |
-
dataset_new = []
|
| 70 |
-
for item_old in dataset_old:
|
| 71 |
-
cname_old = item_old.classname
|
| 72 |
-
cname_new = NEW_CLASSNAMES[cname_old]
|
| 73 |
-
item_new = Datum(impath=item_old.impath, label=item_old.label, classname=cname_new)
|
| 74 |
-
dataset_new.append(item_new)
|
| 75 |
-
return dataset_new
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/MTIL_datasets/fer2013.py
DELETED
|
@@ -1,170 +0,0 @@
|
|
| 1 |
-
import os
|
| 2 |
-
import pickle
|
| 3 |
-
import random
|
| 4 |
-
import warnings
|
| 5 |
-
from collections import defaultdict
|
| 6 |
-
|
| 7 |
-
from .utils import *
|
| 8 |
-
from .oxford_pets import OxfordPets
|
| 9 |
-
|
| 10 |
-
# Synonym sets per class (label index is the list index)
|
| 11 |
-
FER2013_CLASSES_SYNONYMS = [
|
| 12 |
-
['angry'],
|
| 13 |
-
['disgusted', 'disgust'],
|
| 14 |
-
['fearful', 'fear'],
|
| 15 |
-
['happy', 'smiling'],
|
| 16 |
-
['sad', 'depressed'],
|
| 17 |
-
['surprised', 'surprise', 'shocked', 'spooked'],
|
| 18 |
-
['neutral', 'bored'],
|
| 19 |
-
]
|
| 20 |
-
|
| 21 |
-
# Canonical class names are the first synonym in each list
|
| 22 |
-
FER2013_CANONICAL = [syns[0] for syns in FER2013_CLASSES_SYNONYMS]
|
| 23 |
-
|
| 24 |
-
# Prompt templates
|
| 25 |
-
FER2013_TEMPLATES = [
|
| 26 |
-
'a photo of a {} looking face.',
|
| 27 |
-
'a photo of a face showing the emotion: {}.',
|
| 28 |
-
'a photo of a face looking {}.',
|
| 29 |
-
'a face that looks {}.',
|
| 30 |
-
'they look {}.',
|
| 31 |
-
'look at how {} they are.',
|
| 32 |
-
]
|
| 33 |
-
|
| 34 |
-
FER2013_DEBUG = os.environ.get("FER2013_DEBUG", "0") not in ("0", "false", "False", "")
|
| 35 |
-
|
| 36 |
-
def _dbg(msg: str):
|
| 37 |
-
if FER2013_DEBUG:
|
| 38 |
-
print(f"[FER2013][DEBUG] {msg}")
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
def _norm(s: str) -> str:
|
| 42 |
-
s = s.lower().strip()
|
| 43 |
-
for ch in [" ", "_", "-", "."]:
|
| 44 |
-
s = s.replace(ch, "")
|
| 45 |
-
return s
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
def _build_syn_map():
|
| 49 |
-
m = {}
|
| 50 |
-
for y, syns in enumerate(FER2013_CLASSES_SYNONYMS):
|
| 51 |
-
for s in syns:
|
| 52 |
-
m[_norm(s)] = y
|
| 53 |
-
# add common canonical variants for safety
|
| 54 |
-
aliases = {
|
| 55 |
-
'disgust': 1,
|
| 56 |
-
'fear': 2,
|
| 57 |
-
'surprise': 5,
|
| 58 |
-
}
|
| 59 |
-
for k, v in aliases.items():
|
| 60 |
-
m[_norm(k)] = v
|
| 61 |
-
return m
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
class FER2013(DatasetBase):
|
| 65 |
-
|
| 66 |
-
dataset_dir = "fer2013"
|
| 67 |
-
|
| 68 |
-
def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'):
|
| 69 |
-
root = os.path.abspath(os.path.expanduser(root))
|
| 70 |
-
self.dataset_dir = os.path.join(root, self.dataset_dir)
|
| 71 |
-
self.split_path = os.path.join(self.dataset_dir, "split_custom_FER2013.json")
|
| 72 |
-
self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot")
|
| 73 |
-
mkdir_if_missing(self.split_fewshot_dir)
|
| 74 |
-
|
| 75 |
-
train_dir = os.path.join(self.dataset_dir, "train")
|
| 76 |
-
test_dir = os.path.join(self.dataset_dir, "test")
|
| 77 |
-
if not os.path.isdir(train_dir) or not os.path.isdir(test_dir):
|
| 78 |
-
raise ValueError(
|
| 79 |
-
f"FER2013: expected train/test folders under '{self.dataset_dir}'. Got train={os.path.isdir(train_dir)}, test={os.path.isdir(test_dir)}"
|
| 80 |
-
)
|
| 81 |
-
|
| 82 |
-
# try cache
|
| 83 |
-
if os.path.exists(self.split_path):
|
| 84 |
-
try:
|
| 85 |
-
train, val, test = OxfordPets.read_split(self.split_path, self.dataset_dir)
|
| 86 |
-
except Exception as e:
|
| 87 |
-
warnings.warn(f"FER2013: failed to read cached split; rebuilding. Error: {e}")
|
| 88 |
-
train, val, test = self._build_split(train_dir, test_dir)
|
| 89 |
-
try:
|
| 90 |
-
OxfordPets.save_split(train, val, test, self.split_path, self.dataset_dir)
|
| 91 |
-
except Exception as e2:
|
| 92 |
-
warnings.warn(f"FER2013: failed to save split: {e2}")
|
| 93 |
-
else:
|
| 94 |
-
train, val, test = self._build_split(train_dir, test_dir)
|
| 95 |
-
try:
|
| 96 |
-
OxfordPets.save_split(train, val, test, self.split_path, self.dataset_dir)
|
| 97 |
-
except Exception as e:
|
| 98 |
-
warnings.warn(f"FER2013: failed to save split: {e}")
|
| 99 |
-
|
| 100 |
-
if num_shots >= 1:
|
| 101 |
-
preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
|
| 102 |
-
if os.path.exists(preprocessed):
|
| 103 |
-
print(f"Loading preprocessed few-shot data from {preprocessed}")
|
| 104 |
-
with open(preprocessed, "rb") as file:
|
| 105 |
-
data = pickle.load(file)
|
| 106 |
-
train, val = data["train"], data["val"]
|
| 107 |
-
else:
|
| 108 |
-
train = self.generate_fewshot_dataset(train, num_shots=num_shots)
|
| 109 |
-
val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
|
| 110 |
-
data = {"train": train, "val": val}
|
| 111 |
-
print(f"Saving preprocessed few-shot data to {preprocessed}")
|
| 112 |
-
with open(preprocessed, "wb") as file:
|
| 113 |
-
pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)
|
| 114 |
-
|
| 115 |
-
train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample_classes)
|
| 116 |
-
self.templates = FER2013_TEMPLATES
|
| 117 |
-
super().__init__(train_x=train, val=val, test=test)
|
| 118 |
-
|
| 119 |
-
def _build_split(self, train_dir, test_dir, p_val=0.2):
|
| 120 |
-
syn_map = _build_syn_map()
|
| 121 |
-
# read train per class
|
| 122 |
-
tr_items_by_label = defaultdict(list)
|
| 123 |
-
class_dirs = listdir_nohidden(train_dir, sort=True)
|
| 124 |
-
if not class_dirs:
|
| 125 |
-
warnings.warn(f"FER2013: no class folders found in {train_dir}")
|
| 126 |
-
for cls in class_dirs:
|
| 127 |
-
full = os.path.join(train_dir, cls)
|
| 128 |
-
if not os.path.isdir(full):
|
| 129 |
-
continue
|
| 130 |
-
key = _norm(cls)
|
| 131 |
-
y = syn_map.get(key)
|
| 132 |
-
if y is None:
|
| 133 |
-
warnings.warn(f"FER2013: unexpected class folder in train: '{cls}'")
|
| 134 |
-
continue
|
| 135 |
-
cname = FER2013_CANONICAL[y]
|
| 136 |
-
for fname in listdir_nohidden(full):
|
| 137 |
-
impath = os.path.join(full, fname)
|
| 138 |
-
tr_items_by_label[y].append(Datum(impath=impath, label=y, classname=cname))
|
| 139 |
-
|
| 140 |
-
# stratified split train->(train,val)
|
| 141 |
-
train, val = [], []
|
| 142 |
-
for y, items in tr_items_by_label.items():
|
| 143 |
-
random.shuffle(items)
|
| 144 |
-
n_val = max(1, round(len(items) * p_val)) if len(items) > 1 else 0
|
| 145 |
-
val.extend(items[:n_val])
|
| 146 |
-
train.extend(items[n_val:])
|
| 147 |
-
|
| 148 |
-
# read test
|
| 149 |
-
test = []
|
| 150 |
-
class_dirs = listdir_nohidden(test_dir, sort=True)
|
| 151 |
-
if not class_dirs:
|
| 152 |
-
warnings.warn(f"FER2013: no class folders found in {test_dir}")
|
| 153 |
-
for cls in class_dirs:
|
| 154 |
-
full = os.path.join(test_dir, cls)
|
| 155 |
-
if not os.path.isdir(full):
|
| 156 |
-
continue
|
| 157 |
-
key = _norm(cls)
|
| 158 |
-
y = syn_map.get(key)
|
| 159 |
-
if y is None:
|
| 160 |
-
warnings.warn(f"FER2013: unexpected class folder in test: '{cls}'")
|
| 161 |
-
continue
|
| 162 |
-
cname = FER2013_CANONICAL[y]
|
| 163 |
-
for fname in listdir_nohidden(full):
|
| 164 |
-
impath = os.path.join(full, fname)
|
| 165 |
-
test.append(Datum(impath=impath, label=y, classname=cname))
|
| 166 |
-
|
| 167 |
-
# basic sanity
|
| 168 |
-
if not train or not val or not test:
|
| 169 |
-
warnings.warn(f"FER2013: split sizes train={len(train)} val={len(val)} test={len(test)}")
|
| 170 |
-
return train, val, test
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/MTIL_datasets/fgvc_aircraft.py
DELETED
|
@@ -1,73 +0,0 @@
|
|
| 1 |
-
import os
|
| 2 |
-
import pickle
|
| 3 |
-
|
| 4 |
-
from .utils import *
|
| 5 |
-
|
| 6 |
-
from .oxford_pets import OxfordPets
|
| 7 |
-
import torchvision.datasets
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
class FGVCAircraft(DatasetBase):
|
| 11 |
-
|
| 12 |
-
dataset_dir = "fgvc_aircraft"
|
| 13 |
-
|
| 14 |
-
def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'):
|
| 15 |
-
root = os.path.abspath(os.path.expanduser(root))
|
| 16 |
-
self.dataset_dir = os.path.join(root, self.dataset_dir)
|
| 17 |
-
self.image_dir = os.path.join(self.dataset_dir, "images")
|
| 18 |
-
self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot")
|
| 19 |
-
mkdir_if_missing(self.split_fewshot_dir)
|
| 20 |
-
|
| 21 |
-
classnames = []
|
| 22 |
-
with open(os.path.join(self.dataset_dir, "variants.txt"), "r") as f:
|
| 23 |
-
lines = f.readlines()
|
| 24 |
-
for line in lines:
|
| 25 |
-
classnames.append(line.strip())
|
| 26 |
-
cname2lab = {c: i for i, c in enumerate(classnames)}
|
| 27 |
-
|
| 28 |
-
train = self.read_data(cname2lab, "images_variant_train.txt")
|
| 29 |
-
val = self.read_data(cname2lab, "images_variant_val.txt")
|
| 30 |
-
test = self.read_data(cname2lab, "images_variant_test.txt")
|
| 31 |
-
|
| 32 |
-
if num_shots >= 1:
|
| 33 |
-
preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
|
| 34 |
-
|
| 35 |
-
if os.path.exists(preprocessed):
|
| 36 |
-
print(f"Loading preprocessed few-shot data from {preprocessed}")
|
| 37 |
-
with open(preprocessed, "rb") as file:
|
| 38 |
-
data = pickle.load(file)
|
| 39 |
-
train, val = data["train"], data["val"]
|
| 40 |
-
else:
|
| 41 |
-
train = self.generate_fewshot_dataset(train, num_shots=num_shots)
|
| 42 |
-
val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
|
| 43 |
-
data = {"train": train, "val": val}
|
| 44 |
-
print(f"Saving preprocessed few-shot data to {preprocessed}")
|
| 45 |
-
with open(preprocessed, "wb") as file:
|
| 46 |
-
pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)
|
| 47 |
-
|
| 48 |
-
subsample = subsample_classes
|
| 49 |
-
train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample)
|
| 50 |
-
|
| 51 |
-
self.templates = [
|
| 52 |
-
lambda c: f"a photo of a {c}, a type of aircraft.",
|
| 53 |
-
lambda c: f"a photo of the {c}, a type of aircraft.",
|
| 54 |
-
]
|
| 55 |
-
|
| 56 |
-
super().__init__(train_x=train, val=val, test=test)
|
| 57 |
-
|
| 58 |
-
def read_data(self, cname2lab, split_file):
|
| 59 |
-
filepath = os.path.join(self.dataset_dir, split_file)
|
| 60 |
-
items = []
|
| 61 |
-
|
| 62 |
-
with open(filepath, "r") as f:
|
| 63 |
-
lines = f.readlines()
|
| 64 |
-
for line in lines:
|
| 65 |
-
line = line.strip().split(" ")
|
| 66 |
-
imname = line[0] + ".jpg"
|
| 67 |
-
classname = " ".join(line[1:])
|
| 68 |
-
impath = os.path.join(self.image_dir, imname)
|
| 69 |
-
label = cname2lab[classname]
|
| 70 |
-
item = Datum(impath=impath, label=label, classname=classname)
|
| 71 |
-
items.append(item)
|
| 72 |
-
|
| 73 |
-
return items
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/MTIL_datasets/food101.py
DELETED
|
@@ -1,52 +0,0 @@
|
|
| 1 |
-
import os
|
| 2 |
-
import pickle
|
| 3 |
-
|
| 4 |
-
from .utils import *
|
| 5 |
-
|
| 6 |
-
from .oxford_pets import OxfordPets
|
| 7 |
-
from .dtd import DescribableTextures as DTD
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
class Food101(DatasetBase):
|
| 12 |
-
|
| 13 |
-
dataset_dir = "food-101"
|
| 14 |
-
|
| 15 |
-
def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'):
|
| 16 |
-
root = os.path.abspath(os.path.expanduser(root))
|
| 17 |
-
self.dataset_dir = os.path.join(root, self.dataset_dir)
|
| 18 |
-
self.image_dir = os.path.join(self.dataset_dir, "images")
|
| 19 |
-
self.split_path = os.path.join(self.dataset_dir, "split_zhou_Food101.json")
|
| 20 |
-
self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot")
|
| 21 |
-
mkdir_if_missing(self.split_fewshot_dir)
|
| 22 |
-
|
| 23 |
-
if os.path.exists(self.split_path):
|
| 24 |
-
train, val, test = OxfordPets.read_split(self.split_path, self.image_dir)
|
| 25 |
-
else:
|
| 26 |
-
train, val, test = DTD.read_and_split_data(self.image_dir)
|
| 27 |
-
OxfordPets.save_split(train, val, test, self.split_path, self.image_dir)
|
| 28 |
-
|
| 29 |
-
if num_shots >= 1:
|
| 30 |
-
preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
|
| 31 |
-
|
| 32 |
-
if os.path.exists(preprocessed):
|
| 33 |
-
print(f"Loading preprocessed few-shot data from {preprocessed}")
|
| 34 |
-
with open(preprocessed, "rb") as file:
|
| 35 |
-
data = pickle.load(file)
|
| 36 |
-
train, val = data["train"], data["val"]
|
| 37 |
-
else:
|
| 38 |
-
train = self.generate_fewshot_dataset(train, num_shots=num_shots)
|
| 39 |
-
val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
|
| 40 |
-
data = {"train": train, "val": val}
|
| 41 |
-
print(f"Saving preprocessed few-shot data to {preprocessed}")
|
| 42 |
-
with open(preprocessed, "wb") as file:
|
| 43 |
-
pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)
|
| 44 |
-
|
| 45 |
-
subsample = subsample_classes
|
| 46 |
-
train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample)
|
| 47 |
-
|
| 48 |
-
self.templates = [
|
| 49 |
-
lambda c: f"a photo of a {c}, a type of food.",
|
| 50 |
-
]
|
| 51 |
-
|
| 52 |
-
super().__init__(train_x=train, val=val, test=test)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/MTIL_datasets/gtsrb.py
DELETED
|
@@ -1,202 +0,0 @@
|
|
| 1 |
-
import os
|
| 2 |
-
import pickle
|
| 3 |
-
import random
|
| 4 |
-
import warnings
|
| 5 |
-
|
| 6 |
-
from .utils import *
|
| 7 |
-
from .oxford_pets import OxfordPets
|
| 8 |
-
|
| 9 |
-
# 43 classes for GTSRB with human-readable names
|
| 10 |
-
classes = [
|
| 11 |
-
'red and white circle 20 kph speed limit',
|
| 12 |
-
'red and white circle 30 kph speed limit',
|
| 13 |
-
'red and white circle 50 kph speed limit',
|
| 14 |
-
'red and white circle 60 kph speed limit',
|
| 15 |
-
'red and white circle 70 kph speed limit',
|
| 16 |
-
'red and white circle 80 kph speed limit',
|
| 17 |
-
'end / de-restriction of 80 kph speed limit',
|
| 18 |
-
'red and white circle 100 kph speed limit',
|
| 19 |
-
'red and white circle 120 kph speed limit',
|
| 20 |
-
'red and white circle red car and black car no passing',
|
| 21 |
-
'red and white circle red truck and black car no passing',
|
| 22 |
-
'red and white triangle road intersection warning',
|
| 23 |
-
'white and yellow diamond priority road',
|
| 24 |
-
'red and white upside down triangle yield right-of-way',
|
| 25 |
-
'stop',
|
| 26 |
-
'empty red and white circle',
|
| 27 |
-
'red and white circle no truck entry',
|
| 28 |
-
'red circle with white horizonal stripe no entry',
|
| 29 |
-
'red and white triangle with exclamation mark warning',
|
| 30 |
-
'red and white triangle with black left curve approaching warning',
|
| 31 |
-
'red and white triangle with black right curve approaching warning',
|
| 32 |
-
'red and white triangle with black double curve approaching warning',
|
| 33 |
-
'red and white triangle rough / bumpy road warning',
|
| 34 |
-
'red and white triangle car skidding / slipping warning',
|
| 35 |
-
'red and white triangle with merging / narrow lanes warning',
|
| 36 |
-
'red and white triangle with person digging / construction / road work warning',
|
| 37 |
-
'red and white triangle with traffic light approaching warning',
|
| 38 |
-
'red and white triangle with person walking warning',
|
| 39 |
-
'red and white triangle with child and person walking warning',
|
| 40 |
-
'red and white triangle with bicyle warning',
|
| 41 |
-
'red and white triangle with snowflake / ice warning',
|
| 42 |
-
'red and white triangle with deer warning',
|
| 43 |
-
'white circle with gray strike bar no speed limit',
|
| 44 |
-
'blue circle with white right turn arrow mandatory',
|
| 45 |
-
'blue circle with white left turn arrow mandatory',
|
| 46 |
-
'blue circle with white forward arrow mandatory',
|
| 47 |
-
'blue circle with white forward or right turn arrow mandatory',
|
| 48 |
-
'blue circle with white forward or left turn arrow mandatory',
|
| 49 |
-
'blue circle with white keep right arrow mandatory',
|
| 50 |
-
'blue circle with white keep left arrow mandatory',
|
| 51 |
-
'blue circle with white arrows indicating a traffic circle',
|
| 52 |
-
'white circle with gray strike bar indicating no passing for cars has ended',
|
| 53 |
-
'white circle with gray strike bar indicating no passing for trucks has ended',
|
| 54 |
-
]
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
DEBUG = os.environ.get("GTSRB_DEBUG", "0") not in ("0", "false", "False", "")
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
def _dbg(msg: str):
|
| 61 |
-
if DEBUG:
|
| 62 |
-
print(f"[GTSRB][DEBUG] {msg}")
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
def _is_image_file(name):
|
| 66 |
-
name = name.lower()
|
| 67 |
-
return any(name.endswith(ext) for ext in ['.ppm', '.png', '.jpg', '.jpeg', '.bmp', '.webp'])
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
class GTSRB(DatasetBase):
|
| 71 |
-
"""
|
| 72 |
-
German Traffic Sign Recognition Benchmark (GTSRB)
|
| 73 |
-
|
| 74 |
-
Expected structure:
|
| 75 |
-
<root>/gtsrb/
|
| 76 |
-
├─ 00000/*.ppm
|
| 77 |
-
├─ 00001/*.ppm
|
| 78 |
-
└─ ... up to 00042/
|
| 79 |
-
|
| 80 |
-
Note: The dataset has no official test split in this layout, so we will
|
| 81 |
-
randomly split each class into train/val/test (50%/20%/30%), cached to json.
|
| 82 |
-
"""
|
| 83 |
-
|
| 84 |
-
dataset_dir = "gtsrb"
|
| 85 |
-
|
| 86 |
-
def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'):
|
| 87 |
-
root = os.path.abspath(os.path.expanduser(root))
|
| 88 |
-
self.dataset_dir = os.path.join(root, self.dataset_dir)
|
| 89 |
-
self.image_dir = self.dataset_dir
|
| 90 |
-
self.split_path = os.path.join(self.dataset_dir, "split_custom_GTSRB.json")
|
| 91 |
-
self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot")
|
| 92 |
-
mkdir_if_missing(self.split_fewshot_dir)
|
| 93 |
-
_dbg(f"dataset_dir={self.dataset_dir}")
|
| 94 |
-
_dbg(f"image_dir={self.image_dir}")
|
| 95 |
-
_dbg(f"split_path exists? {os.path.exists(self.split_path)}")
|
| 96 |
-
|
| 97 |
-
if os.path.exists(self.split_path):
|
| 98 |
-
try:
|
| 99 |
-
train, val, test = OxfordPets.read_split(self.split_path, self.image_dir)
|
| 100 |
-
except Exception as e:
|
| 101 |
-
warnings.warn(f"GTSRB: failed to read split file '{self.split_path}'; rebuilding split. Error: {e}")
|
| 102 |
-
train, val, test = self.read_and_split_data(self.image_dir)
|
| 103 |
-
try:
|
| 104 |
-
OxfordPets.save_split(train, val, test, self.split_path, self.image_dir)
|
| 105 |
-
except Exception as e2:
|
| 106 |
-
warnings.warn(f"GTSRB: failed to save rebuilt split to '{self.split_path}': {e2}")
|
| 107 |
-
else:
|
| 108 |
-
train, val, test = self.read_and_split_data(self.image_dir)
|
| 109 |
-
try:
|
| 110 |
-
OxfordPets.save_split(train, val, test, self.split_path, self.image_dir)
|
| 111 |
-
except Exception as e:
|
| 112 |
-
warnings.warn(f"GTSRB: failed to save split to '{self.split_path}': {e}")
|
| 113 |
-
|
| 114 |
-
_dbg(f"loaded counts: train={len(train)}, val={len(val)}, test={len(test)}")
|
| 115 |
-
|
| 116 |
-
if num_shots >= 1:
|
| 117 |
-
preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
|
| 118 |
-
if os.path.exists(preprocessed):
|
| 119 |
-
print(f"Loading preprocessed few-shot data from {preprocessed}")
|
| 120 |
-
with open(preprocessed, "rb") as file:
|
| 121 |
-
data = pickle.load(file)
|
| 122 |
-
train, val = data["train"], data["val"]
|
| 123 |
-
else:
|
| 124 |
-
train = self.generate_fewshot_dataset(train, num_shots=num_shots)
|
| 125 |
-
val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
|
| 126 |
-
data = {"train": train, "val": val}
|
| 127 |
-
print(f"Saving preprocessed few-shot data to {preprocessed}")
|
| 128 |
-
with open(preprocessed, "wb") as file:
|
| 129 |
-
pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)
|
| 130 |
-
_dbg(f"few-shot applied: train={len(train)}, val={len(val)} (shots={num_shots})")
|
| 131 |
-
|
| 132 |
-
# Ensure class subsampling behavior matches others
|
| 133 |
-
train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample_classes)
|
| 134 |
-
_dbg(f"after subsample='{subsample_classes}': train={len(train)}, val={len(val)}, test={len(test)})")
|
| 135 |
-
|
| 136 |
-
# Prompt templates provided by user
|
| 137 |
-
self.templates = [
|
| 138 |
-
lambda c: f'a zoomed in photo of a "{c}" traffic sign.',
|
| 139 |
-
lambda c: f'a centered photo of a "{c}" traffic sign.',
|
| 140 |
-
lambda c: f'a close up photo of a "{c}" traffic sign.',
|
| 141 |
-
]
|
| 142 |
-
|
| 143 |
-
super().__init__(train_x=train, val=val, test=test)
|
| 144 |
-
|
| 145 |
-
@staticmethod
|
| 146 |
-
def read_and_split_data(image_root, p_trn=0.5, p_val=0.2):
|
| 147 |
-
# Discover class directories (e.g., 00000 .. 00042)
|
| 148 |
-
try:
|
| 149 |
-
all_entries = listdir_nohidden(image_root, sort=True)
|
| 150 |
-
except Exception as e:
|
| 151 |
-
warnings.warn(f"GTSRB: failed to list directory '{image_root}': {e}")
|
| 152 |
-
raise
|
| 153 |
-
class_dirs = [d for d in all_entries if os.path.isdir(os.path.join(image_root, d))]
|
| 154 |
-
class_dirs.sort()
|
| 155 |
-
assert len(class_dirs) > 0, f"GTSRB: no class folders found under {image_root}"
|
| 156 |
-
_dbg(f"found {len(class_dirs)} class folders; head={class_dirs[:5]}")
|
| 157 |
-
|
| 158 |
-
# Map sorted class dirs to labels 0..N-1 and names from 'classes'
|
| 159 |
-
if len(class_dirs) != len(classes):
|
| 160 |
-
print(f"Warning: detected {len(class_dirs)} class folders but classes list has {len(classes)} entries. Proceeding with min overlap.")
|
| 161 |
-
num_labels = min(len(class_dirs), len(classes))
|
| 162 |
-
|
| 163 |
-
def _collate(paths, y, cname):
|
| 164 |
-
return [Datum(impath=p, label=y, classname=cname) for p in paths]
|
| 165 |
-
|
| 166 |
-
train, val, test = [], [], []
|
| 167 |
-
for y, cls_dir in enumerate(class_dirs[:num_labels]):
|
| 168 |
-
cname = classes[y]
|
| 169 |
-
cdir = os.path.join(image_root, cls_dir)
|
| 170 |
-
try:
|
| 171 |
-
files = listdir_nohidden(cdir, sort=False)
|
| 172 |
-
except Exception as e:
|
| 173 |
-
warnings.warn(f"GTSRB: failed to list class folder '{cdir}': {e}")
|
| 174 |
-
files = []
|
| 175 |
-
imgs = [os.path.join(cdir, f) for f in files if _is_image_file(f)]
|
| 176 |
-
random.shuffle(imgs)
|
| 177 |
-
n_total = len(imgs)
|
| 178 |
-
if n_total == 0:
|
| 179 |
-
warnings.warn(f"GTSRB: empty or unreadable class folder: {cdir}; skipping this class")
|
| 180 |
-
continue
|
| 181 |
-
if n_total < 5:
|
| 182 |
-
warnings.warn(f"GTSRB: very few images in class '{cls_dir}' (n={n_total}); splits may be unstable")
|
| 183 |
-
n_train = round(n_total * p_trn)
|
| 184 |
-
n_val = round(n_total * p_val)
|
| 185 |
-
n_test = n_total - n_train - n_val
|
| 186 |
-
if not (n_train > 0 and n_val > 0 and n_test > 0):
|
| 187 |
-
warnings.warn(f"GTSRB: split would create empty split for class {cls_dir} (n={n_total}); adjusting strategy to keep at least 1 per split")
|
| 188 |
-
# Fallback: enforce at least 1 per split if possible
|
| 189 |
-
if n_total >= 3:
|
| 190 |
-
n_train, n_val, n_test = 1, 1, n_total - 2
|
| 191 |
-
elif n_total == 2:
|
| 192 |
-
n_train, n_val, n_test = 1, 1, 0
|
| 193 |
-
else: # n_total == 1
|
| 194 |
-
n_train, n_val, n_test = 1, 0, 0
|
| 195 |
-
|
| 196 |
-
train.extend(_collate(imgs[:n_train], y, cname))
|
| 197 |
-
val.extend(_collate(imgs[n_train:n_train + n_val], y, cname))
|
| 198 |
-
test.extend(_collate(imgs[n_train + n_val:], y, cname))
|
| 199 |
-
_dbg(f"class {cls_dir} -> label {y}: total={n_total}, train={n_train}, val={n_val}, test={n_test}")
|
| 200 |
-
|
| 201 |
-
_dbg(f"aggregate sizes: train={len(train)}, val={len(val)}, test={len(test)}")
|
| 202 |
-
return train, val, test
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/MTIL_datasets/hatefulmemes.py
DELETED
|
@@ -1,117 +0,0 @@
|
|
| 1 |
-
import os
|
| 2 |
-
import json
|
| 3 |
-
import pickle
|
| 4 |
-
|
| 5 |
-
from .utils import *
|
| 6 |
-
from .oxford_pets import OxfordPets
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
class HatefulMemes(DatasetBase):
|
| 10 |
-
"""
|
| 11 |
-
Hateful Memes dataset loader (image-only for MTIL).
|
| 12 |
-
|
| 13 |
-
Expected structure:
|
| 14 |
-
<root>/hatefulmemes/
|
| 15 |
-
├─ img/*.png|jpg
|
| 16 |
-
├─ train.jsonl
|
| 17 |
-
├─ dev.jsonl
|
| 18 |
-
└─ test.jsonl
|
| 19 |
-
|
| 20 |
-
JSONL lines example:
|
| 21 |
-
{"id":85362, "img":"img/85362.png", "label":0, "text":"..."}
|
| 22 |
-
|
| 23 |
-
We DO NOT use the 'text' field. Only image path and label are used.
|
| 24 |
-
|
| 25 |
-
Classes: ['meme', 'hatespeech meme']
|
| 26 |
-
Templates: ['a {}.']
|
| 27 |
-
"""
|
| 28 |
-
|
| 29 |
-
dataset_dir = "hatefulmemes"
|
| 30 |
-
|
| 31 |
-
def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'):
|
| 32 |
-
root = os.path.abspath(os.path.expanduser(root))
|
| 33 |
-
self.dataset_dir = os.path.join(root, self.dataset_dir)
|
| 34 |
-
if not os.path.isdir(self.dataset_dir):
|
| 35 |
-
# allow alias without trailing 's'
|
| 36 |
-
alt = os.path.join(root, "hatefulmeme")
|
| 37 |
-
if os.path.isdir(alt):
|
| 38 |
-
self.dataset_dir = alt
|
| 39 |
-
else:
|
| 40 |
-
raise ValueError(
|
| 41 |
-
f"HatefulMemes: dataset folder not found at '{self.dataset_dir}' or '{alt}'"
|
| 42 |
-
)
|
| 43 |
-
|
| 44 |
-
split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot")
|
| 45 |
-
mkdir_if_missing(split_fewshot_dir)
|
| 46 |
-
|
| 47 |
-
# paths
|
| 48 |
-
train_json = os.path.join(self.dataset_dir, "train.jsonl")
|
| 49 |
-
dev_json = os.path.join(self.dataset_dir, "dev.jsonl")
|
| 50 |
-
test_json = os.path.join(self.dataset_dir, "test.jsonl")
|
| 51 |
-
|
| 52 |
-
if not os.path.isfile(train_json) or not os.path.isfile(dev_json) or not os.path.isfile(test_json):
|
| 53 |
-
raise ValueError(
|
| 54 |
-
f"HatefulMemes: missing jsonl files. Expected train/dev/test at '{self.dataset_dir}'."
|
| 55 |
-
)
|
| 56 |
-
|
| 57 |
-
# fixed classes
|
| 58 |
-
classes = ["meme", "hatespeech meme"]
|
| 59 |
-
lab_to_name = {0: classes[0], 1: classes[1]}
|
| 60 |
-
|
| 61 |
-
# read splits
|
| 62 |
-
train = self._read_jsonl(train_json, lab_to_name)
|
| 63 |
-
val = self._read_jsonl(dev_json, lab_to_name)
|
| 64 |
-
test = self._read_jsonl(test_json, lab_to_name)
|
| 65 |
-
# Some public releases of Hateful Memes do not include labels for test.jsonl.
|
| 66 |
-
# In that case, fallback to use dev.jsonl for evaluation so zero-shot works.
|
| 67 |
-
if len(test) == 0 and len(val) > 0:
|
| 68 |
-
test = list(val)
|
| 69 |
-
|
| 70 |
-
# few-shot
|
| 71 |
-
if num_shots >= 1:
|
| 72 |
-
preprocessed = os.path.join(split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
|
| 73 |
-
if os.path.exists(preprocessed):
|
| 74 |
-
with open(preprocessed, "rb") as file:
|
| 75 |
-
data = pickle.load(file)
|
| 76 |
-
train, val = data["train"], data["val"]
|
| 77 |
-
else:
|
| 78 |
-
train = self.generate_fewshot_dataset(train, num_shots=num_shots)
|
| 79 |
-
val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
|
| 80 |
-
data = {"train": train, "val": val}
|
| 81 |
-
with open(preprocessed, "wb") as file:
|
| 82 |
-
pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)
|
| 83 |
-
|
| 84 |
-
# subsample behavior consistent with others
|
| 85 |
-
train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample_classes)
|
| 86 |
-
|
| 87 |
-
# templates per user spec
|
| 88 |
-
self.templates = [
|
| 89 |
-
lambda c: f'a {c}.',
|
| 90 |
-
]
|
| 91 |
-
|
| 92 |
-
super().__init__(train_x=train, val=val, test=test)
|
| 93 |
-
|
| 94 |
-
def _read_jsonl(self, filepath, lab_to_name):
|
| 95 |
-
items = []
|
| 96 |
-
with open(filepath, 'r', encoding='utf-8') as f:
|
| 97 |
-
for line in f:
|
| 98 |
-
line = line.strip()
|
| 99 |
-
if not line:
|
| 100 |
-
continue
|
| 101 |
-
try:
|
| 102 |
-
obj = json.loads(line)
|
| 103 |
-
except Exception:
|
| 104 |
-
continue
|
| 105 |
-
# fields: id, img, label, text (ignored)
|
| 106 |
-
img_rel = obj.get('img', '')
|
| 107 |
-
label = obj.get('label', None)
|
| 108 |
-
if img_rel is None or label is None:
|
| 109 |
-
continue
|
| 110 |
-
if label not in (0, 1):
|
| 111 |
-
raise ValueError(f"HatefulMemes: unexpected label {label} in {filepath}")
|
| 112 |
-
impath = os.path.join(self.dataset_dir, img_rel.replace('/', os.sep))
|
| 113 |
-
if not os.path.isfile(impath):
|
| 114 |
-
continue
|
| 115 |
-
cname = lab_to_name[label]
|
| 116 |
-
items.append(Datum(impath=impath, label=label, classname=cname))
|
| 117 |
-
return items
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/MTIL_datasets/imagenet_r.py
DELETED
|
@@ -1,231 +0,0 @@
|
|
| 1 |
-
import os
|
| 2 |
-
import random
|
| 3 |
-
from typing import List, Tuple, Dict
|
| 4 |
-
|
| 5 |
-
from .utils import * # Datum, DatasetBase, listdir_nohidden, mkdir_if_missing
|
| 6 |
-
from .oxford_pets import OxfordPets
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
IMAGENETR_TEMPLATES: List[str] = [
|
| 10 |
-
'a bad photo of a {}.',
|
| 11 |
-
'a photo of many {}.',
|
| 12 |
-
'a sculpture of a {}.',
|
| 13 |
-
'a photo of the hard to see {}.',
|
| 14 |
-
'a low resolution photo of the {}.',
|
| 15 |
-
'a rendering of a {}.',
|
| 16 |
-
'graffiti of a {}.',
|
| 17 |
-
'a bad photo of the {}.',
|
| 18 |
-
'a cropped photo of the {}.',
|
| 19 |
-
'a tattoo of a {}.',
|
| 20 |
-
'the embroidered {}.',
|
| 21 |
-
'a photo of a hard to see {}.',
|
| 22 |
-
'a bright photo of a {}.',
|
| 23 |
-
'a photo of a clean {}.',
|
| 24 |
-
'a photo of a dirty {}.',
|
| 25 |
-
'a dark photo of the {}.',
|
| 26 |
-
'a drawing of a {}.',
|
| 27 |
-
'a photo of my {}.',
|
| 28 |
-
'the plastic {}.',
|
| 29 |
-
'a photo of the cool {}.',
|
| 30 |
-
'a close-up photo of a {}.',
|
| 31 |
-
'a black and white photo of the {}.',
|
| 32 |
-
'a painting of the {}.',
|
| 33 |
-
'a painting of a {}.',
|
| 34 |
-
'a pixelated photo of the {}.',
|
| 35 |
-
'a sculpture of the {}.',
|
| 36 |
-
'a bright photo of the {}.',
|
| 37 |
-
'a cropped photo of a {}.',
|
| 38 |
-
'a plastic {}.',
|
| 39 |
-
'a photo of the dirty {}.',
|
| 40 |
-
'a jpeg corrupted photo of a {}.',
|
| 41 |
-
'a blurry photo of the {}.',
|
| 42 |
-
'a photo of the {}.',
|
| 43 |
-
'a good photo of the {}.',
|
| 44 |
-
'a rendering of the {}.',
|
| 45 |
-
'a {} in a video game.',
|
| 46 |
-
'a photo of one {}.',
|
| 47 |
-
'a doodle of a {}.',
|
| 48 |
-
'a close-up photo of the {}.',
|
| 49 |
-
'a photo of a {}.',
|
| 50 |
-
'the origami {}.',
|
| 51 |
-
'the {} in a video game.',
|
| 52 |
-
'a sketch of a {}.',
|
| 53 |
-
'a doodle of the {}.',
|
| 54 |
-
'a origami {}.',
|
| 55 |
-
'a low resolution photo of a {}.',
|
| 56 |
-
'the toy {}.',
|
| 57 |
-
'a rendition of the {}.',
|
| 58 |
-
'a photo of the clean {}.',
|
| 59 |
-
'a photo of a large {}.',
|
| 60 |
-
'a rendition of a {}.',
|
| 61 |
-
'a photo of a nice {}.',
|
| 62 |
-
'a photo of a weird {}.',
|
| 63 |
-
'a blurry photo of a {}.',
|
| 64 |
-
'a cartoon {}.',
|
| 65 |
-
'art of a {}.',
|
| 66 |
-
'a sketch of the {}.',
|
| 67 |
-
'a embroidered {}.',
|
| 68 |
-
'a pixelated photo of a {}.',
|
| 69 |
-
'itap of the {}.',
|
| 70 |
-
'a jpeg corrupted photo of the {}.',
|
| 71 |
-
'a good photo of a {}.',
|
| 72 |
-
'a plushie {}.',
|
| 73 |
-
'a photo of the nice {}.',
|
| 74 |
-
'a photo of the small {}.',
|
| 75 |
-
'a photo of the weird {}.',
|
| 76 |
-
'the cartoon {}.',
|
| 77 |
-
'art of the {}.',
|
| 78 |
-
'a drawing of the {}.',
|
| 79 |
-
'a photo of the large {}.',
|
| 80 |
-
'a black and white photo of a {}.',
|
| 81 |
-
'the plushie {}.',
|
| 82 |
-
'a dark photo of a {}.',
|
| 83 |
-
'itap of a {}.',
|
| 84 |
-
'graffiti of the {}.',
|
| 85 |
-
'a toy {}.',
|
| 86 |
-
'itap of my {}.',
|
| 87 |
-
'a photo of a cool {}.',
|
| 88 |
-
'a photo of a small {}.',
|
| 89 |
-
'a tattoo of the {}.',
|
| 90 |
-
]
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
class ImageNetR(DatasetBase):
|
| 94 |
-
|
| 95 |
-
dataset_dir = "imagenet-r"
|
| 96 |
-
|
| 97 |
-
def __init__(self, root: str, num_shots: int = 0, seed: int = 1, subsample_classes: str = 'all', test_ratio: float = 0.2):
|
| 98 |
-
"""
|
| 99 |
-
Expect directory structure:
|
| 100 |
-
{root}/imagenet-r/<wnid>/*.jpg|png|jpeg|bmp|webp
|
| 101 |
-
{root}/imagenet-r/classname.txt # lines: "<wnid> <human_readable_name>"
|
| 102 |
-
There is no official test split; we perform a per-class split into train/test, then
|
| 103 |
-
split train into train/val using OxfordPets.split_trainval.
|
| 104 |
-
"""
|
| 105 |
-
rnd = random.Random(seed)
|
| 106 |
-
|
| 107 |
-
root = os.path.abspath(os.path.expanduser(root))
|
| 108 |
-
self.dataset_dir = os.path.join(root, self.dataset_dir)
|
| 109 |
-
self.split_path = os.path.join(self.dataset_dir, "split_custom_ImageNetR.json")
|
| 110 |
-
self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot")
|
| 111 |
-
mkdir_if_missing(self.split_fewshot_dir)
|
| 112 |
-
|
| 113 |
-
# Read class mapping from classname.txt
|
| 114 |
-
class_map_path = os.path.join(self.dataset_dir, "classname.txt")
|
| 115 |
-
if not os.path.exists(class_map_path):
|
| 116 |
-
raise FileNotFoundError(f"Class mapping file not found: {class_map_path}")
|
| 117 |
-
wnids, classnames = self._read_class_map(class_map_path)
|
| 118 |
-
self._wnids = wnids
|
| 119 |
-
self._classnames_ref = classnames
|
| 120 |
-
|
| 121 |
-
# Build or load split
|
| 122 |
-
if os.path.exists(self.split_path):
|
| 123 |
-
train, val, test = OxfordPets.read_split(self.split_path, self.dataset_dir)
|
| 124 |
-
else:
|
| 125 |
-
trainval, test = self._read_from_folders(self.dataset_dir, wnids, classnames, rnd=rnd, test_ratio=test_ratio)
|
| 126 |
-
train, val = self._split_trainval_safe(trainval)
|
| 127 |
-
OxfordPets.save_split(train, val, test, self.split_path, self.dataset_dir)
|
| 128 |
-
|
| 129 |
-
if num_shots >= 1:
|
| 130 |
-
preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
|
| 131 |
-
if os.path.exists(preprocessed):
|
| 132 |
-
print(f"Loading preprocessed few-shot data from {preprocessed}")
|
| 133 |
-
import pickle
|
| 134 |
-
with open(preprocessed, "rb") as file:
|
| 135 |
-
data = pickle.load(file)
|
| 136 |
-
train, val = data["train"], data["val"]
|
| 137 |
-
else:
|
| 138 |
-
train = self.generate_fewshot_dataset(train, num_shots=num_shots)
|
| 139 |
-
val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
|
| 140 |
-
data = {"train": train, "val": val}
|
| 141 |
-
print(f"Saving preprocessed few-shot data to {preprocessed}")
|
| 142 |
-
import pickle
|
| 143 |
-
with open(preprocessed, "wb") as file:
|
| 144 |
-
pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)
|
| 145 |
-
|
| 146 |
-
train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample_classes)
|
| 147 |
-
|
| 148 |
-
# Expose templates
|
| 149 |
-
self.templates = IMAGENETR_TEMPLATES
|
| 150 |
-
|
| 151 |
-
super().__init__(train_x=train, val=val, test=test)
|
| 152 |
-
|
| 153 |
-
@staticmethod
|
| 154 |
-
def _read_class_map(filepath: str) -> Tuple[List[str], List[str]]:
|
| 155 |
-
wnids: List[str] = []
|
| 156 |
-
cnames: List[str] = []
|
| 157 |
-
with open(filepath, 'r', encoding='utf-8') as f:
|
| 158 |
-
for line in f:
|
| 159 |
-
line = line.strip()
|
| 160 |
-
if not line:
|
| 161 |
-
continue
|
| 162 |
-
parts = line.split()
|
| 163 |
-
wnid = parts[0]
|
| 164 |
-
cname = ' '.join(parts[1:]) if len(parts) > 1 else wnid
|
| 165 |
-
wnids.append(wnid)
|
| 166 |
-
cnames.append(cname)
|
| 167 |
-
return wnids, cnames
|
| 168 |
-
|
| 169 |
-
@staticmethod
|
| 170 |
-
def _split_trainval_safe(trainval: List[Datum], p_val: float = 0.2) -> Tuple[List[Datum], List[Datum]]:
|
| 171 |
-
from collections import defaultdict
|
| 172 |
-
tracker: Dict[int, List[int]] = defaultdict(list)
|
| 173 |
-
for idx, item in enumerate(trainval):
|
| 174 |
-
tracker[item.label].append(idx)
|
| 175 |
-
|
| 176 |
-
train, val = [], []
|
| 177 |
-
for label, idxs in tracker.items():
|
| 178 |
-
n = len(idxs)
|
| 179 |
-
if n <= 1:
|
| 180 |
-
for idx in idxs:
|
| 181 |
-
train.append(trainval[idx])
|
| 182 |
-
continue
|
| 183 |
-
n_val = max(1, int(round(n * p_val)))
|
| 184 |
-
if n_val >= n:
|
| 185 |
-
n_val = n - 1
|
| 186 |
-
random.shuffle(idxs)
|
| 187 |
-
for i, idx in enumerate(idxs):
|
| 188 |
-
if i < n_val:
|
| 189 |
-
val.append(trainval[idx])
|
| 190 |
-
else:
|
| 191 |
-
train.append(trainval[idx])
|
| 192 |
-
if len(val) == 0 and len(train) > 0:
|
| 193 |
-
val.append(train[-1])
|
| 194 |
-
train = train[:-1]
|
| 195 |
-
return train, val
|
| 196 |
-
|
| 197 |
-
def _read_from_folders(self, data_dir: str, wnids: List[str], classnames: List[str], rnd: random.Random, test_ratio: float) -> Tuple[List[Datum], List[Datum]]:
|
| 198 |
-
exts = {".png", ".jpg", ".jpeg", ".bmp", ".webp"}
|
| 199 |
-
class_to_label = {wnid: i for i, wnid in enumerate(wnids)}
|
| 200 |
-
label_to_cname = {i: classnames[i] for i in range(len(classnames))}
|
| 201 |
-
|
| 202 |
-
items_by_label: Dict[int, List[Datum]] = {i: [] for i in range(len(wnids))}
|
| 203 |
-
for wnid in wnids:
|
| 204 |
-
cdir = os.path.join(data_dir, wnid)
|
| 205 |
-
if not os.path.isdir(cdir):
|
| 206 |
-
# If a class listed in mapping has no folder, skip gracefully
|
| 207 |
-
continue
|
| 208 |
-
label = class_to_label[wnid]
|
| 209 |
-
cname = label_to_cname[label]
|
| 210 |
-
for fname in listdir_nohidden(cdir, sort=True):
|
| 211 |
-
fext = os.path.splitext(fname)[1].lower()
|
| 212 |
-
if fext not in exts:
|
| 213 |
-
continue
|
| 214 |
-
impath = os.path.join(cdir, fname)
|
| 215 |
-
items_by_label[label].append(Datum(impath=impath, label=label, classname=cname))
|
| 216 |
-
|
| 217 |
-
trainval: List[Datum] = []
|
| 218 |
-
test: List[Datum] = []
|
| 219 |
-
for label, items in items_by_label.items():
|
| 220 |
-
if not items:
|
| 221 |
-
continue
|
| 222 |
-
rnd.shuffle(items)
|
| 223 |
-
if len(items) == 1:
|
| 224 |
-
trainval.extend(items)
|
| 225 |
-
continue
|
| 226 |
-
n_test = max(1, int(round(len(items) * test_ratio)))
|
| 227 |
-
if n_test >= len(items):
|
| 228 |
-
n_test = len(items) - 1
|
| 229 |
-
test.extend(items[:n_test])
|
| 230 |
-
trainval.extend(items[n_test:])
|
| 231 |
-
return trainval, test
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/MTIL_datasets/kitti_distance.py
DELETED
|
@@ -1,228 +0,0 @@
|
|
| 1 |
-
import os
|
| 2 |
-
import random
|
| 3 |
-
from typing import List, Tuple
|
| 4 |
-
|
| 5 |
-
from .utils import * # Datum, DatasetBase, listdir_nohidden, mkdir_if_missing
|
| 6 |
-
from .oxford_pets import OxfordPets
|
| 7 |
-
|
| 8 |
-
# Class names per user specification (ordered)
|
| 9 |
-
KITTI_DISTANCE_CLASSES: List[str] = [
|
| 10 |
-
'a photo i took of a car nearby',
|
| 11 |
-
'a photo i took with a car in the middle distance',
|
| 12 |
-
'a photo i took with a car faraway',
|
| 13 |
-
'a photo i took with no car.',
|
| 14 |
-
]
|
| 15 |
-
|
| 16 |
-
# Keep templates as strings with {}
|
| 17 |
-
KITTI_DISTANCE_TEMPLATES: List[str] = [
|
| 18 |
-
'{}',
|
| 19 |
-
]
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
class KittiDistance(DatasetBase):
|
| 23 |
-
|
| 24 |
-
dataset_dir = "kitti"
|
| 25 |
-
|
| 26 |
-
def __init__(self, root, num_shots=0, seed=1, subsample_classes='all', test_ratio: float = 0.2):
|
| 27 |
-
"""
|
| 28 |
-
Predict the distance category of the closest car from KITTI labels.
|
| 29 |
-
|
| 30 |
-
Expected structure:
|
| 31 |
-
{root}/kitti/
|
| 32 |
-
image/000004.png
|
| 33 |
-
label/000004.txt
|
| 34 |
-
|
| 35 |
-
Label file format: KITTI object detection label lines.
|
| 36 |
-
We retain only lines with type == 'Car'.
|
| 37 |
-
For these, we read the 14th column (index 13, loc_z) as the camera-depth in meters.
|
| 38 |
-
We take the minimum positive loc_z across all 'Car' lines as z_min for the image.
|
| 39 |
-
If there is no valid positive loc_z for 'Car', we assign the 'no car' class.
|
| 40 |
-
|
| 41 |
-
Discretization into classes:
|
| 42 |
-
0: 0 < z_min < 10 -> 'nearby'
|
| 43 |
-
1: 10 <= z_min < 30 -> 'middle distance'
|
| 44 |
-
2: z_min >= 30 -> 'faraway'
|
| 45 |
-
3: no car -> 'no car.'
|
| 46 |
-
|
| 47 |
-
No official test set; we split randomly (per class) into train/test by test_ratio.
|
| 48 |
-
Then we split train into train/val using a safe per-class split.
|
| 49 |
-
Splits are saved to JSON for reproducibility.
|
| 50 |
-
"""
|
| 51 |
-
rnd = random.Random(seed)
|
| 52 |
-
|
| 53 |
-
root = os.path.abspath(os.path.expanduser(root))
|
| 54 |
-
self.dataset_dir = os.path.join(root, self.dataset_dir)
|
| 55 |
-
|
| 56 |
-
# Image and label directories
|
| 57 |
-
image_dir = os.path.join(self.dataset_dir, 'image')
|
| 58 |
-
if not os.path.isdir(image_dir):
|
| 59 |
-
# Graceful fallback to standard KITTI naming variants if provided
|
| 60 |
-
alt1 = os.path.join(self.dataset_dir, 'image_2')
|
| 61 |
-
alt2 = os.path.join(self.dataset_dir, 'images')
|
| 62 |
-
if os.path.isdir(alt1):
|
| 63 |
-
image_dir = alt1
|
| 64 |
-
elif os.path.isdir(alt2):
|
| 65 |
-
image_dir = alt2
|
| 66 |
-
label_dir = os.path.join(self.dataset_dir, 'label')
|
| 67 |
-
if not os.path.isdir(label_dir):
|
| 68 |
-
alt_l1 = os.path.join(self.dataset_dir, 'label_2')
|
| 69 |
-
alt_l2 = os.path.join(self.dataset_dir, 'labels')
|
| 70 |
-
if os.path.isdir(alt_l1):
|
| 71 |
-
label_dir = alt_l1
|
| 72 |
-
elif os.path.isdir(alt_l2):
|
| 73 |
-
label_dir = alt_l2
|
| 74 |
-
|
| 75 |
-
if not os.path.isdir(image_dir):
|
| 76 |
-
raise FileNotFoundError(f"KittiDistance: image dir not found: {image_dir}")
|
| 77 |
-
if not os.path.isdir(label_dir):
|
| 78 |
-
raise FileNotFoundError(f"KittiDistance: label dir not found: {label_dir}")
|
| 79 |
-
|
| 80 |
-
self.image_dir = image_dir
|
| 81 |
-
self.label_dir = label_dir
|
| 82 |
-
|
| 83 |
-
self.split_path = os.path.join(self.dataset_dir, "split_custom_KittiDistance.json")
|
| 84 |
-
self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot")
|
| 85 |
-
mkdir_if_missing(self.split_fewshot_dir)
|
| 86 |
-
|
| 87 |
-
if os.path.exists(self.split_path):
|
| 88 |
-
train, val, test = OxfordPets.read_split(self.split_path, self.image_dir)
|
| 89 |
-
else:
|
| 90 |
-
trainval, test = self._read_and_split(self.image_dir, self.label_dir, rnd=rnd, test_ratio=test_ratio)
|
| 91 |
-
train, val = self._split_trainval_safe(trainval)
|
| 92 |
-
OxfordPets.save_split(train, val, test, self.split_path, self.image_dir)
|
| 93 |
-
|
| 94 |
-
if num_shots >= 1:
|
| 95 |
-
preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
|
| 96 |
-
if os.path.exists(preprocessed):
|
| 97 |
-
print(f"Loading preprocessed few-shot data from {preprocessed}")
|
| 98 |
-
import pickle
|
| 99 |
-
with open(preprocessed, "rb") as file:
|
| 100 |
-
data = pickle.load(file)
|
| 101 |
-
train, val = data["train"], data["val"]
|
| 102 |
-
else:
|
| 103 |
-
train = self.generate_fewshot_dataset(train, num_shots=num_shots)
|
| 104 |
-
val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
|
| 105 |
-
data = {"train": train, "val": val}
|
| 106 |
-
print(f"Saving preprocessed few-shot data to {preprocessed}")
|
| 107 |
-
import pickle
|
| 108 |
-
with open(preprocessed, "wb") as file:
|
| 109 |
-
pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)
|
| 110 |
-
|
| 111 |
-
# Allow class subsampling if requested
|
| 112 |
-
train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample_classes)
|
| 113 |
-
|
| 114 |
-
# Templates per user specification
|
| 115 |
-
self.templates = KITTI_DISTANCE_TEMPLATES
|
| 116 |
-
|
| 117 |
-
super().__init__(train_x=train, val=val, test=test)
|
| 118 |
-
# Override inferred metadata to ensure stable 4-way classification regardless of train label coverage
|
| 119 |
-
self._classnames = KITTI_DISTANCE_CLASSES
|
| 120 |
-
self._lab2cname = {i: c for i, c in enumerate(KITTI_DISTANCE_CLASSES)}
|
| 121 |
-
self._num_classes = len(KITTI_DISTANCE_CLASSES)
|
| 122 |
-
|
| 123 |
-
def _read_and_split(self, image_dir: str, label_dir: str, rnd: random.Random, test_ratio: float) -> Tuple[List[Datum], List[Datum]]:
|
| 124 |
-
exts = {".png", ".jpg", ".jpeg", ".bmp", ".webp"}
|
| 125 |
-
# Collect items with computed labels
|
| 126 |
-
items_by_label = {i: [] for i in range(len(KITTI_DISTANCE_CLASSES))}
|
| 127 |
-
|
| 128 |
-
for fname in listdir_nohidden(image_dir, sort=True):
|
| 129 |
-
fext = os.path.splitext(fname)[1].lower()
|
| 130 |
-
if fext not in exts:
|
| 131 |
-
continue
|
| 132 |
-
impath = os.path.join(image_dir, fname)
|
| 133 |
-
stem = os.path.splitext(fname)[0]
|
| 134 |
-
label_path = os.path.join(label_dir, stem + '.txt')
|
| 135 |
-
label_idx = self._compute_label_from_kitti(label_path)
|
| 136 |
-
cname = KITTI_DISTANCE_CLASSES[label_idx]
|
| 137 |
-
items_by_label[label_idx].append(Datum(impath=impath, label=label_idx, classname=cname))
|
| 138 |
-
|
| 139 |
-
# Per-class balanced split into trainval/test
|
| 140 |
-
trainval, test = [], []
|
| 141 |
-
for label, items in items_by_label.items():
|
| 142 |
-
if not items:
|
| 143 |
-
continue
|
| 144 |
-
rnd.shuffle(items)
|
| 145 |
-
if len(items) == 1:
|
| 146 |
-
trainval.extend(items)
|
| 147 |
-
continue
|
| 148 |
-
n_test = max(1, int(round(len(items) * test_ratio)))
|
| 149 |
-
if n_test >= len(items):
|
| 150 |
-
n_test = len(items) - 1
|
| 151 |
-
test.extend(items[:n_test])
|
| 152 |
-
trainval.extend(items[n_test:])
|
| 153 |
-
|
| 154 |
-
return trainval, test
|
| 155 |
-
|
| 156 |
-
@staticmethod
|
| 157 |
-
def _compute_label_from_kitti(label_path: str) -> int:
|
| 158 |
-
"""
|
| 159 |
-
Parse KITTI label file; focus only on lines with type == 'Car'.
|
| 160 |
-
Extract the 14th column (index 13, loc_z) as meters; consider only positive values.
|
| 161 |
-
If no positive loc_z found -> class 'no car' (index 3).
|
| 162 |
-
|
| 163 |
-
Thresholds:
|
| 164 |
-
0: 0 < z < 10
|
| 165 |
-
1: 10 <= z < 30
|
| 166 |
-
2: z >= 30
|
| 167 |
-
"""
|
| 168 |
-
z_vals = []
|
| 169 |
-
if os.path.isfile(label_path):
|
| 170 |
-
with open(label_path, 'r') as f:
|
| 171 |
-
for line in f:
|
| 172 |
-
line = line.strip()
|
| 173 |
-
if not line:
|
| 174 |
-
continue
|
| 175 |
-
parts = line.split()
|
| 176 |
-
obj_type = parts[0]
|
| 177 |
-
if obj_type != 'Car':
|
| 178 |
-
continue
|
| 179 |
-
if len(parts) < 14:
|
| 180 |
-
# Not a valid KITTI detection line; skip
|
| 181 |
-
continue
|
| 182 |
-
try:
|
| 183 |
-
# parts[13] is loc_z (0-based index), per KITTI format
|
| 184 |
-
z = float(parts[13])
|
| 185 |
-
if z > 0:
|
| 186 |
-
z_vals.append(z)
|
| 187 |
-
except Exception:
|
| 188 |
-
continue
|
| 189 |
-
# Determine class index
|
| 190 |
-
if not z_vals:
|
| 191 |
-
return 3 # no car
|
| 192 |
-
z_min = min(z_vals)
|
| 193 |
-
if z_min < 10:
|
| 194 |
-
return 0
|
| 195 |
-
elif z_min < 30:
|
| 196 |
-
return 1
|
| 197 |
-
else:
|
| 198 |
-
return 2
|
| 199 |
-
|
| 200 |
-
@staticmethod
|
| 201 |
-
def _split_trainval_safe(trainval: List[Datum], p_val: float = 0.2) -> Tuple[List[Datum], List[Datum]]:
|
| 202 |
-
from collections import defaultdict
|
| 203 |
-
tracker = defaultdict(list)
|
| 204 |
-
for idx, item in enumerate(trainval):
|
| 205 |
-
tracker[item.label].append(idx)
|
| 206 |
-
|
| 207 |
-
train, val = [], []
|
| 208 |
-
for label, idxs in tracker.items():
|
| 209 |
-
n = len(idxs)
|
| 210 |
-
if n <= 1:
|
| 211 |
-
for idx in idxs:
|
| 212 |
-
train.append(trainval[idx])
|
| 213 |
-
continue
|
| 214 |
-
n_val = max(1, int(round(n * p_val)))
|
| 215 |
-
if n_val >= n:
|
| 216 |
-
n_val = n - 1
|
| 217 |
-
random.shuffle(idxs)
|
| 218 |
-
for i, idx in enumerate(idxs):
|
| 219 |
-
if i < n_val:
|
| 220 |
-
val.append(trainval[idx])
|
| 221 |
-
else:
|
| 222 |
-
train.append(trainval[idx])
|
| 223 |
-
|
| 224 |
-
if len(val) == 0 and len(train) > 0:
|
| 225 |
-
val.append(train[-1])
|
| 226 |
-
train = train[:-1]
|
| 227 |
-
|
| 228 |
-
return train, val
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/MTIL_datasets/mnist.py
DELETED
|
@@ -1,132 +0,0 @@
|
|
| 1 |
-
import os
|
| 2 |
-
import pickle
|
| 3 |
-
|
| 4 |
-
from .utils import *
|
| 5 |
-
|
| 6 |
-
from .oxford_pets import OxfordPets
|
| 7 |
-
from .dtd import DescribableTextures as DTD
|
| 8 |
-
from .oxford_pets import OxfordPets
|
| 9 |
-
|
| 10 |
-
import torch
|
| 11 |
-
import codecs
|
| 12 |
-
import numpy as np
|
| 13 |
-
import sys
|
| 14 |
-
|
| 15 |
-
classes = [
|
| 16 |
-
"0 - zero",
|
| 17 |
-
"1 - one",
|
| 18 |
-
"2 - two",
|
| 19 |
-
"3 - three",
|
| 20 |
-
"4 - four",
|
| 21 |
-
"5 - five",
|
| 22 |
-
"6 - six",
|
| 23 |
-
"7 - seven",
|
| 24 |
-
"8 - eight",
|
| 25 |
-
"9 - nine",
|
| 26 |
-
]
|
| 27 |
-
|
| 28 |
-
class MNIST(DatasetBase):
|
| 29 |
-
|
| 30 |
-
dataset_dir = "mnist"
|
| 31 |
-
|
| 32 |
-
def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'):
|
| 33 |
-
root = os.path.abspath(os.path.expanduser(root))
|
| 34 |
-
self.dataset_dir = os.path.join(root, self.dataset_dir)
|
| 35 |
-
self.image_dir = self.dataset_dir
|
| 36 |
-
|
| 37 |
-
trainval_image_file = "train-images-idx3-ubyte"
|
| 38 |
-
trainval_data = read_image_file(os.path.join(self.image_dir, trainval_image_file)) # Size([60000, 28, 28]) torch.uint8
|
| 39 |
-
trainval_label_file = "train-labels-idx1-ubyte"
|
| 40 |
-
trainval_targets = read_label_file(os.path.join(self.image_dir, trainval_label_file)) # Size([60000]) torch.int64
|
| 41 |
-
# trainval_names =
|
| 42 |
-
|
| 43 |
-
test_image_file = "t10k-images-idx3-ubyte"
|
| 44 |
-
test_data = read_image_file(os.path.join(self.image_dir, test_image_file))
|
| 45 |
-
test_label_file = "t10k-labels-idx1-ubyte"
|
| 46 |
-
test_targets = read_label_file(os.path.join(self.image_dir, test_label_file))
|
| 47 |
-
|
| 48 |
-
trainval = []
|
| 49 |
-
for idx in range(trainval_data.size(0)):
|
| 50 |
-
item = Datum(impath=Image.fromarray(trainval_data[idx].numpy(), mode="L"),
|
| 51 |
-
label=int(trainval_targets[idx]), classname=classes[trainval_targets[idx]])
|
| 52 |
-
trainval.append(item)
|
| 53 |
-
|
| 54 |
-
test = []
|
| 55 |
-
for idx in range(test_data.size(0)):
|
| 56 |
-
item = Datum(impath=Image.fromarray(test_data[idx].numpy(), mode="L"),
|
| 57 |
-
label=int(test_targets[idx]), classname=classes[test_targets[idx]])
|
| 58 |
-
test.append(item)
|
| 59 |
-
|
| 60 |
-
train, val = OxfordPets.split_trainval(trainval)
|
| 61 |
-
|
| 62 |
-
if num_shots >= 1:
|
| 63 |
-
train = self.generate_fewshot_dataset(train, num_shots=num_shots)
|
| 64 |
-
val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
|
| 65 |
-
|
| 66 |
-
subsample = subsample_classes
|
| 67 |
-
train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample)
|
| 68 |
-
|
| 69 |
-
self.templates = [
|
| 70 |
-
lambda c: f'a photo of the number: "{c}".',
|
| 71 |
-
]
|
| 72 |
-
|
| 73 |
-
super().__init__(train_x=train, val=val, test=test)
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
def _flip_byte_order(t: torch.Tensor) -> torch.Tensor:
|
| 77 |
-
return (
|
| 78 |
-
t.contiguous().view(torch.uint8).view(*t.shape, t.element_size()).flip(-1).view(*t.shape[:-1], -1).view(t.dtype)
|
| 79 |
-
)
|
| 80 |
-
|
| 81 |
-
def get_int(b: bytes) -> int:
|
| 82 |
-
return int(codecs.encode(b, "hex"), 16)
|
| 83 |
-
|
| 84 |
-
SN3_PASCALVINCENT_TYPEMAP = {
|
| 85 |
-
8: torch.uint8,
|
| 86 |
-
9: torch.int8,
|
| 87 |
-
11: torch.int16,
|
| 88 |
-
12: torch.int32,
|
| 89 |
-
13: torch.float32,
|
| 90 |
-
14: torch.float64,
|
| 91 |
-
}
|
| 92 |
-
|
| 93 |
-
def read_sn3_pascalvincent_tensor(path: str, strict: bool = True) -> torch.Tensor:
|
| 94 |
-
"""Read a SN3 file in "Pascal Vincent" format (Lush file 'libidx/idx-io.lsh').
|
| 95 |
-
Argument may be a filename, compressed filename, or file object.
|
| 96 |
-
"""
|
| 97 |
-
# read
|
| 98 |
-
with open(path, "rb") as f:
|
| 99 |
-
data = f.read()
|
| 100 |
-
# parse
|
| 101 |
-
magic = get_int(data[0:4])
|
| 102 |
-
nd = magic % 256
|
| 103 |
-
ty = magic // 256
|
| 104 |
-
assert 1 <= nd <= 3
|
| 105 |
-
assert 8 <= ty <= 14
|
| 106 |
-
torch_type = SN3_PASCALVINCENT_TYPEMAP[ty]
|
| 107 |
-
s = [get_int(data[4 * (i + 1) : 4 * (i + 2)]) for i in range(nd)]
|
| 108 |
-
|
| 109 |
-
parsed = torch.frombuffer(bytearray(data), dtype=torch_type, offset=(4 * (nd + 1)))
|
| 110 |
-
if sys.byteorder == "little" and parsed.element_size() > 1:
|
| 111 |
-
parsed = _flip_byte_order(parsed)
|
| 112 |
-
|
| 113 |
-
assert parsed.shape[0] == np.prod(s) or not strict
|
| 114 |
-
return parsed.view(*s)
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
def read_label_file(path: str) -> torch.Tensor:
|
| 118 |
-
x = read_sn3_pascalvincent_tensor(path, strict=False)
|
| 119 |
-
if x.dtype != torch.uint8:
|
| 120 |
-
raise TypeError(f"x should be of dtype torch.uint8 instead of {x.dtype}")
|
| 121 |
-
if x.ndimension() != 1:
|
| 122 |
-
raise ValueError(f"x should have 1 dimension instead of {x.ndimension()}")
|
| 123 |
-
return x.long()
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
def read_image_file(path: str) -> torch.Tensor:
|
| 127 |
-
x = read_sn3_pascalvincent_tensor(path, strict=False)
|
| 128 |
-
if x.dtype != torch.uint8:
|
| 129 |
-
raise TypeError(f"x should be of dtype torch.uint8 instead of {x.dtype}")
|
| 130 |
-
if x.ndimension() != 3:
|
| 131 |
-
raise ValueError(f"x should have 3 dimension instead of {x.ndimension()}")
|
| 132 |
-
return x
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/MTIL_datasets/oxford_flowers.py
DELETED
|
@@ -1,89 +0,0 @@
|
|
| 1 |
-
import os
|
| 2 |
-
import pickle
|
| 3 |
-
import random
|
| 4 |
-
from scipy.io import loadmat
|
| 5 |
-
from collections import defaultdict
|
| 6 |
-
|
| 7 |
-
from .utils import *
|
| 8 |
-
|
| 9 |
-
from .oxford_pets import OxfordPets
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
class OxfordFlowers(DatasetBase):
|
| 13 |
-
|
| 14 |
-
dataset_dir = "oxford_flowers"
|
| 15 |
-
|
| 16 |
-
def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'):
|
| 17 |
-
root = os.path.abspath(os.path.expanduser(root))
|
| 18 |
-
self.dataset_dir = os.path.join(root, self.dataset_dir)
|
| 19 |
-
self.image_dir = os.path.join(self.dataset_dir, "jpg")
|
| 20 |
-
self.label_file = os.path.join(self.dataset_dir, "imagelabels.mat")
|
| 21 |
-
self.lab2cname_file = os.path.join(self.dataset_dir, "cat_to_name.json")
|
| 22 |
-
self.split_path = os.path.join(self.dataset_dir, "split_zhou_OxfordFlowers.json")
|
| 23 |
-
self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot")
|
| 24 |
-
mkdir_if_missing(self.split_fewshot_dir)
|
| 25 |
-
|
| 26 |
-
if os.path.exists(self.split_path):
|
| 27 |
-
train, val, test = OxfordPets.read_split(self.split_path, self.image_dir)
|
| 28 |
-
else:
|
| 29 |
-
train, val, test = self.read_data()
|
| 30 |
-
OxfordPets.save_split(train, val, test, self.split_path, self.image_dir)
|
| 31 |
-
|
| 32 |
-
if num_shots >= 1:
|
| 33 |
-
preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
|
| 34 |
-
|
| 35 |
-
if os.path.exists(preprocessed):
|
| 36 |
-
print(f"Loading preprocessed few-shot data from {preprocessed}")
|
| 37 |
-
with open(preprocessed, "rb") as file:
|
| 38 |
-
data = pickle.load(file)
|
| 39 |
-
train, val = data["train"], data["val"]
|
| 40 |
-
else:
|
| 41 |
-
train = self.generate_fewshot_dataset(train, num_shots=num_shots)
|
| 42 |
-
val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
|
| 43 |
-
data = {"train": train, "val": val}
|
| 44 |
-
print(f"Saving preprocessed few-shot data to {preprocessed}")
|
| 45 |
-
with open(preprocessed, "wb") as file:
|
| 46 |
-
pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)
|
| 47 |
-
|
| 48 |
-
subsample = subsample_classes
|
| 49 |
-
train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample)
|
| 50 |
-
|
| 51 |
-
self.templates = [
|
| 52 |
-
lambda c: f"a photo of a {c}, a type of flower.",
|
| 53 |
-
]
|
| 54 |
-
|
| 55 |
-
super().__init__(train_x=train, val=val, test=test)
|
| 56 |
-
|
| 57 |
-
def read_data(self):
|
| 58 |
-
tracker = defaultdict(list)
|
| 59 |
-
label_file = loadmat(self.label_file)["labels"][0]
|
| 60 |
-
for i, label in enumerate(label_file):
|
| 61 |
-
imname = f"image_{str(i + 1).zfill(5)}.jpg"
|
| 62 |
-
impath = os.path.join(self.image_dir, imname)
|
| 63 |
-
label = int(label)
|
| 64 |
-
tracker[label].append(impath)
|
| 65 |
-
|
| 66 |
-
print("Splitting data into 50% train, 20% val, and 30% test")
|
| 67 |
-
|
| 68 |
-
def _collate(ims, y, c):
|
| 69 |
-
items = []
|
| 70 |
-
for im in ims:
|
| 71 |
-
item = Datum(impath=im, label=y - 1, classname=c)
|
| 72 |
-
items.append(item)
|
| 73 |
-
return items
|
| 74 |
-
|
| 75 |
-
lab2cname = read_json(self.lab2cname_file)
|
| 76 |
-
train, val, test = [], [], []
|
| 77 |
-
for label, impaths in tracker.items():
|
| 78 |
-
random.shuffle(impaths)
|
| 79 |
-
n_total = len(impaths)
|
| 80 |
-
n_train = round(n_total * 0.5)
|
| 81 |
-
n_val = round(n_total * 0.2)
|
| 82 |
-
n_test = n_total - n_train - n_val
|
| 83 |
-
assert n_train > 0 and n_val > 0 and n_test > 0
|
| 84 |
-
cname = lab2cname[str(label)]
|
| 85 |
-
train.extend(_collate(impaths[:n_train], label, cname))
|
| 86 |
-
val.extend(_collate(impaths[n_train : n_train + n_val], label, cname))
|
| 87 |
-
test.extend(_collate(impaths[n_train + n_val :], label, cname))
|
| 88 |
-
|
| 89 |
-
return train, val, test
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/MTIL_datasets/oxford_pets.py
DELETED
|
@@ -1,176 +0,0 @@
|
|
| 1 |
-
import os
|
| 2 |
-
import pickle
|
| 3 |
-
import math
|
| 4 |
-
import random
|
| 5 |
-
from collections import defaultdict
|
| 6 |
-
|
| 7 |
-
from .utils import *
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
class OxfordPets(DatasetBase):
|
| 11 |
-
|
| 12 |
-
dataset_dir = "oxford_pets"
|
| 13 |
-
|
| 14 |
-
def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'):
|
| 15 |
-
root = os.path.abspath(os.path.expanduser(root))
|
| 16 |
-
self.dataset_dir = os.path.join(root, self.dataset_dir)
|
| 17 |
-
self.image_dir = os.path.join(self.dataset_dir, "images")
|
| 18 |
-
self.anno_dir = os.path.join(self.dataset_dir, "annotations")
|
| 19 |
-
self.split_path = os.path.join(self.dataset_dir, "split_zhou_OxfordPets.json")
|
| 20 |
-
self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot")
|
| 21 |
-
mkdir_if_missing(self.split_fewshot_dir)
|
| 22 |
-
|
| 23 |
-
if os.path.exists(self.split_path):
|
| 24 |
-
train, val, test = self.read_split(self.split_path, self.image_dir)
|
| 25 |
-
else:
|
| 26 |
-
trainval = self.read_data(split_file="trainval.txt")
|
| 27 |
-
test = self.read_data(split_file="test.txt")
|
| 28 |
-
train, val = self.split_trainval(trainval)
|
| 29 |
-
self.save_split(train, val, test, self.split_path, self.image_dir)
|
| 30 |
-
|
| 31 |
-
if num_shots >= 1:
|
| 32 |
-
preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
|
| 33 |
-
|
| 34 |
-
if os.path.exists(preprocessed):
|
| 35 |
-
print(f"Loading preprocessed few-shot data from {preprocessed}")
|
| 36 |
-
with open(preprocessed, "rb") as file:
|
| 37 |
-
data = pickle.load(file)
|
| 38 |
-
train, val = data["train"], data["val"]
|
| 39 |
-
else:
|
| 40 |
-
train = self.generate_fewshot_dataset(train, num_shots=num_shots)
|
| 41 |
-
val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
|
| 42 |
-
data = {"train": train, "val": val}
|
| 43 |
-
print(f"Saving preprocessed few-shot data to {preprocessed}")
|
| 44 |
-
with open(preprocessed, "wb") as file:
|
| 45 |
-
pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)
|
| 46 |
-
|
| 47 |
-
subsample = subsample_classes
|
| 48 |
-
train, val, test = self.subsample_classes(train, val, test, subsample=subsample)
|
| 49 |
-
|
| 50 |
-
self.templates = [
|
| 51 |
-
lambda c: f"a photo of a {c}, a type of pet.",
|
| 52 |
-
]
|
| 53 |
-
|
| 54 |
-
super().__init__(train_x=train, val=val, test=test)
|
| 55 |
-
|
| 56 |
-
def read_data(self, split_file):
|
| 57 |
-
filepath = os.path.join(self.anno_dir, split_file)
|
| 58 |
-
items = []
|
| 59 |
-
|
| 60 |
-
with open(filepath, "r") as f:
|
| 61 |
-
lines = f.readlines()
|
| 62 |
-
for line in lines:
|
| 63 |
-
line = line.strip()
|
| 64 |
-
imname, label, species, _ = line.split(" ")
|
| 65 |
-
breed = imname.split("_")[:-1]
|
| 66 |
-
breed = "_".join(breed)
|
| 67 |
-
breed = breed.lower()
|
| 68 |
-
imname += ".jpg"
|
| 69 |
-
impath = os.path.join(self.image_dir, imname)
|
| 70 |
-
label = int(label) - 1 # convert to 0-based index
|
| 71 |
-
item = Datum(impath=impath, label=label, classname=breed)
|
| 72 |
-
items.append(item)
|
| 73 |
-
|
| 74 |
-
return items
|
| 75 |
-
|
| 76 |
-
@staticmethod
|
| 77 |
-
def split_trainval(trainval, p_val=0.2):
|
| 78 |
-
p_trn = 1 - p_val
|
| 79 |
-
tracker = defaultdict(list)
|
| 80 |
-
for idx, item in enumerate(trainval):
|
| 81 |
-
label = item.label
|
| 82 |
-
tracker[label].append(idx)
|
| 83 |
-
|
| 84 |
-
train, val = [], []
|
| 85 |
-
for label, idxs in tracker.items():
|
| 86 |
-
n_val = round(len(idxs) * p_val)
|
| 87 |
-
assert n_val > 0
|
| 88 |
-
random.shuffle(idxs)
|
| 89 |
-
for n, idx in enumerate(idxs):
|
| 90 |
-
item = trainval[idx]
|
| 91 |
-
if n < n_val:
|
| 92 |
-
val.append(item)
|
| 93 |
-
else:
|
| 94 |
-
train.append(item)
|
| 95 |
-
|
| 96 |
-
return train, val
|
| 97 |
-
|
| 98 |
-
@staticmethod
|
| 99 |
-
def save_split(train, val, test, filepath, path_prefix):
|
| 100 |
-
def _extract(items):
|
| 101 |
-
out = []
|
| 102 |
-
for item in items:
|
| 103 |
-
impath = item.impath
|
| 104 |
-
label = item.label
|
| 105 |
-
classname = item.classname
|
| 106 |
-
impath = impath.replace(path_prefix, "")
|
| 107 |
-
if impath.startswith("/"):
|
| 108 |
-
impath = impath[1:]
|
| 109 |
-
out.append((impath, label, classname))
|
| 110 |
-
return out
|
| 111 |
-
|
| 112 |
-
train = _extract(train)
|
| 113 |
-
val = _extract(val)
|
| 114 |
-
test = _extract(test)
|
| 115 |
-
|
| 116 |
-
split = {"train": train, "val": val, "test": test}
|
| 117 |
-
|
| 118 |
-
write_json(split, filepath)
|
| 119 |
-
print(f"Saved split to {filepath}")
|
| 120 |
-
|
| 121 |
-
@staticmethod
|
| 122 |
-
def read_split(filepath, path_prefix):
|
| 123 |
-
def _convert(items):
|
| 124 |
-
out = []
|
| 125 |
-
for impath, label, classname in items:
|
| 126 |
-
impath = os.path.join(path_prefix, impath)
|
| 127 |
-
item = Datum(impath=impath, label=int(label), classname=classname)
|
| 128 |
-
out.append(item)
|
| 129 |
-
return out
|
| 130 |
-
|
| 131 |
-
print(f"Reading split from {filepath}")
|
| 132 |
-
split = read_json(filepath)
|
| 133 |
-
train = _convert(split["train"])
|
| 134 |
-
val = _convert(split["val"])
|
| 135 |
-
test = _convert(split["test"])
|
| 136 |
-
|
| 137 |
-
return train, val, test
|
| 138 |
-
|
| 139 |
-
@staticmethod
|
| 140 |
-
def subsample_classes(*args, subsample="all"):
|
| 141 |
-
assert subsample in ["all", "base", "new"]
|
| 142 |
-
|
| 143 |
-
if subsample == "all":
|
| 144 |
-
return args
|
| 145 |
-
|
| 146 |
-
dataset = args[0]
|
| 147 |
-
labels = set()
|
| 148 |
-
for item in dataset:
|
| 149 |
-
labels.add(item.label)
|
| 150 |
-
labels = list(labels)
|
| 151 |
-
labels.sort()
|
| 152 |
-
n = len(labels)
|
| 153 |
-
m = math.ceil(n / 2)
|
| 154 |
-
|
| 155 |
-
print(f"SUBSAMPLE {subsample.upper()} CLASSES!")
|
| 156 |
-
if subsample == "base":
|
| 157 |
-
selected = labels[:m]
|
| 158 |
-
else:
|
| 159 |
-
selected = labels[m:]
|
| 160 |
-
relabeler = {y: y_new for y_new, y in enumerate(selected)}
|
| 161 |
-
|
| 162 |
-
output = []
|
| 163 |
-
for dataset in args:
|
| 164 |
-
dataset_new = []
|
| 165 |
-
for item in dataset:
|
| 166 |
-
if item.label not in selected:
|
| 167 |
-
continue
|
| 168 |
-
item_new = Datum(
|
| 169 |
-
impath=item.impath,
|
| 170 |
-
label=relabeler[item.label],
|
| 171 |
-
classname=item.classname
|
| 172 |
-
)
|
| 173 |
-
dataset_new.append(item_new)
|
| 174 |
-
output.append(dataset_new)
|
| 175 |
-
|
| 176 |
-
return output
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/MTIL_datasets/pcam.py
DELETED
|
@@ -1,198 +0,0 @@
|
|
| 1 |
-
import os
|
| 2 |
-
import pickle
|
| 3 |
-
import random
|
| 4 |
-
from typing import List, Tuple
|
| 5 |
-
|
| 6 |
-
import h5py
|
| 7 |
-
import numpy as np
|
| 8 |
-
|
| 9 |
-
from .utils import Datum, DatasetBase, mkdir_if_missing, read_json, write_json
|
| 10 |
-
from .oxford_pets import OxfordPets
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
PCAM_CLASSES = [
|
| 14 |
-
'lymph node tissue without metastatic tumor',
|
| 15 |
-
'metastatic tumor in lymph node tissue',
|
| 16 |
-
]
|
| 17 |
-
|
| 18 |
-
# Multiple domain-specific templates for histopathology microscopy images
|
| 19 |
-
PCAM_TEMPLATES = [
|
| 20 |
-
'a microscopy image patch of {}',
|
| 21 |
-
'a histopathology image of {}',
|
| 22 |
-
'a hematoxylin and eosin stained image of {}',
|
| 23 |
-
'a high-resolution histology patch of {}',
|
| 24 |
-
'a digital pathology slide patch of {}',
|
| 25 |
-
'this is a microscopy image of {}',
|
| 26 |
-
'this is a histopathology image of {}',
|
| 27 |
-
]
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
class PCam(DatasetBase):
|
| 31 |
-
|
| 32 |
-
dataset_dir = 'pcam'
|
| 33 |
-
|
| 34 |
-
def __init__(self, root, num_shots: int = 0, seed: int = 1, subsample_classes: str = 'all', val_ratio: float = 0.2):
|
| 35 |
-
root = os.path.abspath(os.path.expanduser(root))
|
| 36 |
-
self.dataset_dir = os.path.join(root, self.dataset_dir)
|
| 37 |
-
mkdir_if_missing(self.dataset_dir)
|
| 38 |
-
|
| 39 |
-
# HDF5 file paths
|
| 40 |
-
self.train_x_path = os.path.join(self.dataset_dir, 'camelyonpatch_level_2_split_train_x.h5')
|
| 41 |
-
self.train_y_path = os.path.join(self.dataset_dir, 'camelyonpatch_level_2_split_train_y.h5')
|
| 42 |
-
self.test_x_path = os.path.join(self.dataset_dir, 'camelyonpatch_level_2_split_test_x.h5')
|
| 43 |
-
self.test_y_path = os.path.join(self.dataset_dir, 'camelyonpatch_level_2_split_test_y.h5')
|
| 44 |
-
|
| 45 |
-
for p in [self.train_x_path, self.train_y_path, self.test_x_path, self.test_y_path]:
|
| 46 |
-
if not os.path.isfile(p):
|
| 47 |
-
raise FileNotFoundError(f"PCam: expected file not found: {p}")
|
| 48 |
-
|
| 49 |
-
self.split_path = os.path.join(self.dataset_dir, 'split_custom_PCam.json')
|
| 50 |
-
self.split_fewshot_dir = os.path.join(self.dataset_dir, 'split_fewshot')
|
| 51 |
-
mkdir_if_missing(self.split_fewshot_dir)
|
| 52 |
-
|
| 53 |
-
random.seed(seed)
|
| 54 |
-
np.random.seed(seed)
|
| 55 |
-
|
| 56 |
-
if os.path.exists(self.split_path):
|
| 57 |
-
train, val, test = self.read_split(self.split_path)
|
| 58 |
-
else:
|
| 59 |
-
trainval = self._read_train(self.train_x_path, self.train_y_path)
|
| 60 |
-
test = self._read_test(self.test_x_path, self.test_y_path)
|
| 61 |
-
train, val = self.split_trainval(trainval, p_val=val_ratio)
|
| 62 |
-
self.save_split(train, val, test, self.split_path)
|
| 63 |
-
|
| 64 |
-
if num_shots >= 1:
|
| 65 |
-
preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
|
| 66 |
-
if os.path.exists(preprocessed):
|
| 67 |
-
print(f"Loading preprocessed few-shot data from {preprocessed}")
|
| 68 |
-
with open(preprocessed, 'rb') as f:
|
| 69 |
-
data = pickle.load(f)
|
| 70 |
-
train, val = data['train'], data['val']
|
| 71 |
-
else:
|
| 72 |
-
train = self.generate_fewshot_dataset(train, num_shots=num_shots)
|
| 73 |
-
val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
|
| 74 |
-
data = {'train': train, 'val': val}
|
| 75 |
-
print(f"Saving preprocessed few-shot data to {preprocessed}")
|
| 76 |
-
with open(preprocessed, 'wb') as f:
|
| 77 |
-
pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL)
|
| 78 |
-
|
| 79 |
-
# Optional class subsampling (kept for consistency with other datasets)
|
| 80 |
-
train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample_classes)
|
| 81 |
-
|
| 82 |
-
# Templates per user specification
|
| 83 |
-
self.templates = PCAM_TEMPLATES
|
| 84 |
-
|
| 85 |
-
# Debug: print split sizes and label histograms
|
| 86 |
-
def _hist(items):
|
| 87 |
-
from collections import Counter
|
| 88 |
-
cnt = Counter([it.label for it in items])
|
| 89 |
-
# ensure keys present for both binary classes
|
| 90 |
-
out = {i: int(cnt.get(i, 0)) for i in range(len(PCAM_CLASSES))}
|
| 91 |
-
return out
|
| 92 |
-
|
| 93 |
-
super().__init__(train_x=train, val=val, test=test)
|
| 94 |
-
# Ensure stable binary classification metadata
|
| 95 |
-
self._classnames = PCAM_CLASSES
|
| 96 |
-
self._lab2cname = {i: c for i, c in enumerate(PCAM_CLASSES)}
|
| 97 |
-
self._num_classes = len(PCAM_CLASSES)
|
| 98 |
-
|
| 99 |
-
@staticmethod
|
| 100 |
-
def _first_key(h5_path: str) -> str:
|
| 101 |
-
with h5py.File(h5_path, 'r') as f:
|
| 102 |
-
keys = list(f.keys())
|
| 103 |
-
if not keys:
|
| 104 |
-
raise RuntimeError(f"No datasets found in H5 file: {h5_path}")
|
| 105 |
-
return keys[0]
|
| 106 |
-
|
| 107 |
-
@staticmethod
|
| 108 |
-
def _read_labels(h5_path: str) -> np.ndarray:
|
| 109 |
-
key = PCam._first_key(h5_path)
|
| 110 |
-
with h5py.File(h5_path, 'r') as f:
|
| 111 |
-
y = f[key][...]
|
| 112 |
-
y = np.asarray(y).squeeze()
|
| 113 |
-
y = y.astype(np.int64)
|
| 114 |
-
return y
|
| 115 |
-
|
| 116 |
-
def _read_train(self, x_path: str, y_path: str) -> List[Datum]:
|
| 117 |
-
x_key = self._first_key(x_path)
|
| 118 |
-
y = self._read_labels(y_path)
|
| 119 |
-
items: List[Datum] = []
|
| 120 |
-
for i, label in enumerate(y.tolist()):
|
| 121 |
-
label_i = int(label)
|
| 122 |
-
classname = PCAM_CLASSES[label_i]
|
| 123 |
-
# Lazy image reference: ('h5', abs_h5_path, dataset_key, index)
|
| 124 |
-
abs_path = os.path.abspath(x_path)
|
| 125 |
-
impath = ('h5', abs_path, x_key, i)
|
| 126 |
-
items.append(Datum(impath=impath, label=label_i, classname=classname))
|
| 127 |
-
return items
|
| 128 |
-
|
| 129 |
-
def _read_test(self, x_path: str, y_path: str) -> List[Datum]:
|
| 130 |
-
x_key = self._first_key(x_path)
|
| 131 |
-
y = self._read_labels(y_path)
|
| 132 |
-
items: List[Datum] = []
|
| 133 |
-
for i, label in enumerate(y.tolist()):
|
| 134 |
-
label_i = int(label)
|
| 135 |
-
classname = PCAM_CLASSES[label_i]
|
| 136 |
-
abs_path = os.path.abspath(x_path)
|
| 137 |
-
impath = ('h5', abs_path, x_key, i)
|
| 138 |
-
items.append(Datum(impath=impath, label=label_i, classname=classname))
|
| 139 |
-
return items
|
| 140 |
-
|
| 141 |
-
@staticmethod
|
| 142 |
-
def split_trainval(trainval: List[Datum], p_val: float = 0.2) -> Tuple[List[Datum], List[Datum]]:
|
| 143 |
-
from collections import defaultdict
|
| 144 |
-
p_trn = 1 - p_val
|
| 145 |
-
print(f"Splitting PCam train into {p_trn:.0%} train and {p_val:.0%} val")
|
| 146 |
-
tracker = defaultdict(list)
|
| 147 |
-
for idx, item in enumerate(trainval):
|
| 148 |
-
tracker[item.label].append(idx)
|
| 149 |
-
train, val = [], []
|
| 150 |
-
for _, idxs in tracker.items():
|
| 151 |
-
n_val = max(1, round(len(idxs) * p_val))
|
| 152 |
-
random.shuffle(idxs)
|
| 153 |
-
for n, i in enumerate(idxs):
|
| 154 |
-
if n < n_val:
|
| 155 |
-
val.append(trainval[i])
|
| 156 |
-
else:
|
| 157 |
-
train.append(trainval[i])
|
| 158 |
-
return train, val
|
| 159 |
-
|
| 160 |
-
def save_split(self, train: List[Datum], val: List[Datum], test: List[Datum], filepath: str):
|
| 161 |
-
def _ser(items: List[Datum]):
|
| 162 |
-
out = []
|
| 163 |
-
for it in items:
|
| 164 |
-
impath = it.impath
|
| 165 |
-
if isinstance(impath, tuple) and len(impath) == 4 and impath[0] == 'h5':
|
| 166 |
-
tag, abs_path, key, idx = impath
|
| 167 |
-
# store relative path for portability
|
| 168 |
-
rel = os.path.relpath(abs_path, self.dataset_dir)
|
| 169 |
-
impath_ser = [tag, rel, key, int(idx)]
|
| 170 |
-
else:
|
| 171 |
-
raise ValueError('PCam expects H5 tuple paths')
|
| 172 |
-
out.append((impath_ser, int(it.label), it.classname))
|
| 173 |
-
return out
|
| 174 |
-
split = {
|
| 175 |
-
'train': _ser(train),
|
| 176 |
-
'val': _ser(val),
|
| 177 |
-
'test': _ser(test),
|
| 178 |
-
}
|
| 179 |
-
write_json(split, filepath)
|
| 180 |
-
print(f"Saved PCam split to {filepath}")
|
| 181 |
-
|
| 182 |
-
def read_split(self, filepath: str):
|
| 183 |
-
def _deser(items):
|
| 184 |
-
out = []
|
| 185 |
-
for impath_ser, label, classname in items:
|
| 186 |
-
if isinstance(impath_ser, (list, tuple)) and len(impath_ser) == 4 and impath_ser[0] == 'h5':
|
| 187 |
-
tag, rel, key, idx = impath_ser
|
| 188 |
-
fpath = os.path.join(self.dataset_dir, rel)
|
| 189 |
-
impath = (tag, fpath, key, int(idx))
|
| 190 |
-
else:
|
| 191 |
-
raise ValueError('PCam split contains invalid path entries')
|
| 192 |
-
out.append(Datum(impath=impath, label=int(label), classname=classname))
|
| 193 |
-
return out
|
| 194 |
-
split = read_json(filepath)
|
| 195 |
-
train = _deser(split['train'])
|
| 196 |
-
val = _deser(split['val'])
|
| 197 |
-
test = _deser(split['test'])
|
| 198 |
-
return train, val, test
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/MTIL_datasets/resisc.py
DELETED
|
@@ -1,199 +0,0 @@
|
|
| 1 |
-
import os
|
| 2 |
-
import pickle
|
| 3 |
-
import random
|
| 4 |
-
import warnings
|
| 5 |
-
|
| 6 |
-
from .utils import *
|
| 7 |
-
from .oxford_pets import OxfordPets
|
| 8 |
-
|
| 9 |
-
# Canonical class list for RESISC45 (order defines label ids 0..44)
|
| 10 |
-
RESISC_CLASSES = [
|
| 11 |
-
'airplane',
|
| 12 |
-
'airport',
|
| 13 |
-
'baseball diamond',
|
| 14 |
-
'basketball court',
|
| 15 |
-
'beach',
|
| 16 |
-
'bridge',
|
| 17 |
-
'chaparral',
|
| 18 |
-
'church',
|
| 19 |
-
'circular farmland',
|
| 20 |
-
'cloud',
|
| 21 |
-
'commercial area',
|
| 22 |
-
'dense residential',
|
| 23 |
-
'desert',
|
| 24 |
-
'forest',
|
| 25 |
-
'freeway',
|
| 26 |
-
'golf course',
|
| 27 |
-
'ground track field',
|
| 28 |
-
'harbor',
|
| 29 |
-
'industrial area',
|
| 30 |
-
'intersection',
|
| 31 |
-
'island',
|
| 32 |
-
'lake',
|
| 33 |
-
'meadow',
|
| 34 |
-
'medium residential',
|
| 35 |
-
'mobile home park',
|
| 36 |
-
'mountain',
|
| 37 |
-
'overpass',
|
| 38 |
-
'palace',
|
| 39 |
-
'parking lot',
|
| 40 |
-
'railway',
|
| 41 |
-
'railway station',
|
| 42 |
-
'rectangular farmland',
|
| 43 |
-
'river',
|
| 44 |
-
'roundabout',
|
| 45 |
-
'runway',
|
| 46 |
-
'sea ice',
|
| 47 |
-
'ship',
|
| 48 |
-
'snowberg',
|
| 49 |
-
'sparse residential',
|
| 50 |
-
'stadium',
|
| 51 |
-
'storage tank',
|
| 52 |
-
'tennis court',
|
| 53 |
-
'terrace',
|
| 54 |
-
'thermal power station',
|
| 55 |
-
'wetland',
|
| 56 |
-
]
|
| 57 |
-
|
| 58 |
-
# Prompt templates (strings) as provided
|
| 59 |
-
RESISC_TEMPLATES = [
|
| 60 |
-
'satellite imagery of {}.',
|
| 61 |
-
'aerial imagery of {}.',
|
| 62 |
-
'satellite photo of {}.',
|
| 63 |
-
'aerial photo of {}.',
|
| 64 |
-
'satellite view of {}.',
|
| 65 |
-
'aerial view of {}.',
|
| 66 |
-
'satellite imagery of a {}.',
|
| 67 |
-
'aerial imagery of a {}.',
|
| 68 |
-
'satellite photo of a {}.',
|
| 69 |
-
'aerial photo of a {}.',
|
| 70 |
-
'satellite view of a {}.',
|
| 71 |
-
'aerial view of a {}.',
|
| 72 |
-
'satellite imagery of the {}.',
|
| 73 |
-
'aerial imagery of the {}.',
|
| 74 |
-
'satellite photo of the {}.',
|
| 75 |
-
'aerial photo of the {}.',
|
| 76 |
-
'satellite view of the {}.',
|
| 77 |
-
'aerial view of the {}.',
|
| 78 |
-
]
|
| 79 |
-
|
| 80 |
-
RESISC_DEBUG = os.environ.get("RESISC_DEBUG", "0") not in ("0", "false", "False", "")
|
| 81 |
-
|
| 82 |
-
def _dbg(msg: str):
|
| 83 |
-
if RESISC_DEBUG:
|
| 84 |
-
print(f"[RESISC45][DEBUG] {msg}")
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
def _norm_name(s: str) -> str:
|
| 88 |
-
s = s.lower().strip()
|
| 89 |
-
for ch in [" ", "_", "-", "."]:
|
| 90 |
-
s = s.replace(ch, "")
|
| 91 |
-
return s
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
class RESISC45(DatasetBase):
|
| 95 |
-
|
| 96 |
-
dataset_dir = "resisc45"
|
| 97 |
-
|
| 98 |
-
def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'):
|
| 99 |
-
root = os.path.abspath(os.path.expanduser(root))
|
| 100 |
-
self.dataset_dir = os.path.join(root, self.dataset_dir)
|
| 101 |
-
self.image_dir = self.dataset_dir # images are under class subfolders directly
|
| 102 |
-
self.split_path = os.path.join(self.dataset_dir, "split_custom_RESISC45.json")
|
| 103 |
-
self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot")
|
| 104 |
-
mkdir_if_missing(self.split_fewshot_dir)
|
| 105 |
-
_dbg(f"dataset_dir={self.dataset_dir}")
|
| 106 |
-
|
| 107 |
-
if os.path.exists(self.split_path):
|
| 108 |
-
try:
|
| 109 |
-
train, val, test = OxfordPets.read_split(self.split_path, self.image_dir)
|
| 110 |
-
except Exception as e:
|
| 111 |
-
warnings.warn(f"RESISC45: failed to read split file '{self.split_path}'; rebuilding. Error: {e}")
|
| 112 |
-
train, val, test = self.read_and_split_data(self.image_dir)
|
| 113 |
-
try:
|
| 114 |
-
OxfordPets.save_split(train, val, test, self.split_path, self.image_dir)
|
| 115 |
-
except Exception as e2:
|
| 116 |
-
warnings.warn(f"RESISC45: failed to save rebuilt split to '{self.split_path}': {e2}")
|
| 117 |
-
else:
|
| 118 |
-
train, val, test = self.read_and_split_data(self.image_dir)
|
| 119 |
-
try:
|
| 120 |
-
OxfordPets.save_split(train, val, test, self.split_path, self.image_dir)
|
| 121 |
-
except Exception as e:
|
| 122 |
-
warnings.warn(f"RESISC45: failed to save split to '{self.split_path}': {e}")
|
| 123 |
-
|
| 124 |
-
if num_shots >= 1:
|
| 125 |
-
preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
|
| 126 |
-
if os.path.exists(preprocessed):
|
| 127 |
-
print(f"Loading preprocessed few-shot data from {preprocessed}")
|
| 128 |
-
with open(preprocessed, "rb") as file:
|
| 129 |
-
data = pickle.load(file)
|
| 130 |
-
train, val = data["train"], data["val"]
|
| 131 |
-
else:
|
| 132 |
-
train = self.generate_fewshot_dataset(train, num_shots=num_shots)
|
| 133 |
-
val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
|
| 134 |
-
data = {"train": train, "val": val}
|
| 135 |
-
print(f"Saving preprocessed few-shot data to {preprocessed}")
|
| 136 |
-
with open(preprocessed, "wb") as file:
|
| 137 |
-
pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)
|
| 138 |
-
|
| 139 |
-
train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample_classes)
|
| 140 |
-
self.templates = RESISC_TEMPLATES
|
| 141 |
-
super().__init__(train_x=train, val=val, test=test)
|
| 142 |
-
|
| 143 |
-
@staticmethod
|
| 144 |
-
def read_and_split_data(image_dir, p_trn=0.5, p_val=0.2):
|
| 145 |
-
try:
|
| 146 |
-
categories = listdir_nohidden(image_dir, sort=True)
|
| 147 |
-
except Exception as e:
|
| 148 |
-
warnings.warn(f"RESISC45: failed to list directory '{image_dir}': {e}")
|
| 149 |
-
raise
|
| 150 |
-
categories = [c for c in categories if os.path.isdir(os.path.join(image_dir, c))]
|
| 151 |
-
cat_norm = {_norm_name(c): c for c in categories}
|
| 152 |
-
|
| 153 |
-
missing = []
|
| 154 |
-
class_to_dir = {}
|
| 155 |
-
for cname in RESISC_CLASSES:
|
| 156 |
-
key = _norm_name(cname)
|
| 157 |
-
if key in cat_norm:
|
| 158 |
-
class_to_dir[cname] = cat_norm[key]
|
| 159 |
-
else:
|
| 160 |
-
missing.append(cname)
|
| 161 |
-
if missing:
|
| 162 |
-
warnings.warn(f"RESISC45: missing class folders for {len(missing)} classes: {missing[:5]}{' ...' if len(missing)>5 else ''}")
|
| 163 |
-
|
| 164 |
-
def _collate(paths, y, cname):
|
| 165 |
-
return [Datum(impath=p, label=y, classname=cname) for p in paths]
|
| 166 |
-
|
| 167 |
-
train, val, test = [], [], []
|
| 168 |
-
for y, cname in enumerate(RESISC_CLASSES):
|
| 169 |
-
if cname not in class_to_dir:
|
| 170 |
-
continue
|
| 171 |
-
cdir = os.path.join(image_dir, class_to_dir[cname])
|
| 172 |
-
try:
|
| 173 |
-
images = listdir_nohidden(cdir, sort=False)
|
| 174 |
-
except Exception as e:
|
| 175 |
-
warnings.warn(f"RESISC45: failed to list class folder '{cdir}': {e}")
|
| 176 |
-
images = []
|
| 177 |
-
images = [os.path.join(cdir, im) for im in images]
|
| 178 |
-
random.shuffle(images)
|
| 179 |
-
n_total = len(images)
|
| 180 |
-
if n_total == 0:
|
| 181 |
-
warnings.warn(f"RESISC45: empty class folder {cdir}; skipping")
|
| 182 |
-
continue
|
| 183 |
-
n_train = round(n_total * p_trn)
|
| 184 |
-
n_val = round(n_total * p_val)
|
| 185 |
-
n_test = n_total - n_train - n_val
|
| 186 |
-
if not (n_train > 0 and n_val > 0 and n_test > 0):
|
| 187 |
-
# Fallback to keep all splits non-empty where possible
|
| 188 |
-
if n_total >= 3:
|
| 189 |
-
n_train, n_val, n_test = 1, 1, n_total - 2
|
| 190 |
-
elif n_total == 2:
|
| 191 |
-
n_train, n_val, n_test = 1, 1, 0
|
| 192 |
-
else: # 1
|
| 193 |
-
n_train, n_val, n_test = 1, 0, 0
|
| 194 |
-
|
| 195 |
-
train.extend(_collate(images[:n_train], y, cname))
|
| 196 |
-
val.extend(_collate(images[n_train:n_train + n_val], y, cname))
|
| 197 |
-
test.extend(_collate(images[n_train + n_val:], y, cname))
|
| 198 |
-
|
| 199 |
-
return train, val, test
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/MTIL_datasets/sst2.py
DELETED
|
@@ -1,122 +0,0 @@
|
|
| 1 |
-
import os
|
| 2 |
-
import pickle
|
| 3 |
-
|
| 4 |
-
from .utils import *
|
| 5 |
-
from .oxford_pets import OxfordPets
|
| 6 |
-
from .dtd import DescribableTextures as DTD
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
class SST2(DatasetBase):
|
| 10 |
-
"""
|
| 11 |
-
SST2 (rendered) dataset loader for MTIL.
|
| 12 |
-
|
| 13 |
-
Expected structure (either of the following roots):
|
| 14 |
-
<root>/sst2/
|
| 15 |
-
├─ train/{negative,positive}/*.png
|
| 16 |
-
├─ valid/{negative,positive}/*.png
|
| 17 |
-
└─ test/{negative,positive}/*.png
|
| 18 |
-
|
| 19 |
-
<root>/rendered-sst2/ (alias supported)
|
| 20 |
-
├─ train/{negative,positive}/*.png
|
| 21 |
-
├─ valid/{negative,positive}/*.png
|
| 22 |
-
└─ test/{negative,positive}/*.png
|
| 23 |
-
|
| 24 |
-
Classes: ['negative', 'positive']
|
| 25 |
-
Templates: ['a {} review of a movie.']
|
| 26 |
-
"""
|
| 27 |
-
|
| 28 |
-
dataset_dir = "sst2"
|
| 29 |
-
|
| 30 |
-
def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'):
|
| 31 |
-
root = os.path.abspath(os.path.expanduser(root))
|
| 32 |
-
primary_dir = os.path.join(root, self.dataset_dir)
|
| 33 |
-
alt_dir = os.path.join(root, "rendered-sst2")
|
| 34 |
-
|
| 35 |
-
if os.path.isdir(primary_dir):
|
| 36 |
-
self.dataset_dir = primary_dir
|
| 37 |
-
elif os.path.isdir(alt_dir):
|
| 38 |
-
self.dataset_dir = alt_dir
|
| 39 |
-
else:
|
| 40 |
-
raise ValueError(
|
| 41 |
-
"SST2: dataset folder not found. Expected one of: '{}' or '{}'".format(primary_dir, alt_dir)
|
| 42 |
-
)
|
| 43 |
-
|
| 44 |
-
self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot")
|
| 45 |
-
mkdir_if_missing(self.split_fewshot_dir)
|
| 46 |
-
|
| 47 |
-
train_dir = os.path.join(self.dataset_dir, "train")
|
| 48 |
-
valid_dir = os.path.join(self.dataset_dir, "valid")
|
| 49 |
-
test_dir = os.path.join(self.dataset_dir, "test")
|
| 50 |
-
|
| 51 |
-
use_folder_splits = os.path.isdir(train_dir) and os.path.isdir(valid_dir) and os.path.isdir(test_dir)
|
| 52 |
-
|
| 53 |
-
# fixed class order and validation
|
| 54 |
-
classes = ["negative", "positive"]
|
| 55 |
-
class_to_label = {c: i for i, c in enumerate(classes)}
|
| 56 |
-
|
| 57 |
-
if use_folder_splits:
|
| 58 |
-
train = self._read_split_dir(train_dir, class_to_label)
|
| 59 |
-
val = self._read_split_dir(valid_dir, class_to_label)
|
| 60 |
-
test = self._read_split_dir(test_dir, class_to_label)
|
| 61 |
-
else:
|
| 62 |
-
# Fallbacks consistent with other datasets: look for a JSON split, else naive split
|
| 63 |
-
image_dir = os.path.join(self.dataset_dir, "images")
|
| 64 |
-
self.image_dir = image_dir if os.path.isdir(image_dir) else self.dataset_dir
|
| 65 |
-
split_path = os.path.join(self.dataset_dir, "split_zhou_SST2.json")
|
| 66 |
-
if os.path.exists(split_path):
|
| 67 |
-
train, val, test = OxfordPets.read_split(split_path, self.image_dir)
|
| 68 |
-
else:
|
| 69 |
-
train, val, test = DTD.read_and_split_data(self.image_dir)
|
| 70 |
-
OxfordPets.save_split(train, val, test, split_path, self.image_dir)
|
| 71 |
-
|
| 72 |
-
if num_shots >= 1:
|
| 73 |
-
preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
|
| 74 |
-
if os.path.exists(preprocessed):
|
| 75 |
-
print(f"Loading preprocessed few-shot data from {preprocessed}")
|
| 76 |
-
with open(preprocessed, "rb") as file:
|
| 77 |
-
data = pickle.load(file)
|
| 78 |
-
train, val = data["train"], data["val"]
|
| 79 |
-
else:
|
| 80 |
-
train = self.generate_fewshot_dataset(train, num_shots=num_shots)
|
| 81 |
-
val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
|
| 82 |
-
data = {"train": train, "val": val}
|
| 83 |
-
print(f"Saving preprocessed few-shot data to {preprocessed}")
|
| 84 |
-
with open(preprocessed, "wb") as file:
|
| 85 |
-
pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)
|
| 86 |
-
|
| 87 |
-
subsample = subsample_classes
|
| 88 |
-
train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample)
|
| 89 |
-
|
| 90 |
-
# Template per user spec
|
| 91 |
-
self.templates = [
|
| 92 |
-
lambda c: f'a {c} review of a movie.',
|
| 93 |
-
]
|
| 94 |
-
|
| 95 |
-
super().__init__(train_x=train, val=val, test=test)
|
| 96 |
-
|
| 97 |
-
def _read_split_dir(self, split_dir, class_to_label):
|
| 98 |
-
items = []
|
| 99 |
-
if not os.path.isdir(split_dir):
|
| 100 |
-
return items
|
| 101 |
-
codes = listdir_nohidden(split_dir)
|
| 102 |
-
# Validate folders: must be subset of expected classes and cover at least one class
|
| 103 |
-
unexpected = sorted([c for c in codes if c not in class_to_label])
|
| 104 |
-
if unexpected:
|
| 105 |
-
raise ValueError(
|
| 106 |
-
f"SST2: Found unexpected class folders in '{split_dir}': {unexpected}. "
|
| 107 |
-
f"Expected only {list(class_to_label.keys())}."
|
| 108 |
-
)
|
| 109 |
-
for cls in codes:
|
| 110 |
-
class_dir = os.path.join(split_dir, cls)
|
| 111 |
-
if not os.path.isdir(class_dir):
|
| 112 |
-
continue
|
| 113 |
-
label = class_to_label.get(cls)
|
| 114 |
-
if label is None:
|
| 115 |
-
raise ValueError(
|
| 116 |
-
f"SST2: Inconsistent label mapping for class '{cls}' in split '{split_dir}'."
|
| 117 |
-
)
|
| 118 |
-
cname = cls
|
| 119 |
-
for fname in listdir_nohidden(class_dir):
|
| 120 |
-
impath = os.path.join(class_dir, fname)
|
| 121 |
-
items.append(Datum(impath=impath, label=label, classname=cname))
|
| 122 |
-
return items
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/MTIL_datasets/stanford_cars.py
DELETED
|
@@ -1,83 +0,0 @@
|
|
| 1 |
-
import os
|
| 2 |
-
import pickle
|
| 3 |
-
from scipy.io import loadmat
|
| 4 |
-
|
| 5 |
-
from .utils import *
|
| 6 |
-
|
| 7 |
-
from .oxford_pets import OxfordPets
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
class StanfordCars(DatasetBase):
|
| 11 |
-
|
| 12 |
-
dataset_dir = "stanford_cars"
|
| 13 |
-
|
| 14 |
-
def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'):
|
| 15 |
-
root = os.path.abspath(os.path.expanduser(root))
|
| 16 |
-
self.dataset_dir = os.path.join(root, self.dataset_dir)
|
| 17 |
-
self.split_path = os.path.join(self.dataset_dir, "split_zhou_StanfordCars.json")
|
| 18 |
-
self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot")
|
| 19 |
-
mkdir_if_missing(self.split_fewshot_dir)
|
| 20 |
-
|
| 21 |
-
if os.path.exists(self.split_path):
|
| 22 |
-
train, val, test = OxfordPets.read_split(self.split_path, self.dataset_dir)
|
| 23 |
-
else:
|
| 24 |
-
trainval_file = os.path.join(self.dataset_dir, "devkit", "cars_train_annos.mat")
|
| 25 |
-
test_file = os.path.join(self.dataset_dir, "cars_test_annos_withlabels.mat")
|
| 26 |
-
meta_file = os.path.join(self.dataset_dir, "devkit", "cars_meta.mat")
|
| 27 |
-
trainval = self.read_data("cars_train", trainval_file, meta_file)
|
| 28 |
-
test = self.read_data("cars_test", test_file, meta_file)
|
| 29 |
-
train, val = OxfordPets.split_trainval(trainval)
|
| 30 |
-
OxfordPets.save_split(train, val, test, self.split_path, self.dataset_dir)
|
| 31 |
-
|
| 32 |
-
if num_shots >= 1:
|
| 33 |
-
preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
|
| 34 |
-
|
| 35 |
-
if os.path.exists(preprocessed):
|
| 36 |
-
print(f"Loading preprocessed few-shot data from {preprocessed}")
|
| 37 |
-
with open(preprocessed, "rb") as file:
|
| 38 |
-
data = pickle.load(file)
|
| 39 |
-
train, val = data["train"], data["val"]
|
| 40 |
-
else:
|
| 41 |
-
train = self.generate_fewshot_dataset(train, num_shots=num_shots)
|
| 42 |
-
val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
|
| 43 |
-
data = {"train": train, "val": val}
|
| 44 |
-
print(f"Saving preprocessed few-shot data to {preprocessed}")
|
| 45 |
-
with open(preprocessed, "wb") as file:
|
| 46 |
-
pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)
|
| 47 |
-
|
| 48 |
-
subsample = subsample_classes
|
| 49 |
-
train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample)
|
| 50 |
-
|
| 51 |
-
self.templates = [
|
| 52 |
-
lambda c: f"a photo of a {c}, a type of car.",
|
| 53 |
-
lambda c: f"a photo of a {c}.",
|
| 54 |
-
lambda c: f"a photo of the {c}.",
|
| 55 |
-
lambda c: f"a photo of my {c}.",
|
| 56 |
-
lambda c: f"i love my {c}!",
|
| 57 |
-
lambda c: f"a photo of my dirty {c}.",
|
| 58 |
-
lambda c: f"a photo of my clean {c}.",
|
| 59 |
-
lambda c: f"a photo of my new {c}.",
|
| 60 |
-
lambda c: f"a photo of my old {c}.",
|
| 61 |
-
]
|
| 62 |
-
|
| 63 |
-
super().__init__(train_x=train, val=val, test=test)
|
| 64 |
-
|
| 65 |
-
def read_data(self, image_dir, anno_file, meta_file):
|
| 66 |
-
anno_file = loadmat(anno_file)["annotations"][0]
|
| 67 |
-
meta_file = loadmat(meta_file)["class_names"][0]
|
| 68 |
-
items = []
|
| 69 |
-
|
| 70 |
-
for i in range(len(anno_file)):
|
| 71 |
-
imname = anno_file[i]["fname"][0]
|
| 72 |
-
impath = os.path.join(self.dataset_dir, image_dir, imname)
|
| 73 |
-
label = anno_file[i]["class"][0, 0]
|
| 74 |
-
label = int(label) - 1
|
| 75 |
-
classname = meta_file[label][0]
|
| 76 |
-
names = classname.split(" ")
|
| 77 |
-
year = names.pop(-1)
|
| 78 |
-
names.insert(0, year)
|
| 79 |
-
classname = " ".join(names)
|
| 80 |
-
item = Datum(impath=impath, label=label, classname=classname)
|
| 81 |
-
items.append(item)
|
| 82 |
-
|
| 83 |
-
return items
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/MTIL_datasets/stl10.py
DELETED
|
@@ -1,143 +0,0 @@
|
|
| 1 |
-
import os
|
| 2 |
-
import math
|
| 3 |
-
import random
|
| 4 |
-
from typing import List, Tuple
|
| 5 |
-
|
| 6 |
-
from .utils import * # Datum, DatasetBase, listdir_nohidden
|
| 7 |
-
from .oxford_pets import OxfordPets
|
| 8 |
-
|
| 9 |
-
STL10_CLASSES: List[str] = [
|
| 10 |
-
'airplane',
|
| 11 |
-
'bird',
|
| 12 |
-
'car',
|
| 13 |
-
'cat',
|
| 14 |
-
'deer',
|
| 15 |
-
'dog',
|
| 16 |
-
'horse',
|
| 17 |
-
'monkey',
|
| 18 |
-
'ship',
|
| 19 |
-
'truck',
|
| 20 |
-
]
|
| 21 |
-
|
| 22 |
-
# keep templates as strings with {}
|
| 23 |
-
STL10_TEMPLATES: List[str] = [
|
| 24 |
-
'a photo of a {}.',
|
| 25 |
-
'a photo of the {}.',
|
| 26 |
-
]
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
class STL10(DatasetBase):
|
| 30 |
-
|
| 31 |
-
dataset_dir = "stl10"
|
| 32 |
-
|
| 33 |
-
def __init__(self, root, num_shots=0, seed=1, subsample_classes='all', test_ratio: float = 0.2):
|
| 34 |
-
"""
|
| 35 |
-
Expect directory structure:
|
| 36 |
-
{root}/stl10/<class_name>/*.png|jpg|jpeg|bmp|webp
|
| 37 |
-
No official test set provided; we split per-class into train/test.
|
| 38 |
-
Then we further split train into train/val using OxfordPets.split_trainval.
|
| 39 |
-
"""
|
| 40 |
-
rnd = random.Random(seed)
|
| 41 |
-
|
| 42 |
-
root = os.path.abspath(os.path.expanduser(root))
|
| 43 |
-
self.dataset_dir = os.path.join(root, self.dataset_dir)
|
| 44 |
-
self.split_path = os.path.join(self.dataset_dir, "split_custom_STL10.json")
|
| 45 |
-
self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot")
|
| 46 |
-
mkdir_if_missing(self.split_fewshot_dir)
|
| 47 |
-
|
| 48 |
-
# If we have a saved split, reuse for reproducibility
|
| 49 |
-
if os.path.exists(self.split_path):
|
| 50 |
-
train, val, test = OxfordPets.read_split(self.split_path, self.dataset_dir)
|
| 51 |
-
else:
|
| 52 |
-
trainval, test = self._read_from_folders(self.dataset_dir, rnd=rnd, test_ratio=test_ratio)
|
| 53 |
-
train, val = self._split_trainval_safe(trainval)
|
| 54 |
-
OxfordPets.save_split(train, val, test, self.split_path, self.dataset_dir)
|
| 55 |
-
|
| 56 |
-
if num_shots >= 1:
|
| 57 |
-
preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
|
| 58 |
-
if os.path.exists(preprocessed):
|
| 59 |
-
print(f"Loading preprocessed few-shot data from {preprocessed}")
|
| 60 |
-
import pickle
|
| 61 |
-
with open(preprocessed, "rb") as file:
|
| 62 |
-
data = pickle.load(file)
|
| 63 |
-
train, val = data["train"], data["val"]
|
| 64 |
-
else:
|
| 65 |
-
train = self.generate_fewshot_dataset(train, num_shots=num_shots)
|
| 66 |
-
val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
|
| 67 |
-
data = {"train": train, "val": val}
|
| 68 |
-
print(f"Saving preprocessed few-shot data to {preprocessed}")
|
| 69 |
-
import pickle
|
| 70 |
-
with open(preprocessed, "wb") as file:
|
| 71 |
-
pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)
|
| 72 |
-
|
| 73 |
-
train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample_classes)
|
| 74 |
-
|
| 75 |
-
self.templates = STL10_TEMPLATES
|
| 76 |
-
|
| 77 |
-
super().__init__(train_x=train, val=val, test=test)
|
| 78 |
-
|
| 79 |
-
def _read_from_folders(self, data_dir: str, rnd: random.Random, test_ratio: float) -> Tuple[List[Datum], List[Datum]]:
|
| 80 |
-
exts = {".png", ".jpg", ".jpeg", ".bmp", ".webp"}
|
| 81 |
-
class_to_label = {name: i for i, name in enumerate(STL10_CLASSES)}
|
| 82 |
-
|
| 83 |
-
items_by_label = {i: [] for i in range(len(STL10_CLASSES))}
|
| 84 |
-
for cname in STL10_CLASSES:
|
| 85 |
-
cdir = os.path.join(data_dir, cname)
|
| 86 |
-
if not os.path.isdir(cdir):
|
| 87 |
-
raise FileNotFoundError(f"Class folder not found: {cdir}")
|
| 88 |
-
for fname in listdir_nohidden(cdir, sort=True):
|
| 89 |
-
fext = os.path.splitext(fname)[1].lower()
|
| 90 |
-
if fext not in exts:
|
| 91 |
-
continue
|
| 92 |
-
impath = os.path.join(cdir, fname)
|
| 93 |
-
label = class_to_label[cname]
|
| 94 |
-
items_by_label[label].append(Datum(impath=impath, label=label, classname=cname))
|
| 95 |
-
|
| 96 |
-
trainval, test = [], []
|
| 97 |
-
for label, items in items_by_label.items():
|
| 98 |
-
if not items:
|
| 99 |
-
continue
|
| 100 |
-
rnd.shuffle(items)
|
| 101 |
-
if len(items) == 1:
|
| 102 |
-
# keep the only sample for training to preserve class presence in train
|
| 103 |
-
trainval.extend(items)
|
| 104 |
-
continue
|
| 105 |
-
n_test = max(1, int(round(len(items) * test_ratio)))
|
| 106 |
-
if n_test >= len(items):
|
| 107 |
-
n_test = len(items) - 1
|
| 108 |
-
test.extend(items[:n_test])
|
| 109 |
-
trainval.extend(items[n_test:])
|
| 110 |
-
|
| 111 |
-
return trainval, test
|
| 112 |
-
|
| 113 |
-
@staticmethod
|
| 114 |
-
def _split_trainval_safe(trainval: List[Datum], p_val: float = 0.2) -> Tuple[List[Datum], List[Datum]]:
|
| 115 |
-
from collections import defaultdict
|
| 116 |
-
tracker = defaultdict(list)
|
| 117 |
-
for idx, item in enumerate(trainval):
|
| 118 |
-
tracker[item.label].append(idx)
|
| 119 |
-
|
| 120 |
-
train, val = [], []
|
| 121 |
-
for label, idxs in tracker.items():
|
| 122 |
-
n = len(idxs)
|
| 123 |
-
if n <= 1:
|
| 124 |
-
# not enough to create a val sample for this class
|
| 125 |
-
for idx in idxs:
|
| 126 |
-
train.append(trainval[idx])
|
| 127 |
-
continue
|
| 128 |
-
n_val = max(1, int(round(n * p_val)))
|
| 129 |
-
if n_val >= n:
|
| 130 |
-
n_val = n - 1
|
| 131 |
-
random.shuffle(idxs)
|
| 132 |
-
for i, idx in enumerate(idxs):
|
| 133 |
-
if i < n_val:
|
| 134 |
-
val.append(trainval[idx])
|
| 135 |
-
else:
|
| 136 |
-
train.append(trainval[idx])
|
| 137 |
-
|
| 138 |
-
# If val ended up empty (degenerate tiny dataset), move one from train to val
|
| 139 |
-
if len(val) == 0 and len(train) > 0:
|
| 140 |
-
val.append(train[-1])
|
| 141 |
-
train = train[:-1]
|
| 142 |
-
|
| 143 |
-
return train, val
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/MTIL_datasets/sun397.py
DELETED
|
@@ -1,81 +0,0 @@
|
|
| 1 |
-
import os
|
| 2 |
-
import pickle
|
| 3 |
-
|
| 4 |
-
from .utils import *
|
| 5 |
-
|
| 6 |
-
from .oxford_pets import OxfordPets
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
class SUN397(DatasetBase):
|
| 10 |
-
|
| 11 |
-
dataset_dir = "sun397"
|
| 12 |
-
|
| 13 |
-
def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'):
|
| 14 |
-
root = os.path.abspath(os.path.expanduser(root))
|
| 15 |
-
self.dataset_dir = os.path.join(root, self.dataset_dir)
|
| 16 |
-
self.image_dir = os.path.join(self.dataset_dir, "SUN397")
|
| 17 |
-
self.split_path = os.path.join(self.dataset_dir, "split_zhou_SUN397.json")
|
| 18 |
-
self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot")
|
| 19 |
-
mkdir_if_missing(self.split_fewshot_dir)
|
| 20 |
-
|
| 21 |
-
if os.path.exists(self.split_path):
|
| 22 |
-
train, val, test = OxfordPets.read_split(self.split_path, self.image_dir)
|
| 23 |
-
else:
|
| 24 |
-
classnames = []
|
| 25 |
-
with open(os.path.join(self.dataset_dir, "ClassName.txt"), "r") as f:
|
| 26 |
-
lines = f.readlines()
|
| 27 |
-
for line in lines:
|
| 28 |
-
line = line.strip()[1:]
|
| 29 |
-
classnames.append(line)
|
| 30 |
-
cname2lab = {c: i for i, c in enumerate(classnames)}
|
| 31 |
-
trainval = self.read_data(cname2lab, "Training_01.txt")
|
| 32 |
-
test = self.read_data(cname2lab, "Testing_01.txt")
|
| 33 |
-
train, val = OxfordPets.split_trainval(trainval)
|
| 34 |
-
OxfordPets.save_split(train, val, test, self.split_path, self.image_dir)
|
| 35 |
-
|
| 36 |
-
if num_shots >= 1:
|
| 37 |
-
preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
|
| 38 |
-
|
| 39 |
-
if os.path.exists(preprocessed):
|
| 40 |
-
print(f"Loading preprocessed few-shot data from {preprocessed}")
|
| 41 |
-
with open(preprocessed, "rb") as file:
|
| 42 |
-
data = pickle.load(file)
|
| 43 |
-
train, val = data["train"], data["val"]
|
| 44 |
-
else:
|
| 45 |
-
train = self.generate_fewshot_dataset(train, num_shots=num_shots)
|
| 46 |
-
val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
|
| 47 |
-
data = {"train": train, "val": val}
|
| 48 |
-
print(f"Saving preprocessed few-shot data to {preprocessed}")
|
| 49 |
-
with open(preprocessed, "wb") as file:
|
| 50 |
-
pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)
|
| 51 |
-
|
| 52 |
-
subsample = subsample_classes
|
| 53 |
-
train, val, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample)
|
| 54 |
-
|
| 55 |
-
self.templates = [
|
| 56 |
-
lambda c: f"a photo of a {c}.",
|
| 57 |
-
lambda c: f"a photo of the {c}.",
|
| 58 |
-
]
|
| 59 |
-
|
| 60 |
-
super().__init__(train_x=train, val=val, test=test)
|
| 61 |
-
|
| 62 |
-
def read_data(self, cname2lab, text_file):
|
| 63 |
-
text_file = os.path.join(self.dataset_dir, text_file)
|
| 64 |
-
items = []
|
| 65 |
-
|
| 66 |
-
with open(text_file, "r") as f:
|
| 67 |
-
lines = f.readlines()
|
| 68 |
-
for line in lines:
|
| 69 |
-
imname = line.strip()[1:]
|
| 70 |
-
classname = os.path.dirname(imname)
|
| 71 |
-
label = cname2lab[classname]
|
| 72 |
-
impath = os.path.join(self.image_dir, imname)
|
| 73 |
-
|
| 74 |
-
names = classname.split("/")[1:]
|
| 75 |
-
names = names[::-1]
|
| 76 |
-
classname = " ".join(names)
|
| 77 |
-
|
| 78 |
-
item = Datum(impath=impath, label=label, classname=classname)
|
| 79 |
-
items.append(item)
|
| 80 |
-
|
| 81 |
-
return items
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/MTIL_datasets/ucf101.py
DELETED
|
@@ -1,360 +0,0 @@
|
|
| 1 |
-
import os
|
| 2 |
-
import pickle
|
| 3 |
-
import random
|
| 4 |
-
import math
|
| 5 |
-
from collections import defaultdict
|
| 6 |
-
from typing import List, Tuple
|
| 7 |
-
|
| 8 |
-
from .utils import * # Datum, DatasetBase, mkdir_if_missing, read_json, write_json, listdir_nohidden
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
# Canonical class names as provided
|
| 12 |
-
UCF101_CANONICAL: List[str] = [
|
| 13 |
-
'Apply Eye Makeup',
|
| 14 |
-
'Apply Lipstick',
|
| 15 |
-
'Archery',
|
| 16 |
-
'Baby Crawling',
|
| 17 |
-
'Balance Beam',
|
| 18 |
-
'Band Marching',
|
| 19 |
-
'Baseball Pitch',
|
| 20 |
-
'Basketball',
|
| 21 |
-
'Basketball Dunk',
|
| 22 |
-
'Bench Press',
|
| 23 |
-
'Biking',
|
| 24 |
-
'Billiards',
|
| 25 |
-
'Blow Dry Hair',
|
| 26 |
-
'Blowing Candles',
|
| 27 |
-
'Body Weight Squats',
|
| 28 |
-
'Bowling',
|
| 29 |
-
'Boxing Punching Bag',
|
| 30 |
-
'Boxing Speed Bag',
|
| 31 |
-
'Breast Stroke',
|
| 32 |
-
'Brushing Teeth',
|
| 33 |
-
'Clean And Jerk',
|
| 34 |
-
'Cliff Diving',
|
| 35 |
-
'Cricket Bowling',
|
| 36 |
-
'Cricket Shot',
|
| 37 |
-
'Cutting In Kitchen',
|
| 38 |
-
'Diving',
|
| 39 |
-
'Drumming',
|
| 40 |
-
'Fencing',
|
| 41 |
-
'Field Hockey Penalty',
|
| 42 |
-
'Floor Gymnastics',
|
| 43 |
-
'Frisbee Catch',
|
| 44 |
-
'Front Crawl',
|
| 45 |
-
'Golf Swing',
|
| 46 |
-
'Haircut',
|
| 47 |
-
'Hammer Throw',
|
| 48 |
-
'Hammering',
|
| 49 |
-
'Hand Stand Pushups',
|
| 50 |
-
'Handstand Walking',
|
| 51 |
-
'Head Massage',
|
| 52 |
-
'High Jump',
|
| 53 |
-
'Horse Race',
|
| 54 |
-
'Horse Riding',
|
| 55 |
-
'Hula Hoop',
|
| 56 |
-
'Ice Dancing',
|
| 57 |
-
'Javelin Throw',
|
| 58 |
-
'Juggling Balls',
|
| 59 |
-
'Jump Rope',
|
| 60 |
-
'Jumping Jack',
|
| 61 |
-
'Kayaking',
|
| 62 |
-
'Knitting',
|
| 63 |
-
'Long Jump',
|
| 64 |
-
'Lunges',
|
| 65 |
-
'Military Parade',
|
| 66 |
-
'Mixing',
|
| 67 |
-
'Mopping Floor',
|
| 68 |
-
'Nunchucks',
|
| 69 |
-
'Parallel Bars',
|
| 70 |
-
'Pizza Tossing',
|
| 71 |
-
'Playing Cello',
|
| 72 |
-
'Playing Daf',
|
| 73 |
-
'Playing Dhol',
|
| 74 |
-
'Playing Flute',
|
| 75 |
-
'Playing Guitar',
|
| 76 |
-
'Playing Piano',
|
| 77 |
-
'Playing Sitar',
|
| 78 |
-
'Playing Tabla',
|
| 79 |
-
'Playing Violin',
|
| 80 |
-
'Pole Vault',
|
| 81 |
-
'Pommel Horse',
|
| 82 |
-
'Pull Ups',
|
| 83 |
-
'Punch',
|
| 84 |
-
'Push Ups',
|
| 85 |
-
'Rafting',
|
| 86 |
-
'Rock Climbing Indoor',
|
| 87 |
-
'Rope Climbing',
|
| 88 |
-
'Rowing',
|
| 89 |
-
'Salsa Spin',
|
| 90 |
-
'Shaving Beard',
|
| 91 |
-
'Shotput',
|
| 92 |
-
'Skate Boarding',
|
| 93 |
-
'Skiing',
|
| 94 |
-
'Skijet',
|
| 95 |
-
'Sky Diving',
|
| 96 |
-
'Soccer Juggling',
|
| 97 |
-
'Soccer Penalty',
|
| 98 |
-
'Still Rings',
|
| 99 |
-
'Sumo Wrestling',
|
| 100 |
-
'Surfing',
|
| 101 |
-
'Swing',
|
| 102 |
-
'Table Tennis Shot',
|
| 103 |
-
'Tai Chi',
|
| 104 |
-
'Tennis Swing',
|
| 105 |
-
'Throw Discus',
|
| 106 |
-
'Trampoline Jumping',
|
| 107 |
-
'Typing',
|
| 108 |
-
'Uneven Bars',
|
| 109 |
-
'Volleyball Spiking',
|
| 110 |
-
'Walking With Dog',
|
| 111 |
-
'Wall Pushups',
|
| 112 |
-
'Writing On Board',
|
| 113 |
-
'Yo Yo',
|
| 114 |
-
]
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
UCF101_TEMPLATES: List[str] = [
|
| 118 |
-
'a photo of a person {}.',
|
| 119 |
-
'a video of a person {}.',
|
| 120 |
-
'a example of a person {}.',
|
| 121 |
-
'a demonstration of a person {}.',
|
| 122 |
-
'a photo of the person {}.',
|
| 123 |
-
'a video of the person {}.',
|
| 124 |
-
'a example of the person {}.',
|
| 125 |
-
'a demonstration of the person {}.',
|
| 126 |
-
'a photo of a person using {}.',
|
| 127 |
-
'a video of a person using {}.',
|
| 128 |
-
'a example of a person using {}.',
|
| 129 |
-
'a demonstration of a person using {}.',
|
| 130 |
-
'a photo of the person using {}.',
|
| 131 |
-
'a video of the person using {}.',
|
| 132 |
-
'a example of the person using {}.',
|
| 133 |
-
'a demonstration of the person using {}.',
|
| 134 |
-
'a photo of a person doing {}.',
|
| 135 |
-
'a video of a person doing {}.',
|
| 136 |
-
'a example of a person doing {}.',
|
| 137 |
-
'a demonstration of a person doing {}.',
|
| 138 |
-
'a photo of the person doing {}.',
|
| 139 |
-
'a video of the person doing {}.',
|
| 140 |
-
'a example of the person doing {}.',
|
| 141 |
-
'a demonstration of the person doing {}.',
|
| 142 |
-
'a photo of a person during {}.',
|
| 143 |
-
'a video of a person during {}.',
|
| 144 |
-
'a example of a person during {}.',
|
| 145 |
-
'a demonstration of a person during {}.',
|
| 146 |
-
'a photo of the person during {}.',
|
| 147 |
-
'a video of the person during {}.',
|
| 148 |
-
'a example of the person during {}.',
|
| 149 |
-
'a demonstration of the person during {}.',
|
| 150 |
-
'a photo of a person performing {}.',
|
| 151 |
-
'a video of a person performing {}.',
|
| 152 |
-
'a example of a person performing {}.',
|
| 153 |
-
'a demonstration of a person performing {}.',
|
| 154 |
-
'a photo of the person performing {}.',
|
| 155 |
-
'a video of the person performing {}.',
|
| 156 |
-
'a example of the person performing {}.',
|
| 157 |
-
'a demonstration of the person performing {}.',
|
| 158 |
-
'a photo of a person practicing {}.',
|
| 159 |
-
'a video of a person practicing {}.',
|
| 160 |
-
'a example of a person practicing {}.',
|
| 161 |
-
'a demonstration of a person practicing {}.',
|
| 162 |
-
'a photo of the person practicing {}.',
|
| 163 |
-
'a video of the person practicing {}.',
|
| 164 |
-
'a example of the person practicing {}.',
|
| 165 |
-
'a demonstration of the person practicing {}.',
|
| 166 |
-
]
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
class UCF101(DatasetBase):
|
| 170 |
-
"""
|
| 171 |
-
UCF101 midframes classification dataset adapter for MTIL.
|
| 172 |
-
Expected structure:
|
| 173 |
-
root/ucf101/
|
| 174 |
-
UCF-101-midframes/<Class_Folder>/<image>.jpg
|
| 175 |
-
split_zhou_UCF101.json
|
| 176 |
-
"""
|
| 177 |
-
|
| 178 |
-
dataset_dir = "ucf101"
|
| 179 |
-
|
| 180 |
-
def __init__(self, root, num_shots=0, seed=1, subsample_classes='all'):
|
| 181 |
-
root = os.path.abspath(os.path.expanduser(root))
|
| 182 |
-
self.dataset_dir = os.path.join(root, self.dataset_dir)
|
| 183 |
-
self.image_dir = os.path.join(self.dataset_dir, "UCF-101-midframes")
|
| 184 |
-
self.split_path = os.path.join(self.dataset_dir, "split_zhou_UCF101.json")
|
| 185 |
-
self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot")
|
| 186 |
-
mkdir_if_missing(self.split_fewshot_dir)
|
| 187 |
-
|
| 188 |
-
if os.path.exists(self.split_path):
|
| 189 |
-
train, val, test = self.read_split(self.split_path, self.image_dir)
|
| 190 |
-
else:
|
| 191 |
-
# Fallback: build from directory and split train/val; use val as test too.
|
| 192 |
-
trainval = self._read_from_dir()
|
| 193 |
-
train, val = self.split_trainval(trainval)
|
| 194 |
-
test = list(val)
|
| 195 |
-
self.save_split(train, val, test, self.split_path, self.image_dir)
|
| 196 |
-
|
| 197 |
-
if num_shots >= 1:
|
| 198 |
-
preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl")
|
| 199 |
-
if os.path.exists(preprocessed):
|
| 200 |
-
print(f"Loading preprocessed few-shot data from {preprocessed}")
|
| 201 |
-
with open(preprocessed, "rb") as file:
|
| 202 |
-
data = pickle.load(file)
|
| 203 |
-
train, val = data["train"], data["val"]
|
| 204 |
-
else:
|
| 205 |
-
train = self.generate_fewshot_dataset(train, num_shots=num_shots)
|
| 206 |
-
val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4))
|
| 207 |
-
data = {"train": train, "val": val}
|
| 208 |
-
print(f"Saving preprocessed few-shot data to {preprocessed}")
|
| 209 |
-
with open(preprocessed, "wb") as file:
|
| 210 |
-
pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)
|
| 211 |
-
|
| 212 |
-
# Optionally subsample classes (keep interface consistent)
|
| 213 |
-
train, val, test = self.subsample_classes(train, val, test, subsample=subsample_classes)
|
| 214 |
-
|
| 215 |
-
self.templates = UCF101_TEMPLATES
|
| 216 |
-
|
| 217 |
-
super().__init__(train_x=train, val=val, test=test)
|
| 218 |
-
|
| 219 |
-
@staticmethod
|
| 220 |
-
def subsample_classes(*args, subsample="all"):
|
| 221 |
-
assert subsample in ["all", "base", "new"]
|
| 222 |
-
|
| 223 |
-
if subsample == "all":
|
| 224 |
-
return args
|
| 225 |
-
|
| 226 |
-
dataset = args[0]
|
| 227 |
-
labels = set()
|
| 228 |
-
for item in dataset:
|
| 229 |
-
labels.add(item.label)
|
| 230 |
-
labels = list(labels)
|
| 231 |
-
labels.sort()
|
| 232 |
-
n = len(labels)
|
| 233 |
-
m = math.ceil(n / 2)
|
| 234 |
-
|
| 235 |
-
print(f"SUBSAMPLE {subsample.upper()} CLASSES!")
|
| 236 |
-
if subsample == "base":
|
| 237 |
-
selected = labels[:m]
|
| 238 |
-
else:
|
| 239 |
-
selected = labels[m:]
|
| 240 |
-
relabeler = {y: y_new for y_new, y in enumerate(selected)}
|
| 241 |
-
|
| 242 |
-
output = []
|
| 243 |
-
for dataset in args:
|
| 244 |
-
dataset_new = []
|
| 245 |
-
for item in dataset:
|
| 246 |
-
if item.label not in selected:
|
| 247 |
-
continue
|
| 248 |
-
item_new = Datum(
|
| 249 |
-
impath=item.impath,
|
| 250 |
-
label=relabeler[item.label],
|
| 251 |
-
classname=item.classname
|
| 252 |
-
)
|
| 253 |
-
dataset_new.append(item_new)
|
| 254 |
-
output.append(dataset_new)
|
| 255 |
-
|
| 256 |
-
return output
|
| 257 |
-
|
| 258 |
-
# ---- IO helpers (compatible with OxfordPets) ----
|
| 259 |
-
@staticmethod
|
| 260 |
-
def save_split(train, val, test, filepath, path_prefix):
|
| 261 |
-
def _extract(items):
|
| 262 |
-
out = []
|
| 263 |
-
for item in items:
|
| 264 |
-
impath = item.impath
|
| 265 |
-
label = item.label
|
| 266 |
-
classname = item.classname
|
| 267 |
-
impath = impath.replace(path_prefix, "")
|
| 268 |
-
if impath.startswith("/"):
|
| 269 |
-
impath = impath[1:]
|
| 270 |
-
out.append((impath, label, classname))
|
| 271 |
-
return out
|
| 272 |
-
|
| 273 |
-
train = _extract(train)
|
| 274 |
-
val = _extract(val)
|
| 275 |
-
test = _extract(test)
|
| 276 |
-
split = {"train": train, "val": val, "test": test}
|
| 277 |
-
write_json(split, filepath)
|
| 278 |
-
print(f"Saved split to {filepath}")
|
| 279 |
-
|
| 280 |
-
@staticmethod
|
| 281 |
-
def read_split(filepath, path_prefix):
|
| 282 |
-
def _convert(items):
|
| 283 |
-
out = []
|
| 284 |
-
for impath, label, classname in items:
|
| 285 |
-
impath = os.path.join(path_prefix, impath)
|
| 286 |
-
item = Datum(impath=impath, label=int(label), classname=classname)
|
| 287 |
-
out.append(item)
|
| 288 |
-
return out
|
| 289 |
-
|
| 290 |
-
print(f"Reading split from {filepath}")
|
| 291 |
-
split = read_json(filepath)
|
| 292 |
-
train = _convert(split["train"])
|
| 293 |
-
val = _convert(split["val"])
|
| 294 |
-
test = _convert(split["test"])
|
| 295 |
-
return train, val, test
|
| 296 |
-
|
| 297 |
-
@staticmethod
|
| 298 |
-
def split_trainval(trainval: List[Datum], p_val=0.2) -> Tuple[List[Datum], List[Datum]]:
|
| 299 |
-
p_trn = 1 - p_val
|
| 300 |
-
tracker = defaultdict(list)
|
| 301 |
-
for idx, item in enumerate(trainval):
|
| 302 |
-
tracker[item.label].append(idx)
|
| 303 |
-
train, val = [], []
|
| 304 |
-
for label, idxs in tracker.items():
|
| 305 |
-
n_val = max(1, round(len(idxs) * p_val))
|
| 306 |
-
random.shuffle(idxs)
|
| 307 |
-
for n, idx in enumerate(idxs):
|
| 308 |
-
item = trainval[idx]
|
| 309 |
-
if n < n_val:
|
| 310 |
-
val.append(item)
|
| 311 |
-
else:
|
| 312 |
-
train.append(item)
|
| 313 |
-
return train, val
|
| 314 |
-
|
| 315 |
-
# ---- Directory reader (fallback if JSON is missing) ----
|
| 316 |
-
def _read_from_dir(self) -> List[Datum]:
|
| 317 |
-
if not os.path.isdir(self.image_dir):
|
| 318 |
-
raise FileNotFoundError(f"Image directory not found: {self.image_dir}")
|
| 319 |
-
|
| 320 |
-
# Build canonical key mapping (lowercased, remove spaces/underscores)
|
| 321 |
-
def canon_key(s: str) -> str:
|
| 322 |
-
return ''.join(ch for ch in s.lower() if ch.isalnum())
|
| 323 |
-
|
| 324 |
-
canonical_to_label = {name: idx for idx, name in enumerate(UCF101_CANONICAL)}
|
| 325 |
-
key_to_canonical = {canon_key(name): name for name in UCF101_CANONICAL}
|
| 326 |
-
|
| 327 |
-
items: List[Datum] = []
|
| 328 |
-
class_dirs = listdir_nohidden(self.image_dir, sort=True)
|
| 329 |
-
for cls_dir in class_dirs:
|
| 330 |
-
cls_path = os.path.join(self.image_dir, cls_dir)
|
| 331 |
-
if not os.path.isdir(cls_path):
|
| 332 |
-
continue
|
| 333 |
-
# Try to map folder name to canonical class
|
| 334 |
-
k = canon_key(cls_dir.replace('_', ' '))
|
| 335 |
-
cname = key_to_canonical.get(k, None)
|
| 336 |
-
if cname is None:
|
| 337 |
-
# Try removing underscores without space
|
| 338 |
-
k2 = canon_key(cls_dir.replace('_', ''))
|
| 339 |
-
cname = key_to_canonical.get(k2, None)
|
| 340 |
-
if cname is None:
|
| 341 |
-
# As a last resort, use the folder name with underscores replaced
|
| 342 |
-
cname = cls_dir.replace('_', ' ').strip()
|
| 343 |
-
if cname not in canonical_to_label:
|
| 344 |
-
# Unknown class; skip
|
| 345 |
-
continue
|
| 346 |
-
label = canonical_to_label[cname]
|
| 347 |
-
# Collect images
|
| 348 |
-
for vid in listdir_nohidden(cls_path, sort=False):
|
| 349 |
-
vpath = os.path.join(cls_path, vid)
|
| 350 |
-
if os.path.isdir(vpath):
|
| 351 |
-
# some datasets might nest frames under video folder; include all frames
|
| 352 |
-
for frame in listdir_nohidden(vpath, sort=False):
|
| 353 |
-
impath = os.path.join(vpath, frame)
|
| 354 |
-
if os.path.isfile(impath):
|
| 355 |
-
items.append(Datum(impath=impath, label=label, classname=cname))
|
| 356 |
-
else:
|
| 357 |
-
# direct frames under class folder
|
| 358 |
-
if os.path.isfile(vpath):
|
| 359 |
-
items.append(Datum(impath=vpath, label=label, classname=cname))
|
| 360 |
-
return items
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/MTIL_datasets/utils.py
DELETED
|
@@ -1,299 +0,0 @@
|
|
| 1 |
-
import os
|
| 2 |
-
import random
|
| 3 |
-
import os.path as osp
|
| 4 |
-
import tarfile
|
| 5 |
-
import zipfile
|
| 6 |
-
from collections import defaultdict
|
| 7 |
-
import gdown
|
| 8 |
-
import errno
|
| 9 |
-
import warnings
|
| 10 |
-
import json
|
| 11 |
-
import numpy as np
|
| 12 |
-
import h5py
|
| 13 |
-
from torch.utils.data import Dataset as TorchDataset
|
| 14 |
-
from PIL import Image
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
class Datum:
|
| 18 |
-
def __init__(self, impath="", label=0, domain=0, classname=""):
|
| 19 |
-
self._impath = impath
|
| 20 |
-
self._label = label
|
| 21 |
-
self._domain = domain
|
| 22 |
-
self._classname = classname
|
| 23 |
-
|
| 24 |
-
@property
|
| 25 |
-
def impath(self):
|
| 26 |
-
return self._impath
|
| 27 |
-
|
| 28 |
-
@property
|
| 29 |
-
def label(self):
|
| 30 |
-
return self._label
|
| 31 |
-
|
| 32 |
-
@property
|
| 33 |
-
def domain(self):
|
| 34 |
-
return self._domain
|
| 35 |
-
|
| 36 |
-
@property
|
| 37 |
-
def classname(self):
|
| 38 |
-
return self._classname
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
class DatasetBase:
|
| 42 |
-
dataset_dir = ""
|
| 43 |
-
domains = []
|
| 44 |
-
|
| 45 |
-
def __init__(self, train_x=None, train_u=None, val=None, test=None):
|
| 46 |
-
self._train_x = train_x
|
| 47 |
-
self._train_u = train_u
|
| 48 |
-
self._val = val
|
| 49 |
-
self._test = test
|
| 50 |
-
self._num_classes = self.get_num_classes(train_x)
|
| 51 |
-
self._lab2cname, self._classnames = self.get_lab2cname(train_x)
|
| 52 |
-
|
| 53 |
-
@property
|
| 54 |
-
def train_x(self):
|
| 55 |
-
return self._train_x
|
| 56 |
-
|
| 57 |
-
@property
|
| 58 |
-
def train_u(self):
|
| 59 |
-
return self._train_u
|
| 60 |
-
|
| 61 |
-
@property
|
| 62 |
-
def val(self):
|
| 63 |
-
return self._val
|
| 64 |
-
|
| 65 |
-
@property
|
| 66 |
-
def test(self):
|
| 67 |
-
return self._test
|
| 68 |
-
|
| 69 |
-
@property
|
| 70 |
-
def lab2cname(self):
|
| 71 |
-
return self._lab2cname
|
| 72 |
-
|
| 73 |
-
@property
|
| 74 |
-
def classnames(self):
|
| 75 |
-
return self._classnames
|
| 76 |
-
|
| 77 |
-
@property
|
| 78 |
-
def num_classes(self):
|
| 79 |
-
return self._num_classes
|
| 80 |
-
|
| 81 |
-
@staticmethod
|
| 82 |
-
def get_num_classes(data_source):
|
| 83 |
-
label_set = set()
|
| 84 |
-
for item in data_source:
|
| 85 |
-
label_set.add(item.label)
|
| 86 |
-
return max(label_set) + 1
|
| 87 |
-
|
| 88 |
-
@staticmethod
|
| 89 |
-
def get_lab2cname(data_source):
|
| 90 |
-
container = set()
|
| 91 |
-
for item in data_source:
|
| 92 |
-
container.add((item.label, item.classname))
|
| 93 |
-
mapping = {label: classname for label, classname in container}
|
| 94 |
-
labels = list(mapping.keys())
|
| 95 |
-
labels.sort()
|
| 96 |
-
classnames = [mapping[label] for label in labels]
|
| 97 |
-
return mapping, classnames
|
| 98 |
-
|
| 99 |
-
@property
|
| 100 |
-
def template(self):
|
| 101 |
-
return self.templates[0]
|
| 102 |
-
|
| 103 |
-
def check_input_domains(self, source_domains, target_domains):
|
| 104 |
-
assert len(source_domains) > 0, "source_domains (list) is empty"
|
| 105 |
-
assert len(target_domains) > 0, "target_domains (list) is empty"
|
| 106 |
-
self.is_input_domain_valid(source_domains)
|
| 107 |
-
self.is_input_domain_valid(target_domains)
|
| 108 |
-
|
| 109 |
-
def is_input_domain_valid(self, input_domains):
|
| 110 |
-
for domain in input_domains:
|
| 111 |
-
if domain not in self.domains:
|
| 112 |
-
raise ValueError(
|
| 113 |
-
"Input domain must belong to {}, "
|
| 114 |
-
"but got [{}]".format(self.domains, domain)
|
| 115 |
-
)
|
| 116 |
-
|
| 117 |
-
def download_data(self, url, dst, from_gdrive=True):
|
| 118 |
-
if not osp.exists(osp.dirname(dst)):
|
| 119 |
-
os.makedirs(osp.dirname(dst))
|
| 120 |
-
|
| 121 |
-
if from_gdrive:
|
| 122 |
-
gdown.download(url, dst, quiet=False)
|
| 123 |
-
else:
|
| 124 |
-
raise NotImplementedError
|
| 125 |
-
|
| 126 |
-
print("Extracting file ...")
|
| 127 |
-
|
| 128 |
-
if dst.endswith(".zip"):
|
| 129 |
-
zip_ref = zipfile.ZipFile(dst, "r")
|
| 130 |
-
zip_ref.extractall(osp.dirname(dst))
|
| 131 |
-
zip_ref.close()
|
| 132 |
-
|
| 133 |
-
elif dst.endswith(".tar"):
|
| 134 |
-
tar = tarfile.open(dst, "r:")
|
| 135 |
-
tar.extractall(osp.dirname(dst))
|
| 136 |
-
tar.close()
|
| 137 |
-
|
| 138 |
-
elif dst.endswith(".tar.gz"):
|
| 139 |
-
tar = tarfile.open(dst, "r:gz")
|
| 140 |
-
tar.extractall(osp.dirname(dst))
|
| 141 |
-
tar.close()
|
| 142 |
-
|
| 143 |
-
else:
|
| 144 |
-
raise NotImplementedError
|
| 145 |
-
|
| 146 |
-
print("File extracted to {}".format(osp.dirname(dst)))
|
| 147 |
-
|
| 148 |
-
def generate_fewshot_dataset(
|
| 149 |
-
self, *data_sources, num_shots=-1, repeat=False
|
| 150 |
-
):
|
| 151 |
-
if num_shots < 1:
|
| 152 |
-
if len(data_sources) == 1:
|
| 153 |
-
return data_sources[0]
|
| 154 |
-
return data_sources
|
| 155 |
-
|
| 156 |
-
print(f"Creating a {num_shots}-shot dataset")
|
| 157 |
-
|
| 158 |
-
output = []
|
| 159 |
-
|
| 160 |
-
for data_source in data_sources:
|
| 161 |
-
tracker = self.split_dataset_by_label(data_source)
|
| 162 |
-
dataset = []
|
| 163 |
-
|
| 164 |
-
for label, items in tracker.items():
|
| 165 |
-
if len(items) >= num_shots:
|
| 166 |
-
sampled_items = random.sample(items, num_shots)
|
| 167 |
-
else:
|
| 168 |
-
if repeat:
|
| 169 |
-
sampled_items = random.choices(items, k=num_shots)
|
| 170 |
-
else:
|
| 171 |
-
sampled_items = items
|
| 172 |
-
dataset.extend(sampled_items)
|
| 173 |
-
|
| 174 |
-
output.append(dataset)
|
| 175 |
-
|
| 176 |
-
if len(output) == 1:
|
| 177 |
-
return output[0]
|
| 178 |
-
|
| 179 |
-
return output
|
| 180 |
-
|
| 181 |
-
def split_dataset_by_label(self, data_source):
|
| 182 |
-
output = defaultdict(list)
|
| 183 |
-
|
| 184 |
-
for item in data_source:
|
| 185 |
-
output[item.label].append(item)
|
| 186 |
-
|
| 187 |
-
return output
|
| 188 |
-
|
| 189 |
-
def split_dataset_by_domain(self, data_source):
|
| 190 |
-
output = defaultdict(list)
|
| 191 |
-
|
| 192 |
-
for item in data_source:
|
| 193 |
-
output[item.domain].append(item)
|
| 194 |
-
|
| 195 |
-
return output
|
| 196 |
-
|
| 197 |
-
|
| 198 |
-
class DatasetWrapper(TorchDataset):
|
| 199 |
-
|
| 200 |
-
def __init__(self, data_source, transform=None, is_train=False):
|
| 201 |
-
self.data_source = data_source
|
| 202 |
-
self.transform = transform
|
| 203 |
-
self.is_train = is_train
|
| 204 |
-
self._h5_cache = {}
|
| 205 |
-
self._h5_info_printed = set()
|
| 206 |
-
|
| 207 |
-
def __len__(self):
|
| 208 |
-
return len(self.data_source)
|
| 209 |
-
|
| 210 |
-
def __getitem__(self, idx):
|
| 211 |
-
item = self.data_source[idx]
|
| 212 |
-
|
| 213 |
-
impath = item.impath
|
| 214 |
-
if isinstance(impath, str):
|
| 215 |
-
img0 = Image.open(impath).convert("RGB")
|
| 216 |
-
elif isinstance(impath, tuple) and len(impath) == 4 and impath[0] == 'h5':
|
| 217 |
-
_, fpath, key, index = impath
|
| 218 |
-
f = self._h5_cache.get(fpath)
|
| 219 |
-
if f is None:
|
| 220 |
-
f = h5py.File(fpath, 'r')
|
| 221 |
-
self._h5_cache[fpath] = f
|
| 222 |
-
# Print file info once
|
| 223 |
-
try:
|
| 224 |
-
keys = list(f.keys())
|
| 225 |
-
print(f"H5 open: {os.path.basename(fpath)} keys={keys[:5]}{'...' if len(keys) > 5 else ''}")
|
| 226 |
-
except Exception as e:
|
| 227 |
-
print(f"H5 open (keys) failed for {fpath}: {e}")
|
| 228 |
-
# Print dataset info per file the first time we see this path
|
| 229 |
-
if fpath not in self._h5_info_printed:
|
| 230 |
-
try:
|
| 231 |
-
ds = f[key]
|
| 232 |
-
print(f"H5 dataset: {os.path.basename(fpath)}[{key}] shape={getattr(ds, 'shape', '?')} dtype={getattr(ds, 'dtype', '?')}")
|
| 233 |
-
except Exception as e:
|
| 234 |
-
print(f"H5 dataset info failed for {fpath}[{key}]: {e}")
|
| 235 |
-
self._h5_info_printed.add(fpath)
|
| 236 |
-
arr = f[key][int(index)]
|
| 237 |
-
arr = np.asarray(arr)
|
| 238 |
-
# Convert CHW -> HWC if needed
|
| 239 |
-
if arr.ndim == 3 and arr.shape[0] in (1, 3) and arr.shape[-1] not in (1, 3):
|
| 240 |
-
arr = np.transpose(arr, (1, 2, 0))
|
| 241 |
-
# Ensure HWC and uint8
|
| 242 |
-
if arr.ndim == 3 and arr.shape[-1] in (1, 3):
|
| 243 |
-
pass
|
| 244 |
-
else:
|
| 245 |
-
raise ValueError(f"Unexpected H5 image shape: {arr.shape}")
|
| 246 |
-
if arr.dtype != np.uint8:
|
| 247 |
-
arr = arr.astype(np.uint8)
|
| 248 |
-
if arr.shape[-1] == 1:
|
| 249 |
-
img0 = Image.fromarray(arr.squeeze(-1), mode='L').convert('RGB')
|
| 250 |
-
else:
|
| 251 |
-
img0 = Image.fromarray(arr, mode='RGB')
|
| 252 |
-
else:
|
| 253 |
-
# Fallback: if already PIL Image
|
| 254 |
-
if isinstance(impath, Image.Image):
|
| 255 |
-
img0 = impath
|
| 256 |
-
else:
|
| 257 |
-
raise ValueError("Unsupported impath type in DatasetWrapper")
|
| 258 |
-
|
| 259 |
-
if self.transform:
|
| 260 |
-
img = self.transform(img0)
|
| 261 |
-
else:
|
| 262 |
-
img = img0
|
| 263 |
-
|
| 264 |
-
return img, item.label
|
| 265 |
-
|
| 266 |
-
|
| 267 |
-
def check_isfile(fpath):
|
| 268 |
-
isfile = osp.isfile(fpath)
|
| 269 |
-
if not isfile:
|
| 270 |
-
warnings.warn('No file found at "{}"'.format(fpath))
|
| 271 |
-
return isfile
|
| 272 |
-
|
| 273 |
-
|
| 274 |
-
def mkdir_if_missing(dirname):
|
| 275 |
-
if not osp.exists(dirname):
|
| 276 |
-
try:
|
| 277 |
-
os.makedirs(dirname)
|
| 278 |
-
except OSError as e:
|
| 279 |
-
if e.errno != errno.EEXIST:
|
| 280 |
-
raise
|
| 281 |
-
|
| 282 |
-
|
| 283 |
-
def listdir_nohidden(path, sort=False):
|
| 284 |
-
items = [f for f in os.listdir(path) if not f.startswith(".")]
|
| 285 |
-
if sort:
|
| 286 |
-
items.sort()
|
| 287 |
-
return items
|
| 288 |
-
|
| 289 |
-
|
| 290 |
-
def read_json(fpath):
|
| 291 |
-
with open(fpath, "r") as f:
|
| 292 |
-
obj = json.load(f)
|
| 293 |
-
return obj
|
| 294 |
-
|
| 295 |
-
|
| 296 |
-
def write_json(obj, fpath):
|
| 297 |
-
mkdir_if_missing(osp.dirname(fpath))
|
| 298 |
-
with open(fpath, "w") as f:
|
| 299 |
-
json.dump(obj, f, indent=4, separators=(",", ": "))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/MTIL_datasets/voc2007.py
DELETED
|
@@ -1,173 +0,0 @@
|
|
| 1 |
-
import os
|
| 2 |
-
import os.path as osp
|
| 3 |
-
from typing import List, Dict, Tuple
|
| 4 |
-
from collections import defaultdict
|
| 5 |
-
|
| 6 |
-
from .utils import Datum, DatasetBase, listdir_nohidden
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
VOC2007_CLASSES = [
|
| 10 |
-
'aeroplane', 'bicycle', 'bird', 'boat', 'bottle',
|
| 11 |
-
'bus', 'car', 'cat', 'chair', 'cow',
|
| 12 |
-
'dog', 'horse', 'motorbike', 'person', 'sheep',
|
| 13 |
-
'sofa', 'diningtable', 'pottedplant', 'train', 'tvmonitor',
|
| 14 |
-
]
|
| 15 |
-
|
| 16 |
-
VOC2007_TEMPLATES = [
|
| 17 |
-
'a photo of a {}.',
|
| 18 |
-
]
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
class VOC2007(DatasetBase):
|
| 22 |
-
"""
|
| 23 |
-
VOC2007 multi-label classification dataset adapter for MTIL evaluation.
|
| 24 |
-
|
| 25 |
-
Expects directory structure:
|
| 26 |
-
data/VOC2007/
|
| 27 |
-
JPEGImages/
|
| 28 |
-
Main/
|
| 29 |
-
<class>_trainval.txt
|
| 30 |
-
<class>_test.txt
|
| 31 |
-
|
| 32 |
-
Each *_split.txt has lines: "<image_id> <label>", where label in {1, 0, -1}.
|
| 33 |
-
We treat 1 as positive and others as negative for that class.
|
| 34 |
-
"""
|
| 35 |
-
|
| 36 |
-
dataset_dir = 'VOC2007'
|
| 37 |
-
|
| 38 |
-
def __init__(self, root: str, seed: int = 32, single_label: bool = False):
|
| 39 |
-
self.root = os.path.expanduser(root)
|
| 40 |
-
self.dataset_dir = osp.join(self.root, self.dataset_dir)
|
| 41 |
-
self.images_dir = osp.join(self.dataset_dir, 'JPEGImages')
|
| 42 |
-
self.main_dir = osp.join(self.dataset_dir, 'Main')
|
| 43 |
-
|
| 44 |
-
classes = VOC2007_CLASSES
|
| 45 |
-
class_to_label = {c: i for i, c in enumerate(classes)}
|
| 46 |
-
|
| 47 |
-
if not single_label:
|
| 48 |
-
# Multi-label mode (for Zero-Shot evaluation)
|
| 49 |
-
test_list = self._read_split('test', classes)
|
| 50 |
-
test = self._build_data_list(test_list, class_to_label)
|
| 51 |
-
|
| 52 |
-
# Create a meta train set with one sample per class (integer labels) for proper initialization
|
| 53 |
-
# Try to point to a real image containing that class as positive; fallback to any available image or empty path
|
| 54 |
-
meta_train: List[Datum] = []
|
| 55 |
-
for ci, cname in enumerate(classes):
|
| 56 |
-
impath = None
|
| 57 |
-
for d in test:
|
| 58 |
-
try:
|
| 59 |
-
if isinstance(d.label, (list, tuple)) and ci < len(d.label) and int(d.label[ci]) == 1:
|
| 60 |
-
impath = d.impath
|
| 61 |
-
break
|
| 62 |
-
except Exception:
|
| 63 |
-
continue
|
| 64 |
-
if impath is None:
|
| 65 |
-
impath = test[0].impath if len(test) > 0 else ''
|
| 66 |
-
meta_train.append(Datum(impath=impath, label=ci, classname=cname))
|
| 67 |
-
|
| 68 |
-
train_x: List[Datum] = meta_train
|
| 69 |
-
val: List[Datum] = []
|
| 70 |
-
self.templates = VOC2007_TEMPLATES
|
| 71 |
-
super().__init__(train_x=train_x, val=val, test=test)
|
| 72 |
-
else:
|
| 73 |
-
# Single-label mode (for CIL downstream training/evaluation)
|
| 74 |
-
# Build trainval and test splits as single-label datasets by selecting the first positive class per image
|
| 75 |
-
trainval_multi = self._read_split('trainval', classes)
|
| 76 |
-
test_multi = self._read_split('test', classes)
|
| 77 |
-
train_x = self._build_single_label_list(trainval_multi, classes)
|
| 78 |
-
val = []
|
| 79 |
-
test = self._build_single_label_list(test_multi, classes)
|
| 80 |
-
self.templates = VOC2007_TEMPLATES
|
| 81 |
-
super().__init__(train_x=train_x, val=val, test=test)
|
| 82 |
-
|
| 83 |
-
def _read_split(self, split: str, classes: List[str]) -> Dict[str, List[int]]:
|
| 84 |
-
"""Return mapping: image_id -> multi-hot list for the given split."""
|
| 85 |
-
# Collect image ids present in this split
|
| 86 |
-
img_ids = set()
|
| 87 |
-
by_class_labels: Dict[str, Dict[str, int]] = {}
|
| 88 |
-
for cname in classes:
|
| 89 |
-
split_file = osp.join(self.main_dir, f"{cname}_{split}.txt")
|
| 90 |
-
if not osp.isfile(split_file):
|
| 91 |
-
# Try alternate common path name
|
| 92 |
-
split_file = osp.join(self.dataset_dir, 'ImageSets', 'Main', f"{cname}_{split}.txt")
|
| 93 |
-
if not osp.isfile(split_file):
|
| 94 |
-
# If missing, skip this class
|
| 95 |
-
continue
|
| 96 |
-
class_map: Dict[str, int] = {}
|
| 97 |
-
with open(split_file, 'r') as f:
|
| 98 |
-
for line in f:
|
| 99 |
-
parts = line.strip().split()
|
| 100 |
-
if len(parts) < 2:
|
| 101 |
-
continue
|
| 102 |
-
img_id, label_str = parts[0], parts[1]
|
| 103 |
-
try:
|
| 104 |
-
label = int(label_str)
|
| 105 |
-
except Exception:
|
| 106 |
-
continue
|
| 107 |
-
class_map[img_id] = 1 if label == 1 else 0
|
| 108 |
-
img_ids.add(img_id)
|
| 109 |
-
by_class_labels[cname] = class_map
|
| 110 |
-
|
| 111 |
-
# Aggregate into multi-hot vectors
|
| 112 |
-
result: Dict[str, List[int]] = {}
|
| 113 |
-
for img_id in img_ids:
|
| 114 |
-
vec = [0] * len(classes)
|
| 115 |
-
for ci, cname in enumerate(classes):
|
| 116 |
-
cmap = by_class_labels.get(cname, {})
|
| 117 |
-
vec[ci] = int(cmap.get(img_id, 0))
|
| 118 |
-
result[img_id] = vec
|
| 119 |
-
return result
|
| 120 |
-
|
| 121 |
-
def _build_data_list(self, id_to_vec: Dict[str, List[int]], class_to_label: Dict[str, int]) -> List[Datum]:
|
| 122 |
-
data: List[Datum] = []
|
| 123 |
-
for img_id, vec in id_to_vec.items():
|
| 124 |
-
# Try common image extensions
|
| 125 |
-
impath = None
|
| 126 |
-
for ext in ['.jpg', '.jpeg', '.png']:
|
| 127 |
-
p = osp.join(self.images_dir, img_id + ext)
|
| 128 |
-
if osp.isfile(p):
|
| 129 |
-
impath = p
|
| 130 |
-
break
|
| 131 |
-
if impath is None:
|
| 132 |
-
# As a fallback, if there's exactly one file starting with img_id
|
| 133 |
-
try:
|
| 134 |
-
candidates = [f for f in listdir_nohidden(self.images_dir) if f.startswith(img_id + '.')]
|
| 135 |
-
if candidates:
|
| 136 |
-
impath = osp.join(self.images_dir, candidates[0])
|
| 137 |
-
except Exception:
|
| 138 |
-
pass
|
| 139 |
-
if impath is None:
|
| 140 |
-
# Skip missing images
|
| 141 |
-
continue
|
| 142 |
-
# For multi-label, we store the full vector as the label
|
| 143 |
-
data.append(Datum(impath=impath, label=vec, classname=''))
|
| 144 |
-
return data
|
| 145 |
-
|
| 146 |
-
def _build_single_label_list(self, id_to_vec: Dict[str, List[int]], classes: List[str]) -> List[Datum]:
|
| 147 |
-
data: List[Datum] = []
|
| 148 |
-
for img_id, vec in id_to_vec.items():
|
| 149 |
-
# choose the first positive class; skip if none
|
| 150 |
-
try:
|
| 151 |
-
cls_idx = next((i for i, v in enumerate(vec) if int(v) == 1), None)
|
| 152 |
-
except Exception:
|
| 153 |
-
cls_idx = None
|
| 154 |
-
if cls_idx is None:
|
| 155 |
-
continue
|
| 156 |
-
# image path resolution
|
| 157 |
-
impath = None
|
| 158 |
-
for ext in ['.jpg', '.jpeg', '.png']:
|
| 159 |
-
p = osp.join(self.images_dir, img_id + ext)
|
| 160 |
-
if osp.isfile(p):
|
| 161 |
-
impath = p
|
| 162 |
-
break
|
| 163 |
-
if impath is None:
|
| 164 |
-
try:
|
| 165 |
-
candidates = [f for f in listdir_nohidden(self.images_dir) if f.startswith(img_id + '.')]
|
| 166 |
-
if candidates:
|
| 167 |
-
impath = osp.join(self.images_dir, candidates[0])
|
| 168 |
-
except Exception:
|
| 169 |
-
pass
|
| 170 |
-
if impath is None:
|
| 171 |
-
continue
|
| 172 |
-
data.append(Datum(impath=impath, label=int(cls_idx), classname=classes[cls_idx]))
|
| 173 |
-
return data
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/README.md
DELETED
|
@@ -1,227 +0,0 @@
|
|
| 1 |
-
# DFA-MoE: Tackling Dual Forgetting in Vision-Language Continual Learning
|
| 2 |
-
|
| 3 |
-
[](https://www.python.org/)
|
| 4 |
-
[](https://pytorch.org/)
|
| 5 |
-
|
| 6 |
-
Official PyTorch implementation of the paper "Don't Forget Why You Started: Tackling Dual Forgetting in Vision-Language Continual Learning" ICML2026.
|
| 7 |
-
|
| 8 |
-
## Abstract
|
| 9 |
-
|
| 10 |
-
Vision-Language Models (VLMs) are strong continual learners, but standard class-incremental learning often damages the pre-trained vision-language alignment that underpins zero-shot generalization. This repository studies that dual-forgetting problem from two perspectives: Incremental Knowledge Forgetting (IKF), which harms previously learned classes, and Pre-trained Knowledge Forgetting (PKF), which erodes the original zero-shot capabilities of the VLM. To address this issue, we implement the Dual-Forgetting-Aware Class-Incremental Learning (DFA-CIL) framework, the Similarity-Calibrated Retention (SCR) metric, and DFA-MoE, a functionally heterogeneous PEFT method that decouples alignment preservation from task adaptation.
|
| 11 |
-
|
| 12 |
-

|
| 13 |
-
|
| 14 |
-
## Key Contributions
|
| 15 |
-
|
| 16 |
-
- Dual-forgetting evaluation for VLM continual learning through the DFA-CIL protocol.
|
| 17 |
-
- Similarity-Calibrated Retention (SCR) utilizing similarity based weight to disentangle genuine foundational retention from the confounding effects of positive transfer.
|
| 18 |
-
- DFA-MoE with two functional pathways:
|
| 19 |
-
- Alignment Pathway: task-agnostic contrastive expert for PKF mitigation.
|
| 20 |
-
- Plasticity Pathway: task-specific experts with classification and auxiliary contrastive learning for IKF mitigation.
|
| 21 |
-
- Hierarchical routing with an inner router over task-specific experts and an outer router balancing alignment and plasticity outputs.
|
| 22 |
-
|
| 23 |
-
## Installation
|
| 24 |
-
|
| 25 |
-
### Setup Environment
|
| 26 |
-
|
| 27 |
-
```bash
|
| 28 |
-
git clone https://github.com/RL-MIND/DFA-MoE
|
| 29 |
-
cd DFA_MoE
|
| 30 |
-
pip install -r requirements.txt
|
| 31 |
-
```
|
| 32 |
-
|
| 33 |
-
## Supported Downstream Datasets
|
| 34 |
-
|
| 35 |
-
The current `configs/class` directory provides downstream class-incremental configs for the following dataset indices. These indices are the values used in `+train_dataset=[id]`.
|
| 36 |
-
|
| 37 |
-
- `0`: `FGVCAircraft`
|
| 38 |
-
- `1`: `Caltech101`
|
| 39 |
-
- `2`: `CIFAR100`
|
| 40 |
-
- `3`: `DescribableTextures`
|
| 41 |
-
- `4`: `EuroSAT`
|
| 42 |
-
- `5`: `OxfordFlowers`
|
| 43 |
-
- `6`: `Food101`
|
| 44 |
-
- `7`: `MNIST`
|
| 45 |
-
- `8`: `OxfordPets`
|
| 46 |
-
- `9`: `StanfordCars`
|
| 47 |
-
- `10`: `SUN397`
|
| 48 |
-
- `11`: `Country211`
|
| 49 |
-
- `14`: `GTSRB`
|
| 50 |
-
- `15`: `RESISC45`
|
| 51 |
-
- `16`: `FER2013`
|
| 52 |
-
- `17`: `UCF101`
|
| 53 |
-
- `18`: `CIFAR10`
|
| 54 |
-
- `19`: `STL10`
|
| 55 |
-
- `20`: `VOC2007`
|
| 56 |
-
- `21`: `ImageNetR`
|
| 57 |
-
- `22`: `KittiDistance`
|
| 58 |
-
- `24`: `CLEVRCount`
|
| 59 |
-
|
| 60 |
-
The datasets pool also contains:
|
| 61 |
-
|
| 62 |
-
- `12`: `SST2`
|
| 63 |
-
- `13`: `HatefulMemes`
|
| 64 |
-
- `23`: `PCam`
|
| 65 |
-
|
| 66 |
-
These datasets are not supported as `--downstream-dataset` for `calculate_sim.py` because they are binary classification datasets.
|
| 67 |
-
|
| 68 |
-
## Data Preparation
|
| 69 |
-
|
| 70 |
-
We will soon release all the datasets used in this work at [google drive](https://drive.google.com/drive/folders/1DxD6MixpyTcsLSv8Gah5V0Cp3yMupYZO?usp=sharing).
|
| 71 |
-
|
| 72 |
-
1. Download or prepare the datasets under a common `dataset_root`.
|
| 73 |
-
2. Update `dataset_root` in the command line when launching experiments.
|
| 74 |
-
3. Choose the dataset-specific config from `configs/class/`.
|
| 75 |
-
|
| 76 |
-
Example dataset config files:
|
| 77 |
-
|
| 78 |
-
- `configs/class/eurosat.yaml`
|
| 79 |
-
- `configs/class/flower.yaml`
|
| 80 |
-
- `configs/class/cifar100.yaml`
|
| 81 |
-
- `configs/class/aircraft.yaml`
|
| 82 |
-
|
| 83 |
-
## Quick Start
|
| 84 |
-
|
| 85 |
-
### Train a DFA-CIL Run
|
| 86 |
-
|
| 87 |
-
Example: EuroSAT with 5 class-incremental splits.
|
| 88 |
-
|
| 89 |
-
```bash
|
| 90 |
-
python main.py \
|
| 91 |
-
--config-path ./configs/class \
|
| 92 |
-
--config-name eurosat.yaml \
|
| 93 |
-
dataset_root="/path/to/data" \
|
| 94 |
-
+train_dataset=[4] \
|
| 95 |
-
+cil_splits=[5]
|
| 96 |
-
```
|
| 97 |
-
|
| 98 |
-
Example: OxfordFlowers with 17 splits.
|
| 99 |
-
|
| 100 |
-
```bash
|
| 101 |
-
python main.py \
|
| 102 |
-
--config-path ./configs/class \
|
| 103 |
-
--config-name flower.yaml \
|
| 104 |
-
dataset_root="/path/to/data" \
|
| 105 |
-
+train_dataset=[5] \
|
| 106 |
-
+cil_splits=[17]
|
| 107 |
-
```
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
## Important Configuration Options
|
| 111 |
-
|
| 112 |
-
This project uses Hydra-based configuration. Key parameters include:
|
| 113 |
-
|
| 114 |
-
```yaml
|
| 115 |
-
model_name: "ViT-B/16"
|
| 116 |
-
prompt_template: "a bad photo of a {}."
|
| 117 |
-
|
| 118 |
-
batch_size: 128
|
| 119 |
-
weight_decay: 0.0
|
| 120 |
-
ls: 0.0
|
| 121 |
-
|
| 122 |
-
epochs_a: 1
|
| 123 |
-
epochs_b: 1
|
| 124 |
-
|
| 125 |
-
lr_e1: 1.0e-5
|
| 126 |
-
lr_e2: 5.0e-4
|
| 127 |
-
text_lr_e2: 1.0e-5
|
| 128 |
-
lr_e2_router: 1.0e-3
|
| 129 |
-
lr_top_router: 1.0e-5
|
| 130 |
-
|
| 131 |
-
tau_con: 0.15
|
| 132 |
-
tau_b_con: 0.17
|
| 133 |
-
lambda_b_con: 0.001
|
| 134 |
-
|
| 135 |
-
num_task_experts: 2
|
| 136 |
-
e2_top_k: 2
|
| 137 |
-
moco_queue_size: 128
|
| 138 |
-
```
|
| 139 |
-
|
| 140 |
-
## Pre-task Zero-shot Baseline
|
| 141 |
-
|
| 142 |
-
To compute SCR correctly, the metric log must contain the original CLIP zero-shot baseline `A_k^0`. Enable this by setting:
|
| 143 |
-
|
| 144 |
-
```yaml
|
| 145 |
-
pre_task_zero_shot_eval: true
|
| 146 |
-
zero_shot_eval: true
|
| 147 |
-
```
|
| 148 |
-
|
| 149 |
-
or from the command line:
|
| 150 |
-
|
| 151 |
-
```bash
|
| 152 |
-
python main.py \
|
| 153 |
-
--config-path ./configs/class \
|
| 154 |
-
--config-name eurosat.yaml \
|
| 155 |
-
dataset_root="/path/to/data" \
|
| 156 |
-
+train_dataset=[4] \
|
| 157 |
-
+cil_splits=[5] \
|
| 158 |
-
pre_task_zero_shot_eval=true \
|
| 159 |
-
zero_shot_eval=true
|
| 160 |
-
```
|
| 161 |
-
|
| 162 |
-
This writes a `task: -1` entry with `zs_pre` into `metrics.json`, which is required by `calculate_SCR.py`.
|
| 163 |
-
|
| 164 |
-
## SCR Evaluation Workflow
|
| 165 |
-
|
| 166 |
-
### Step 1: Run continual learning and save `metrics.json`
|
| 167 |
-
|
| 168 |
-
Run `main.py` with `pre_task_zero_shot_eval=true` and `zero_shot_eval=true`.
|
| 169 |
-
|
| 170 |
-
### Step 2: Generate the similarity matrix
|
| 171 |
-
|
| 172 |
-
Example: EuroSAT with 5 splits.
|
| 173 |
-
|
| 174 |
-
```bash
|
| 175 |
-
python calculate_sim.py \
|
| 176 |
-
--dataset-root "/path/to/data" \
|
| 177 |
-
--downstream-dataset EuroSAT \
|
| 178 |
-
--cil-split 5 \
|
| 179 |
-
--output eurosat_similarity.json
|
| 180 |
-
```
|
| 181 |
-
|
| 182 |
-
This script:
|
| 183 |
-
|
| 184 |
-
- uses the original frozen CLIP model,
|
| 185 |
-
- produces a task-to-upstream similarity matrix for SCR.
|
| 186 |
-
|
| 187 |
-
### Step 3: Compute SCR
|
| 188 |
-
|
| 189 |
-
```bash
|
| 190 |
-
python calculate_SCR.py \
|
| 191 |
-
--metric-path /path/to/metrics.json \
|
| 192 |
-
--sim-json /path/to/eurosat_similarity_sim.json
|
| 193 |
-
```
|
| 194 |
-
|
| 195 |
-
`calculate_SCR.py` now strictly requires:
|
| 196 |
-
|
| 197 |
-
- a `task: -1` row with non-empty `zs_pre`,
|
| 198 |
-
- per-task `zs` results in `metrics.json`,
|
| 199 |
-
- a similarity JSON generated by `calculate_sim.py`.
|
| 200 |
-
|
| 201 |
-
## Reference Script
|
| 202 |
-
|
| 203 |
-
The repository also contains `run.sh` as a reference batch script.
|
| 204 |
-
|
| 205 |
-
## Notes
|
| 206 |
-
|
| 207 |
-
- `main.py`, `calculate_sim.py`, and `calculate_SCR.py` are the recommended entry points for experiments and evaluation.
|
| 208 |
-
|
| 209 |
-
## Citation
|
| 210 |
-
|
| 211 |
-
If you find this repository useful in your research, please cite the paper:
|
| 212 |
-
|
| 213 |
-
```bibtex
|
| 214 |
-
@inproceedings{kang2026dont,
|
| 215 |
-
title={Don't Forget Why You Started: Tackling Dual Forgetting in Vision-Language Continual Learning},
|
| 216 |
-
author={Kang, Borui and Gu, Jinrui and Feng, Tao and Fan, Qi and Shi, Yinghuan and Wang, Lei and Li, Wenbin and Gao, Yang},
|
| 217 |
-
booktitle={Proceedings of the 43rd International Conference on Machine Learning},
|
| 218 |
-
year={2026}
|
| 219 |
-
}
|
| 220 |
-
```
|
| 221 |
-
|
| 222 |
-
## Acknowledgement
|
| 223 |
-
|
| 224 |
-
This repository is built upon and modified from [MoE-Adapters4CL](https://github.com/JiazuoYu/MoE-Adapters4CL). We thank the original authors for making their code publicly available.
|
| 225 |
-
|
| 226 |
-
The dataset processing pipeline in this repository is also based on [DIKI](https://github.com/lloongx/DIKI). We thank the authors for releasing their implementation.
|
| 227 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/calculate_SCR.py
DELETED
|
@@ -1,266 +0,0 @@
|
|
| 1 |
-
from __future__ import annotations
|
| 2 |
-
|
| 3 |
-
import argparse
|
| 4 |
-
import json
|
| 5 |
-
from typing import List, Dict, Any, Tuple
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
DATASET_NAME_ALIASES = {
|
| 9 |
-
"Aircraft": "FGVCAircraft",
|
| 10 |
-
"FGVCAircraft": "FGVCAircraft",
|
| 11 |
-
"DTD": "DescribableTextures",
|
| 12 |
-
"DescribableTextures": "DescribableTextures",
|
| 13 |
-
}
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
def canonicalize_dataset_name(name: str) -> str:
|
| 17 |
-
return DATASET_NAME_ALIASES.get(str(name), str(name))
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
def format_matrix(mat: List[List[float]], decimals: int = 3) -> str:
|
| 21 |
-
if not mat:
|
| 22 |
-
return "[]"
|
| 23 |
-
fmt = f"{{:.{decimals}f}}"
|
| 24 |
-
lines = []
|
| 25 |
-
for row in mat:
|
| 26 |
-
row_str = ", ".join(fmt.format(v) for v in row)
|
| 27 |
-
lines.append(f"[{row_str}]")
|
| 28 |
-
return "[" + "\n" + ",\n".join(lines) + "\n]"
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
def load_lines(path: str) -> List[Dict[str, Any]]:
|
| 32 |
-
rows = []
|
| 33 |
-
with open(path, 'r', encoding='utf-8') as f:
|
| 34 |
-
for line in f:
|
| 35 |
-
line = line.strip()
|
| 36 |
-
if not line:
|
| 37 |
-
continue
|
| 38 |
-
try:
|
| 39 |
-
rows.append(json.loads(line))
|
| 40 |
-
except Exception as e:
|
| 41 |
-
print(f"[WARN] skip invalid JSON line: {e}")
|
| 42 |
-
rows.sort(key=lambda r: int(r.get('task', 0)))
|
| 43 |
-
return rows
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
def build_zs_matrix(rows: List[Dict[str, Any]]) -> Tuple[List[List[float]], List[str]]:
|
| 47 |
-
if not rows:
|
| 48 |
-
return [], []
|
| 49 |
-
|
| 50 |
-
baseline_row = None
|
| 51 |
-
task_rows = []
|
| 52 |
-
|
| 53 |
-
for r in rows:
|
| 54 |
-
task_id = int(r.get('task', 0))
|
| 55 |
-
zs_pre = r.get('zs_pre')
|
| 56 |
-
zs_task = r.get('zs')
|
| 57 |
-
zs_task_alt = r.get('zs_mtil')
|
| 58 |
-
|
| 59 |
-
if task_id == -1 and isinstance(zs_pre, dict) and len(zs_pre) > 0:
|
| 60 |
-
baseline_row = zs_pre
|
| 61 |
-
continue
|
| 62 |
-
if task_id >= 0:
|
| 63 |
-
if isinstance(zs_task, dict) and len(zs_task) > 0:
|
| 64 |
-
task_rows.append((task_id, zs_task))
|
| 65 |
-
elif isinstance(zs_task_alt, dict) and len(zs_task_alt) > 0:
|
| 66 |
-
task_rows.append((task_id, zs_task_alt))
|
| 67 |
-
|
| 68 |
-
if baseline_row is None:
|
| 69 |
-
raise ValueError(
|
| 70 |
-
"metric.json must contain a pre-task baseline row with `task: -1` and non-empty `zs_pre`."
|
| 71 |
-
)
|
| 72 |
-
if not task_rows:
|
| 73 |
-
return [], []
|
| 74 |
-
|
| 75 |
-
names = list(baseline_row.keys())
|
| 76 |
-
matrix: List[List[float]] = []
|
| 77 |
-
matrix.append([float(baseline_row.get(name, 0.0)) for name in names])
|
| 78 |
-
for _task_id, zs in sorted(task_rows, key=lambda item: item[0]):
|
| 79 |
-
row = [float(zs.get(name, 0.0)) for name in names]
|
| 80 |
-
matrix.append(row)
|
| 81 |
-
return matrix, names
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
def load_similarity_json(path: str) -> Tuple[List[List[float]], List[str]]:
|
| 85 |
-
with open(path, 'r', encoding='utf-8') as f:
|
| 86 |
-
payload = json.load(f)
|
| 87 |
-
sim_mat_raw = payload.get("similarity_matrix", [])
|
| 88 |
-
sim_names_raw = payload.get("upstream_names", [])
|
| 89 |
-
sim_mat = [[float(x) for x in row] for row in sim_mat_raw]
|
| 90 |
-
sim_names = [canonicalize_dataset_name(x) for x in sim_names_raw]
|
| 91 |
-
return sim_mat, sim_names
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
def compute_scores_with_matrix(
|
| 95 |
-
zs_mat: List[List[float]],
|
| 96 |
-
zs_names: List[str],
|
| 97 |
-
sim_mat: List[List[float]],
|
| 98 |
-
sim_names: List[str],
|
| 99 |
-
) -> List[float]:
|
| 100 |
-
if not zs_mat or not sim_mat:
|
| 101 |
-
return []
|
| 102 |
-
|
| 103 |
-
zs_names = [canonicalize_dataset_name(nm) for nm in zs_names]
|
| 104 |
-
zs_name_to_idx = {nm: i for i, nm in enumerate(zs_names)}
|
| 105 |
-
sim_name_to_idx = {nm: i for i, nm in enumerate(sim_names)}
|
| 106 |
-
common = [nm for nm in zs_names if nm in sim_name_to_idx]
|
| 107 |
-
if len(common) < len(zs_names):
|
| 108 |
-
missing = [nm for nm in zs_names if nm not in sim_name_to_idx]
|
| 109 |
-
print(f"[WARN] Missing similarity columns for: {missing}")
|
| 110 |
-
zs_cols = [zs_name_to_idx[nm] for nm in common]
|
| 111 |
-
sim_cols = [sim_name_to_idx[nm] for nm in common]
|
| 112 |
-
|
| 113 |
-
base = zs_mat[0]
|
| 114 |
-
scores: List[float] = []
|
| 115 |
-
T = min(len(sim_mat), max(0, len(zs_mat) - 1))
|
| 116 |
-
if T < (len(zs_mat) - 1):
|
| 117 |
-
print(f"[WARN] sim-matrix rows ({len(sim_mat)}) < tasks ({len(zs_mat) - 1}); truncating to {T}")
|
| 118 |
-
for t in range(T):
|
| 119 |
-
row = zs_mat[t + 1]
|
| 120 |
-
total = 0.0
|
| 121 |
-
sum_w = 0.0
|
| 122 |
-
sims = [float(sim_mat[t][sim_cols[k]]) for k in range(len(common))]
|
| 123 |
-
risks = [max(0.0, 1-s) for s in sims]
|
| 124 |
-
norm = sum(risks) if risks else 0.0
|
| 125 |
-
default_weight = 1.0 / max(1, len(common))
|
| 126 |
-
for k in range(len(common)):
|
| 127 |
-
i = zs_cols[k]
|
| 128 |
-
diff = float(row[i]) - float(base[i])
|
| 129 |
-
w = (risks[k] / norm) if norm > 0.0 else default_weight
|
| 130 |
-
total += diff * w
|
| 131 |
-
sum_w += w
|
| 132 |
-
avg = (total / sum_w) if sum_w > 0.0 else 0.0
|
| 133 |
-
scores.append(round(avg, 2))
|
| 134 |
-
return scores
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
def compute_scores_grouped(
|
| 138 |
-
zs_mat: List[List[float]],
|
| 139 |
-
zs_names: List[str],
|
| 140 |
-
sim_mat: List[List[float]],
|
| 141 |
-
sim_names: List[str],
|
| 142 |
-
) -> Dict[str, Any]:
|
| 143 |
-
if not zs_mat or not sim_mat:
|
| 144 |
-
return {
|
| 145 |
-
"weighted": {"low": [], "mid": [], "high": []},
|
| 146 |
-
}
|
| 147 |
-
|
| 148 |
-
zs_names = [canonicalize_dataset_name(nm) for nm in zs_names]
|
| 149 |
-
zs_name_to_idx = {nm: i for i, nm in enumerate(zs_names)}
|
| 150 |
-
sim_name_to_idx = {nm: i for i, nm in enumerate(sim_names)}
|
| 151 |
-
common = [nm for nm in zs_names if nm in sim_name_to_idx]
|
| 152 |
-
if len(common) < len(zs_names):
|
| 153 |
-
missing = [nm for nm in zs_names if nm not in sim_name_to_idx]
|
| 154 |
-
print(f"[WARN] Missing similarity columns for grouped score: {missing}")
|
| 155 |
-
zs_cols = [zs_name_to_idx[nm] for nm in common]
|
| 156 |
-
sim_cols = [sim_name_to_idx[nm] for nm in common]
|
| 157 |
-
|
| 158 |
-
base = zs_mat[0]
|
| 159 |
-
T = min(len(sim_mat), max(0, len(zs_mat) - 1))
|
| 160 |
-
if T < (len(zs_mat) - 1):
|
| 161 |
-
print(f"[WARN] sim-matrix rows ({len(sim_mat)}) < tasks ({len(zs_mat) - 1}); truncating to {T}")
|
| 162 |
-
|
| 163 |
-
def calc_weighted(items):
|
| 164 |
-
total = sum(item["diff"] * item["weight"] for item in items)
|
| 165 |
-
sum_w = sum(item["weight"] for item in items)
|
| 166 |
-
return round((total / sum_w) if sum_w > 0.0 else 0.0, 2)
|
| 167 |
-
|
| 168 |
-
w_low, w_mid, w_high = [], [], []
|
| 169 |
-
|
| 170 |
-
for t in range(T):
|
| 171 |
-
row = zs_mat[t + 1]
|
| 172 |
-
sims = [float(sim_mat[t][sim_cols[k]]) for k in range(len(common))]
|
| 173 |
-
risks = [max(0.0, 1.0 - s) for s in sims]
|
| 174 |
-
norm = sum(risks) if risks else 0.0
|
| 175 |
-
default_weight = 1.0 / max(1, len(common))
|
| 176 |
-
triplets = []
|
| 177 |
-
for k in range(len(common)):
|
| 178 |
-
i = zs_cols[k]
|
| 179 |
-
sim_val = sims[k]
|
| 180 |
-
diff = float(row[i]) - float(base[i])
|
| 181 |
-
w = (risks[k] / norm) if norm > 0.0 else default_weight
|
| 182 |
-
triplets.append({
|
| 183 |
-
"sim": sim_val,
|
| 184 |
-
"diff": diff,
|
| 185 |
-
"weight": w,
|
| 186 |
-
"index": i + 1,
|
| 187 |
-
})
|
| 188 |
-
triplets.sort(key=lambda x: x["sim"])
|
| 189 |
-
n = len(triplets)
|
| 190 |
-
q = n // 3
|
| 191 |
-
low = triplets[:q]
|
| 192 |
-
mid = triplets[q:2 * q]
|
| 193 |
-
high = triplets[2 * q:]
|
| 194 |
-
|
| 195 |
-
w_low.append(calc_weighted(low))
|
| 196 |
-
w_mid.append(calc_weighted(mid))
|
| 197 |
-
w_high.append(calc_weighted(high))
|
| 198 |
-
|
| 199 |
-
return {
|
| 200 |
-
"weighted": {"low": w_low, "mid": w_mid, "high": w_high},
|
| 201 |
-
}
|
| 202 |
-
|
| 203 |
-
|
| 204 |
-
def main() -> None:
|
| 205 |
-
parser = argparse.ArgumentParser(
|
| 206 |
-
description="Compute SCR metrics from metric.json and calculate_sim similarity JSON.",
|
| 207 |
-
)
|
| 208 |
-
parser.add_argument("--metric-path", type=str, default="metric.json", help="Path to the metric.json file")
|
| 209 |
-
parser.add_argument("--sim-json", type=str, required=True, help="Path to the similarity JSON generated by calculate_sim.py")
|
| 210 |
-
parser.add_argument("--zs-decimals", type=int, default=2, help="Decimals when printing ZS matrix")
|
| 211 |
-
args = parser.parse_args()
|
| 212 |
-
|
| 213 |
-
rows = load_lines(args.metric_path)
|
| 214 |
-
try:
|
| 215 |
-
zs_mat, zs_names = build_zs_matrix(rows)
|
| 216 |
-
except ValueError as e:
|
| 217 |
-
print(f"[ERROR] {e}")
|
| 218 |
-
return
|
| 219 |
-
sim_mat, sim_names = load_similarity_json(args.sim_json)
|
| 220 |
-
|
| 221 |
-
if not zs_mat:
|
| 222 |
-
print("[ERROR] No zero-shot matrix could be extracted from metric.json.")
|
| 223 |
-
return
|
| 224 |
-
if not sim_mat:
|
| 225 |
-
print("[ERROR] No similarity matrix could be extracted from the calculate_sim JSON.")
|
| 226 |
-
return
|
| 227 |
-
|
| 228 |
-
num_cols = len(zs_mat[0]) if zs_mat else 0
|
| 229 |
-
if any(len(row) != num_cols for row in zs_mat):
|
| 230 |
-
print("[ERROR] Extracted zero-shot matrix rows must all have the same length.")
|
| 231 |
-
return
|
| 232 |
-
|
| 233 |
-
sim_cols = len(sim_mat[0]) if sim_mat else 0
|
| 234 |
-
if any(len(row) != sim_cols for row in sim_mat):
|
| 235 |
-
print("[ERROR] similarity_matrix rows in the calculate_sim JSON must all have the same length.")
|
| 236 |
-
return
|
| 237 |
-
if sim_names and len(sim_names) != sim_cols:
|
| 238 |
-
print("[ERROR] upstream_names length must match similarity_matrix columns.")
|
| 239 |
-
return
|
| 240 |
-
if not sim_names:
|
| 241 |
-
sim_names = zs_names
|
| 242 |
-
|
| 243 |
-
print("1. Zero-shot matrix (baseline + tasks):")
|
| 244 |
-
print(format_matrix(zs_mat, decimals=args.zs_decimals))
|
| 245 |
-
|
| 246 |
-
scores_weighted = compute_scores_with_matrix(zs_mat, zs_names, sim_mat, sim_names)
|
| 247 |
-
avg_weighted = round(sum(scores_weighted) / len(scores_weighted), 2) if scores_weighted else 0.0
|
| 248 |
-
print("\nSCR:")
|
| 249 |
-
print(avg_weighted)
|
| 250 |
-
|
| 251 |
-
grouped = compute_scores_grouped(zs_mat, zs_names, sim_mat, sim_names)
|
| 252 |
-
|
| 253 |
-
def mean_or_zero(arr: list) -> float:
|
| 254 |
-
return round(sum(arr) / len(arr), 2) if arr else 0.0
|
| 255 |
-
|
| 256 |
-
weighted = grouped["weighted"]
|
| 257 |
-
low_w_mean = mean_or_zero(weighted["low"])
|
| 258 |
-
mid_w_mean = mean_or_zero(weighted["mid"])
|
| 259 |
-
high_w_mean = mean_or_zero(weighted["high"])
|
| 260 |
-
|
| 261 |
-
print("\nLow-Similarity SCR:", low_w_mean)
|
| 262 |
-
print("Mid-Similarity SCR:", mid_w_mean)
|
| 263 |
-
print("High-Similarity SCR:", high_w_mean)
|
| 264 |
-
|
| 265 |
-
if __name__ == "__main__":
|
| 266 |
-
main()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/calculate_sim.py
DELETED
|
@@ -1,456 +0,0 @@
|
|
| 1 |
-
import argparse
|
| 2 |
-
import os
|
| 3 |
-
from typing import List, Dict, Tuple
|
| 4 |
-
|
| 5 |
-
import numpy as np
|
| 6 |
-
import torch
|
| 7 |
-
from omegaconf import OmegaConf
|
| 8 |
-
|
| 9 |
-
# Project-local imports
|
| 10 |
-
from mtil_datasets import get_dataset as get_mtil_dataset
|
| 11 |
-
from continual_clip.mtil_cil import build_mtil_cil_scenarios
|
| 12 |
-
from continual_clip.clip_original import load as load_orig_clip, tokenize as tokenize_orig
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
DEFAULT_NAMES_ORDER1 = [
|
| 16 |
-
"FGVCAircraft", "Caltech101", "CIFAR100", "DescribableTextures", "EuroSAT",
|
| 17 |
-
"OxfordFlowers", "Food101", "MNIST", "OxfordPets", "StanfordCars",
|
| 18 |
-
"SUN397", "Country211", "SST2", "HatefulMemes", "GTSRB",
|
| 19 |
-
"RESISC45", "FER2013", "UCF101", "CIFAR10", "STL10",
|
| 20 |
-
"VOC2007", "ImageNetR", "KittiDistance", "PCam", "CLEVRCount",
|
| 21 |
-
]
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
DATASET_NAME_TO_CONFIG_STEM = {
|
| 25 |
-
"FGVCAircraft": "aircraft",
|
| 26 |
-
"Aircraft": "aircraft",
|
| 27 |
-
"Caltech101": "caltech",
|
| 28 |
-
"CIFAR100": "cifar100",
|
| 29 |
-
"DescribableTextures": "dtd",
|
| 30 |
-
"DTD": "dtd",
|
| 31 |
-
"EuroSAT": "eurosat",
|
| 32 |
-
"OxfordFlowers": "flower",
|
| 33 |
-
"Food101": "food",
|
| 34 |
-
"MNIST": "mnist",
|
| 35 |
-
"OxfordPets": "pet",
|
| 36 |
-
"StanfordCars": "car",
|
| 37 |
-
"SUN397": "sun",
|
| 38 |
-
"Country211": "conuntry",
|
| 39 |
-
"GTSRB": "gtsrb",
|
| 40 |
-
"RESISC45": "resisc45",
|
| 41 |
-
"FER2013": "fer2013",
|
| 42 |
-
"UCF101": "ucf",
|
| 43 |
-
"CIFAR10": "cifar10",
|
| 44 |
-
"STL10": "stl",
|
| 45 |
-
"VOC2007": "voc",
|
| 46 |
-
"ImageNetR": "image-r",
|
| 47 |
-
"KittiDistance": "kitti",
|
| 48 |
-
"CLEVRCount": "clevr",
|
| 49 |
-
}
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
UNSUPPORTED_DOWNSTREAM_DATASETS = {
|
| 53 |
-
"SST2",
|
| 54 |
-
"HatefulMemes",
|
| 55 |
-
"PCam",
|
| 56 |
-
}
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
def resolve_downstream_seed(repo_root: str, downstream_name: str) -> int:
|
| 60 |
-
if downstream_name in UNSUPPORTED_DOWNSTREAM_DATASETS:
|
| 61 |
-
raise ValueError(
|
| 62 |
-
f"Downstream dataset '{downstream_name}' is currently unsupported as `--downstream-dataset` because no matching `configs/class/*.yaml` file is available."
|
| 63 |
-
)
|
| 64 |
-
config_stem = DATASET_NAME_TO_CONFIG_STEM.get(downstream_name)
|
| 65 |
-
if not config_stem:
|
| 66 |
-
raise ValueError(
|
| 67 |
-
f"No `configs/class/*.yaml` mapping is defined for downstream dataset '{downstream_name}'."
|
| 68 |
-
)
|
| 69 |
-
config_path = os.path.join(repo_root, "configs", "class", f"{config_stem}.yaml")
|
| 70 |
-
if not os.path.exists(config_path):
|
| 71 |
-
raise FileNotFoundError(
|
| 72 |
-
f"Expected downstream config file '{config_path}' for dataset '{downstream_name}'."
|
| 73 |
-
)
|
| 74 |
-
cfg = OmegaConf.load(config_path)
|
| 75 |
-
seed = OmegaConf.select(cfg, "seed")
|
| 76 |
-
if seed is None:
|
| 77 |
-
raise ValueError(
|
| 78 |
-
f"Config file '{config_path}' does not define a `seed` entry."
|
| 79 |
-
)
|
| 80 |
-
return int(seed)
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
def encode_texts(model, device, sentences: List[str], batch_size: int = 256) -> torch.Tensor:
|
| 84 |
-
feats_all = []
|
| 85 |
-
with torch.no_grad():
|
| 86 |
-
for i in range(0, len(sentences), batch_size):
|
| 87 |
-
chunk = sentences[i:i+batch_size]
|
| 88 |
-
tokens = tokenize_orig(chunk).to(device)
|
| 89 |
-
feats = model.encode_text(tokens)
|
| 90 |
-
feats = feats / (feats.norm(dim=-1, keepdim=True) + 1e-12)
|
| 91 |
-
feats_all.append(feats)
|
| 92 |
-
if not feats_all:
|
| 93 |
-
return torch.zeros((0, model.text_projection.shape[1]), device=device)
|
| 94 |
-
return torch.cat(feats_all, dim=0)
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
def build_text_class_prototypes(model, device, classnames: List[str], templates, batch_size: int = 256) -> torch.Tensor:
|
| 98 |
-
D = model.text_projection.shape[1]
|
| 99 |
-
class_vecs: List[torch.Tensor] = []
|
| 100 |
-
for cname in classnames:
|
| 101 |
-
sents: List[str] = []
|
| 102 |
-
if templates and len(templates) > 0:
|
| 103 |
-
for t in templates:
|
| 104 |
-
try:
|
| 105 |
-
sents.append(t(cname) if callable(t) else str(t).format(cname))
|
| 106 |
-
except Exception:
|
| 107 |
-
continue
|
| 108 |
-
else:
|
| 109 |
-
sents = [f"a photo of a {cname}."]
|
| 110 |
-
feats = encode_texts(model, device, sents, batch_size=batch_size)
|
| 111 |
-
if feats.numel() == 0:
|
| 112 |
-
v = torch.zeros(D, device=device)
|
| 113 |
-
else:
|
| 114 |
-
v = feats.mean(dim=0)
|
| 115 |
-
v = v / (v.norm() + 1e-12)
|
| 116 |
-
class_vecs.append(v)
|
| 117 |
-
if not class_vecs:
|
| 118 |
-
return torch.zeros((0, D), device=device)
|
| 119 |
-
return torch.stack(class_vecs, dim=0)
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
def normalize_labels_to_list(lab) -> List[int]:
|
| 123 |
-
if isinstance(lab, (int, np.integer)):
|
| 124 |
-
return [int(lab)]
|
| 125 |
-
import torch as _torch
|
| 126 |
-
if _torch.is_tensor(lab):
|
| 127 |
-
arr = lab.detach().cpu().numpy()
|
| 128 |
-
if arr.ndim == 0:
|
| 129 |
-
return [int(arr)]
|
| 130 |
-
if arr.ndim == 1 and arr.size > 1 and set(np.unique(arr)).issubset({0,1}):
|
| 131 |
-
return [int(x) for x in np.where(arr > 0.5)[0].tolist()]
|
| 132 |
-
return [int(x) for x in arr.flatten().tolist()]
|
| 133 |
-
if isinstance(lab, (list, tuple, np.ndarray)):
|
| 134 |
-
arr = np.asarray(lab)
|
| 135 |
-
if arr.ndim == 0:
|
| 136 |
-
return [int(arr)]
|
| 137 |
-
if arr.ndim == 1 and arr.size > 1 and set(np.unique(arr)).issubset({0,1}):
|
| 138 |
-
return [int(x) for x in np.where(arr > 0.5)[0].tolist()]
|
| 139 |
-
if arr.ndim == 1:
|
| 140 |
-
return [int(x) for x in arr.tolist()]
|
| 141 |
-
return [int(x) for x in np.where(arr.flatten() > 0.5)[0].tolist()]
|
| 142 |
-
return []
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
def build_visual_class_prototypes(model, device, ds_wrapper, num_classes: int, max_per_class: int, batch_size: int, preprocess_eval) -> torch.Tensor:
|
| 146 |
-
D = None
|
| 147 |
-
sums: Dict[int, torch.Tensor] = {}
|
| 148 |
-
counts: Dict[int, int] = {i: 0 for i in range(num_classes)}
|
| 149 |
-
imgs_batch: List[torch.Tensor] = []
|
| 150 |
-
labels_batch_multi: List[List[int]] = []
|
| 151 |
-
with torch.no_grad():
|
| 152 |
-
for i in range(len(ds_wrapper)):
|
| 153 |
-
img, lab = ds_wrapper[i]
|
| 154 |
-
lab_ids = [lid for lid in normalize_labels_to_list(lab) if 0 <= lid < num_classes and counts[lid] < max_per_class]
|
| 155 |
-
if not lab_ids:
|
| 156 |
-
continue
|
| 157 |
-
if isinstance(img, torch.Tensor):
|
| 158 |
-
tensor_img = img
|
| 159 |
-
else:
|
| 160 |
-
tensor_img = preprocess_eval(img)
|
| 161 |
-
imgs_batch.append(tensor_img.unsqueeze(0))
|
| 162 |
-
labels_batch_multi.append(lab_ids)
|
| 163 |
-
if len(imgs_batch) >= max(1, batch_size):
|
| 164 |
-
batch = torch.cat(imgs_batch, dim=0).to(device)
|
| 165 |
-
feats = model.encode_image(batch)
|
| 166 |
-
feats = feats / (feats.norm(dim=-1, keepdim=True) + 1e-12)
|
| 167 |
-
if D is None:
|
| 168 |
-
D = int(feats.shape[1])
|
| 169 |
-
for f, ls in zip(feats, labels_batch_multi):
|
| 170 |
-
for l in ls:
|
| 171 |
-
if counts[l] >= max_per_class:
|
| 172 |
-
continue
|
| 173 |
-
if l not in sums:
|
| 174 |
-
sums[l] = f.detach().clone()
|
| 175 |
-
else:
|
| 176 |
-
sums[l] = sums[l] + f.detach()
|
| 177 |
-
counts[l] += 1
|
| 178 |
-
imgs_batch.clear()
|
| 179 |
-
labels_batch_multi.clear()
|
| 180 |
-
if all(counts[l] >= max_per_class for l in range(num_classes)):
|
| 181 |
-
break
|
| 182 |
-
if imgs_batch:
|
| 183 |
-
batch = torch.cat(imgs_batch, dim=0).to(device)
|
| 184 |
-
feats = model.encode_image(batch)
|
| 185 |
-
feats = feats / (feats.norm(dim=-1, keepdim=True) + 1e-12)
|
| 186 |
-
if D is None:
|
| 187 |
-
D = int(feats.shape[1])
|
| 188 |
-
for f, ls in zip(feats, labels_batch_multi):
|
| 189 |
-
for l in ls:
|
| 190 |
-
if counts[l] >= max_per_class:
|
| 191 |
-
continue
|
| 192 |
-
if l not in sums:
|
| 193 |
-
sums[l] = f.detach().clone()
|
| 194 |
-
else:
|
| 195 |
-
sums[l] = sums[l] + f.detach()
|
| 196 |
-
counts[l] += 1
|
| 197 |
-
imgs_batch.clear()
|
| 198 |
-
labels_batch_multi.clear()
|
| 199 |
-
class_vecs: List[torch.Tensor] = []
|
| 200 |
-
for lid in range(num_classes):
|
| 201 |
-
c = counts.get(lid, 0)
|
| 202 |
-
if c <= 0:
|
| 203 |
-
class_vecs.append(torch.zeros(int(D or 0), device=device))
|
| 204 |
-
else:
|
| 205 |
-
m = sums[lid] / float(c)
|
| 206 |
-
m = m / (m.norm() + 1e-12)
|
| 207 |
-
class_vecs.append(m)
|
| 208 |
-
if not class_vecs:
|
| 209 |
-
return torch.zeros((0, int(D or 0)), device=device)
|
| 210 |
-
return torch.stack(class_vecs, dim=0)
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
def build_visual_class_prototypes_subset(model, device, ds_wrapper, label_ids: List[int], max_per_class: int, batch_size: int, preprocess_eval) -> torch.Tensor:
|
| 214 |
-
D = None
|
| 215 |
-
sums: Dict[int, torch.Tensor] = {}
|
| 216 |
-
counts: Dict[int, int] = {lid: 0 for lid in label_ids}
|
| 217 |
-
label_set = set(label_ids)
|
| 218 |
-
imgs_batch: List[torch.Tensor] = []
|
| 219 |
-
labels_batch_multi: List[List[int]] = []
|
| 220 |
-
with torch.no_grad():
|
| 221 |
-
for i in range(len(ds_wrapper)):
|
| 222 |
-
img, lab = ds_wrapper[i]
|
| 223 |
-
lab_all = [lid for lid in normalize_labels_to_list(lab) if lid in label_set and counts.get(lid, 0) < max_per_class]
|
| 224 |
-
if not lab_all:
|
| 225 |
-
continue
|
| 226 |
-
tensor_img = img if isinstance(img, torch.Tensor) else preprocess_eval(img)
|
| 227 |
-
imgs_batch.append(tensor_img.unsqueeze(0))
|
| 228 |
-
labels_batch_multi.append(lab_all)
|
| 229 |
-
if len(imgs_batch) >= max(1, batch_size):
|
| 230 |
-
batch = torch.cat(imgs_batch, dim=0).to(device)
|
| 231 |
-
feats = model.encode_image(batch)
|
| 232 |
-
feats = feats / (feats.norm(dim=-1, keepdim=True) + 1e-12)
|
| 233 |
-
if D is None:
|
| 234 |
-
D = int(feats.shape[1])
|
| 235 |
-
for f, ls in zip(feats, labels_batch_multi):
|
| 236 |
-
for l in ls:
|
| 237 |
-
if counts[l] >= max_per_class:
|
| 238 |
-
continue
|
| 239 |
-
if l not in sums:
|
| 240 |
-
sums[l] = f.detach().clone()
|
| 241 |
-
else:
|
| 242 |
-
sums[l] = sums[l] + f.detach()
|
| 243 |
-
counts[l] += 1
|
| 244 |
-
imgs_batch.clear()
|
| 245 |
-
labels_batch_multi.clear()
|
| 246 |
-
if all(counts[lid] >= max_per_class for lid in label_ids):
|
| 247 |
-
break
|
| 248 |
-
if imgs_batch:
|
| 249 |
-
batch = torch.cat(imgs_batch, dim=0).to(device)
|
| 250 |
-
feats = model.encode_image(batch)
|
| 251 |
-
feats = feats / (feats.norm(dim=-1, keepdim=True) + 1e-12)
|
| 252 |
-
if D is None:
|
| 253 |
-
D = int(feats.shape[1])
|
| 254 |
-
for f, ls in zip(feats, labels_batch_multi):
|
| 255 |
-
for l in ls:
|
| 256 |
-
if counts[l] >= max_per_class:
|
| 257 |
-
continue
|
| 258 |
-
if l not in sums:
|
| 259 |
-
sums[l] = f.detach().clone()
|
| 260 |
-
else:
|
| 261 |
-
sums[l] = sums[l] + f.detach()
|
| 262 |
-
counts[l] += 1
|
| 263 |
-
imgs_batch.clear()
|
| 264 |
-
labels_batch_multi.clear()
|
| 265 |
-
class_vecs: List[torch.Tensor] = []
|
| 266 |
-
for lid in label_ids:
|
| 267 |
-
c = counts.get(lid, 0)
|
| 268 |
-
if c <= 0:
|
| 269 |
-
class_vecs.append(torch.zeros(int(D or 0), device=device))
|
| 270 |
-
else:
|
| 271 |
-
m = sums[lid] / float(c)
|
| 272 |
-
m = m / (m.norm() + 1e-12)
|
| 273 |
-
class_vecs.append(m)
|
| 274 |
-
if not class_vecs:
|
| 275 |
-
return torch.zeros((0, int(D or 0)), device=device)
|
| 276 |
-
return torch.stack(class_vecs, dim=0)
|
| 277 |
-
|
| 278 |
-
|
| 279 |
-
def cosine_distance_matrix(A: torch.Tensor, B: torch.Tensor) -> torch.Tensor:
|
| 280 |
-
S = (A @ B.t()).clamp(-1.0, 1.0)
|
| 281 |
-
return 1.0 - S
|
| 282 |
-
|
| 283 |
-
|
| 284 |
-
def directed_covering_distance(T_src: torch.Tensor, V_src: torch.Tensor,
|
| 285 |
-
T_tgt: torch.Tensor, V_tgt: torch.Tensor) -> float:
|
| 286 |
-
if V_src.size(0) == 0 or V_tgt.size(0) == 0:
|
| 287 |
-
return 0.0
|
| 288 |
-
F_src = torch.cat([V_src, T_src], dim=1)
|
| 289 |
-
F_tgt = torch.cat([V_tgt, T_tgt], dim=1)
|
| 290 |
-
F_src = F_src / (F_src.norm(dim=-1, keepdim=True) + 1e-12)
|
| 291 |
-
F_tgt = F_tgt / (F_tgt.norm(dim=-1, keepdim=True) + 1e-12)
|
| 292 |
-
C = cosine_distance_matrix(F_src, F_tgt)
|
| 293 |
-
d_row_min = C.min(dim=1).values
|
| 294 |
-
return float(d_row_min.mean().item())
|
| 295 |
-
|
| 296 |
-
|
| 297 |
-
def tasks_to_upstream_similarity_matrix(vision_model, device,
|
| 298 |
-
upstream_names: List[str], name_to_idx: Dict[str, int],
|
| 299 |
-
dataset_list, classes_names_list, templates_list,
|
| 300 |
-
downstream_name: str, cil_splits: int,
|
| 301 |
-
preprocess_eval,
|
| 302 |
-
max_images_per_class: int, vision_batch_size: int,
|
| 303 |
-
dataset_root: str,
|
| 304 |
-
beta: float,
|
| 305 |
-
clip_text_model=None,
|
| 306 |
-
seed: int = 32) -> Tuple[np.ndarray, List[str]]:
|
| 307 |
-
T_up_list: List[torch.Tensor] = []
|
| 308 |
-
V_up_list: List[torch.Tensor] = []
|
| 309 |
-
for nm in upstream_names:
|
| 310 |
-
i = name_to_idx[nm]
|
| 311 |
-
classnames = classes_names_list[i]
|
| 312 |
-
templates = templates_list[i]
|
| 313 |
-
ds_wrapper = dataset_list[i]
|
| 314 |
-
K = len(classnames)
|
| 315 |
-
T_k = build_text_class_prototypes(clip_text_model, device, classnames, templates, batch_size=256)
|
| 316 |
-
V_k = build_visual_class_prototypes(vision_model, device, ds_wrapper, K, max_per_class=int(max_images_per_class), batch_size=int(vision_batch_size), preprocess_eval=preprocess_eval)
|
| 317 |
-
T_up_list.append(T_k)
|
| 318 |
-
V_up_list.append(V_k)
|
| 319 |
-
|
| 320 |
-
ds_idx = name_to_idx[downstream_name]
|
| 321 |
-
cfg_ds = type("Cfg", (), {})()
|
| 322 |
-
cfg_ds.dataset = "MTIL"
|
| 323 |
-
cfg_ds.dataset_root = dataset_root
|
| 324 |
-
cfg_ds.MTIL_order_2 = False
|
| 325 |
-
cfg_ds.train_one_dataset = ds_idx
|
| 326 |
-
cfg_ds.seed = int(seed)
|
| 327 |
-
cfg_ds.use_validation = False
|
| 328 |
-
train_list, train_classes_names, train_templates, _ = get_mtil_dataset(
|
| 329 |
-
cfg_ds, 'train', transforms=preprocess_eval
|
| 330 |
-
)
|
| 331 |
-
test_list, _, _, _ = get_mtil_dataset(
|
| 332 |
-
cfg_ds, 'test', transforms=preprocess_eval
|
| 333 |
-
)
|
| 334 |
-
assert len(train_list) == 1 and len(test_list) == 1, "Expected single selected dataset for downstream"
|
| 335 |
-
classnames_single = train_classes_names[0]
|
| 336 |
-
templates_single = train_templates[0]
|
| 337 |
-
_, _, _, class_ids_per_task, _ = build_mtil_cil_scenarios(
|
| 338 |
-
train_list[0], test_list[0], classnames_single, cil_splits, seed=int(seed)
|
| 339 |
-
)
|
| 340 |
-
|
| 341 |
-
distances = np.zeros((cil_splits, len(upstream_names)), dtype=float)
|
| 342 |
-
for t, cls_ids in enumerate(class_ids_per_task):
|
| 343 |
-
cls_names_t = [classnames_single[c] for c in cls_ids]
|
| 344 |
-
T_t = build_text_class_prototypes(clip_text_model, device, cls_names_t, templates_single, batch_size=256)
|
| 345 |
-
ds_down = train_list[0]
|
| 346 |
-
V_t = build_visual_class_prototypes_subset(vision_model, device, ds_down, list(cls_ids), max_per_class=int(max_images_per_class), batch_size=int(vision_batch_size), preprocess_eval=preprocess_eval)
|
| 347 |
-
for j, nm in enumerate(upstream_names):
|
| 348 |
-
distances[t, j] = directed_covering_distance(T_t, V_t, T_up_list[j], V_up_list[j])
|
| 349 |
-
|
| 350 |
-
similarity = np.exp(-float(beta) * distances)
|
| 351 |
-
similarity = np.clip(similarity, 0.0, 1.0)
|
| 352 |
-
return similarity, upstream_names
|
| 353 |
-
|
| 354 |
-
|
| 355 |
-
def main():
|
| 356 |
-
parser = argparse.ArgumentParser(description="Dataset similarity based on CLIP text and visual prototypes")
|
| 357 |
-
parser.add_argument("--dataset-root", type=str, default=os.environ.get("DATASET_ROOT", "data"))
|
| 358 |
-
parser.add_argument("--model-name", type=str, default="ViT-B/16")
|
| 359 |
-
parser.add_argument("--vision-batch-size", type=int, default=64)
|
| 360 |
-
parser.add_argument("--max-images-per-class", type=int, default=64, help="Maximum number of samples to use for each class when building visual prototypes. For efficiency, Chamfer distance is not computed using all samples in a dataset; instead, each class is sampled with at most this many examples.")
|
| 361 |
-
parser.add_argument("--beta", type=float, default=1.0, help="Similarity mapping exp(-beta * dist)")
|
| 362 |
-
parser.add_argument("--output", type=str, default="")
|
| 363 |
-
parser.add_argument("--downstream-dataset", type=str, default="", help="Name of downstream dataset (e.g., CIFAR100, PCam, FGVCAircraft, DescribableTextures)")
|
| 364 |
-
parser.add_argument("--cil-split", type=int, default=0, help="Number of CIL splits for downstream dataset; if >0, output [cil_split x 24] matrix")
|
| 365 |
-
args = parser.parse_args()
|
| 366 |
-
|
| 367 |
-
names = DEFAULT_NAMES_ORDER1
|
| 368 |
-
|
| 369 |
-
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
|
| 370 |
-
clip_model, _, preprocess_eval_clip = load_orig_clip(args.model_name, device=device, jit=False)
|
| 371 |
-
clip_model.eval()
|
| 372 |
-
|
| 373 |
-
vision_model = clip_model
|
| 374 |
-
preprocess_eval = preprocess_eval_clip
|
| 375 |
-
clip_text_model = clip_model
|
| 376 |
-
|
| 377 |
-
cfg = type("Cfg", (), {})()
|
| 378 |
-
cfg.dataset = "MTIL"
|
| 379 |
-
cfg.dataset_root = args.dataset_root
|
| 380 |
-
cfg.MTIL_order_2 = False
|
| 381 |
-
cfg.train_one_dataset = -1
|
| 382 |
-
cfg.seed = 32
|
| 383 |
-
cfg.use_validation = False
|
| 384 |
-
|
| 385 |
-
dataset_list, classes_names_list, templates_list, _ = get_mtil_dataset(
|
| 386 |
-
cfg, 'test', transforms=preprocess_eval
|
| 387 |
-
)
|
| 388 |
-
|
| 389 |
-
n = min(len(names), len(classes_names_list))
|
| 390 |
-
names = names[:n]
|
| 391 |
-
classes_names_list = classes_names_list[:n]
|
| 392 |
-
templates_list = templates_list[:n] if templates_list is not None else [None] * n
|
| 393 |
-
|
| 394 |
-
if args.downstream_dataset and int(args.cil_split) > 0:
|
| 395 |
-
repo_root = os.path.dirname(os.path.abspath(__file__))
|
| 396 |
-
alias_in = (args.downstream_dataset or '').strip()
|
| 397 |
-
alias_map = {
|
| 398 |
-
"Aircraft": "FGVCAircraft",
|
| 399 |
-
"FGVCAircraft": "FGVCAircraft",
|
| 400 |
-
"DTD": "DescribableTextures",
|
| 401 |
-
"DescribableTextures": "DescribableTextures",
|
| 402 |
-
}
|
| 403 |
-
ds_internal = alias_map.get(alias_in, alias_in) if alias_in else None
|
| 404 |
-
downstream_seed = resolve_downstream_seed(repo_root, ds_internal)
|
| 405 |
-
name_to_idx = {nm: idx for idx, nm in enumerate(names)}
|
| 406 |
-
if ds_internal not in name_to_idx:
|
| 407 |
-
raise ValueError(f"Downstream dataset '{args.downstream_dataset}' (mapped to '{ds_internal}') not found in MTIL names.")
|
| 408 |
-
cfg.seed = downstream_seed
|
| 409 |
-
dataset_list, classes_names_list, templates_list, _ = get_mtil_dataset(
|
| 410 |
-
cfg, 'test', transforms=preprocess_eval
|
| 411 |
-
)
|
| 412 |
-
n = min(len(names), len(classes_names_list))
|
| 413 |
-
names = names[:n]
|
| 414 |
-
classes_names_list = classes_names_list[:n]
|
| 415 |
-
templates_list = templates_list[:n] if templates_list is not None else [None] * n
|
| 416 |
-
upstream_names = [nm for nm in names if nm != ds_internal]
|
| 417 |
-
similarity_matrix, upstream_names = tasks_to_upstream_similarity_matrix(
|
| 418 |
-
vision_model, device,
|
| 419 |
-
upstream_names, name_to_idx,
|
| 420 |
-
dataset_list, classes_names_list, templates_list,
|
| 421 |
-
ds_internal, int(args.cil_split),
|
| 422 |
-
preprocess_eval,
|
| 423 |
-
int(args.max_images_per_class), int(args.vision_batch_size),
|
| 424 |
-
args.dataset_root, float(args.beta), clip_text_model=clip_text_model,
|
| 425 |
-
seed=downstream_seed,
|
| 426 |
-
)
|
| 427 |
-
print("DEFAULT_SIM_MATRIX = [")
|
| 428 |
-
fmt = "{:.3f}"
|
| 429 |
-
for i in range(similarity_matrix.shape[0]):
|
| 430 |
-
row_str = ", ".join(fmt.format(float(x)) for x in similarity_matrix[i])
|
| 431 |
-
print(f" [{row_str}],")
|
| 432 |
-
print("]")
|
| 433 |
-
names_py = ", ".join([f'"{n}"' for n in upstream_names])
|
| 434 |
-
print(f"DEFAULT_SIM_UPSTREAM_NAMES = [{names_py}]")
|
| 435 |
-
if args.output:
|
| 436 |
-
import json
|
| 437 |
-
out = {
|
| 438 |
-
"downstream": args.downstream_dataset,
|
| 439 |
-
"cil_split": int(args.cil_split),
|
| 440 |
-
"upstream_names": upstream_names,
|
| 441 |
-
"similarity_matrix": similarity_matrix.tolist(),
|
| 442 |
-
"beta": float(args.beta),
|
| 443 |
-
"seed": int(downstream_seed),
|
| 444 |
-
"max_images_per_class": int(args.max_images_per_class),
|
| 445 |
-
"vision_encoder": "clip",
|
| 446 |
-
"text_encoder": "clip",
|
| 447 |
-
}
|
| 448 |
-
base, ext = os.path.splitext(args.output)
|
| 449 |
-
out2 = base + "_sim" + ext
|
| 450 |
-
with open(out2, 'w', encoding='utf-8') as f:
|
| 451 |
-
json.dump(out, f, ensure_ascii=False, indent=2)
|
| 452 |
-
print(f"Saved similarity JSON to {out2}")
|
| 453 |
-
return
|
| 454 |
-
|
| 455 |
-
if __name__ == "__main__":
|
| 456 |
-
main()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/class_orders/cifar100.yaml
DELETED
|
@@ -1 +0,0 @@
|
|
| 1 |
-
class_order: [87, 0, 52, 58, 44, 91, 68, 97, 51, 15, 94, 92, 10, 72, 49, 78, 61, 14, 8, 86, 84, 96, 18, 24, 32, 45, 88, 11, 4, 67, 69, 66, 77, 47, 79, 93, 29, 50, 57, 83, 17, 81, 41, 12, 37, 59, 25, 20, 80, 73, 1, 28, 6, 46, 62, 82, 53, 9, 31, 75, 38, 63, 33, 74, 27, 22, 36, 3, 16, 21, 60, 19, 70, 90, 89, 43, 5, 42, 65, 76, 40, 30, 23, 85, 2, 95, 56, 48, 71, 64, 98, 13, 99, 7, 34, 55, 54, 26, 35, 39]
|
|
|
|
|
|
DFA-MoE/class_orders/imagenet100.yaml
DELETED
|
@@ -1,11 +0,0 @@
|
|
| 1 |
-
class_order: [
|
| 2 |
-
68, 56, 78, 8, 23, 84, 90, 65, 74, 76, 40,
|
| 3 |
-
89, 3, 92, 55, 9, 26, 80, 43, 38, 58, 70,
|
| 4 |
-
77, 1, 85, 19, 17, 50, 28, 53, 13, 81, 45,
|
| 5 |
-
82, 6, 59, 83, 16, 15, 44, 91, 41, 72, 60,
|
| 6 |
-
79, 52, 20, 10, 31, 54, 37, 95, 14, 71, 96,
|
| 7 |
-
98, 97, 2, 64, 66, 42, 22, 35, 86, 24, 34,
|
| 8 |
-
87, 21, 99, 0, 88, 27, 18, 94, 11, 12, 47,
|
| 9 |
-
25, 30, 46, 62, 69, 36, 61, 7, 63, 75, 5, 32,
|
| 10 |
-
4, 51, 48, 73, 93, 39, 67, 29, 49, 57, 33
|
| 11 |
-
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/class_orders/imagenet1000.yaml
DELETED
|
@@ -1,48 +0,0 @@
|
|
| 1 |
-
class_order: [
|
| 2 |
-
54, 7, 894, 512, 126, 337, 988, 11, 284, 493, 133, 783, 192, 979, 622, 215, 240, 548, 238, 419, 274, 108,
|
| 3 |
-
928, 856, 494, 836, 473, 650, 85, 262, 508, 590, 390, 174, 637, 288, 658, 219, 912, 142, 852, 160, 704, 289,
|
| 4 |
-
123, 323, 600, 542, 999, 634, 391, 761, 490, 842, 127, 850, 665, 990, 597, 722, 748, 14, 77, 437, 394, 859,
|
| 5 |
-
279, 539, 75, 466, 886, 312, 303, 62, 966, 413, 959, 782, 509, 400, 471, 632, 275, 730, 105, 523, 224, 186,
|
| 6 |
-
478, 507, 470, 906, 699, 989, 324, 812, 260, 911, 446, 44, 765, 759, 67, 36, 5, 30, 184, 797, 159, 741, 954,
|
| 7 |
-
465, 533, 585, 150, 101, 897, 363, 818, 620, 824, 154, 956, 176, 588, 986, 172, 223, 461, 94, 141, 621, 659,
|
| 8 |
-
360, 136, 578, 163, 427, 70, 226, 925, 596, 336, 412, 731, 755, 381, 810, 69, 898, 310, 120, 752, 93, 39,
|
| 9 |
-
326, 537, 905, 448, 347, 51, 615, 601, 229, 947, 348, 220, 949, 972, 73, 913, 522, 193, 753, 921, 257, 957,
|
| 10 |
-
691, 155, 820, 584, 948, 92, 582, 89, 379, 392, 64, 904, 169, 216, 694, 103, 410, 374, 515, 484, 624, 409,
|
| 11 |
-
156, 455, 846, 344, 371, 468, 844, 276, 740, 562, 503, 831, 516, 663, 630, 763, 456, 179, 996, 936, 248,
|
| 12 |
-
333, 941, 63, 738, 802, 372, 828, 74, 540, 299, 750, 335, 177, 822, 643, 593, 800, 459, 580, 933, 306, 378,
|
| 13 |
-
76, 227, 426, 403, 322, 321, 808, 393, 27, 200, 764, 651, 244, 479, 3, 415, 23, 964, 671, 195, 569, 917,
|
| 14 |
-
611, 644, 707, 355, 855, 8, 534, 657, 571, 811, 681, 543, 313, 129, 978, 592, 573, 128, 243, 520, 887, 892,
|
| 15 |
-
696, 26, 551, 168, 71, 398, 778, 529, 526, 792, 868, 266, 443, 24, 57, 15, 871, 678, 745, 845, 208, 188,
|
| 16 |
-
674, 175, 406, 421, 833, 106, 994, 815, 581, 676, 49, 619, 217, 631, 934, 932, 568, 353, 863, 827, 425, 420,
|
| 17 |
-
99, 823, 113, 974, 438, 874, 343, 118, 340, 472, 552, 937, 0, 10, 675, 316, 879, 561, 387, 726, 255, 407,
|
| 18 |
-
56, 927, 655, 809, 839, 640, 297, 34, 497, 210, 606, 971, 589, 138, 263, 587, 993, 973, 382, 572, 735, 535,
|
| 19 |
-
139, 524, 314, 463, 895, 376, 939, 157, 858, 457, 935, 183, 114, 903, 767, 666, 22, 525, 902, 233, 250, 825,
|
| 20 |
-
79, 843, 221, 214, 205, 166, 431, 860, 292, 976, 739, 899, 475, 242, 961, 531, 110, 769, 55, 701, 532, 586,
|
| 21 |
-
729, 253, 486, 787, 774, 165, 627, 32, 291, 962, 922, 222, 705, 454, 356, 445, 746, 776, 404, 950, 241, 452,
|
| 22 |
-
245, 487, 706, 2, 137, 6, 98, 647, 50, 91, 202, 556, 38, 68, 649, 258, 345, 361, 464, 514, 958, 504, 826,
|
| 23 |
-
668, 880, 28, 920, 918, 339, 315, 320, 768, 201, 733, 575, 781, 864, 617, 171, 795, 132, 145, 368, 147, 327,
|
| 24 |
-
713, 688, 848, 690, 975, 354, 853, 148, 648, 300, 436, 780, 693, 682, 246, 449, 492, 162, 97, 59, 357, 198,
|
| 25 |
-
519, 90, 236, 375, 359, 230, 476, 784, 117, 940, 396, 849, 102, 122, 282, 181, 130, 467, 88, 271, 793, 151,
|
| 26 |
-
847, 914, 42, 834, 521, 121, 29, 806, 607, 510, 837, 301, 669, 78, 256, 474, 840, 52, 505, 547, 641, 987,
|
| 27 |
-
801, 629, 491, 605, 112, 429, 401, 742, 528, 87, 442, 910, 638, 785, 264, 711, 369, 428, 805, 744, 380, 725,
|
| 28 |
-
480, 318, 997, 153, 384, 252, 985, 538, 654, 388, 100, 432, 832, 565, 908, 367, 591, 294, 272, 231, 213,
|
| 29 |
-
196, 743, 817, 433, 328, 970, 969, 4, 613, 182, 685, 724, 915, 311, 931, 865, 86, 119, 203, 268, 718, 317,
|
| 30 |
-
926, 269, 161, 209, 807, 645, 513, 261, 518, 305, 758, 872, 58, 65, 146, 395, 481, 747, 41, 283, 204, 564,
|
| 31 |
-
185, 777, 33, 500, 609, 286, 567, 80, 228, 683, 757, 942, 134, 673, 616, 960, 450, 350, 544, 830, 736, 170,
|
| 32 |
-
679, 838, 819, 485, 430, 190, 566, 511, 482, 232, 527, 411, 560, 281, 342, 614, 662, 47, 771, 861, 692, 686,
|
| 33 |
-
277, 373, 16, 946, 265, 35, 9, 884, 909, 610, 358, 18, 737, 977, 677, 803, 595, 135, 458, 12, 46, 418, 599,
|
| 34 |
-
187, 107, 992, 770, 298, 104, 351, 893, 698, 929, 502, 273, 20, 96, 791, 636, 708, 267, 867, 772, 604, 618,
|
| 35 |
-
346, 330, 554, 816, 664, 716, 189, 31, 721, 712, 397, 43, 943, 804, 296, 109, 576, 869, 955, 17, 506, 963,
|
| 36 |
-
786, 720, 628, 779, 982, 633, 891, 734, 980, 386, 365, 794, 325, 841, 878, 370, 695, 293, 951, 66, 594, 717,
|
| 37 |
-
116, 488, 796, 983, 646, 499, 53, 1, 603, 45, 424, 875, 254, 237, 199, 414, 307, 362, 557, 866, 341, 19,
|
| 38 |
-
965, 143, 555, 687, 235, 790, 125, 173, 364, 882, 727, 728, 563, 495, 21, 558, 709, 719, 877, 352, 83, 998,
|
| 39 |
-
991, 469, 967, 760, 498, 814, 612, 715, 290, 72, 131, 259, 441, 924, 773, 48, 625, 501, 440, 82, 684, 862,
|
| 40 |
-
574, 309, 408, 680, 623, 439, 180, 652, 968, 889, 334, 61, 766, 399, 598, 798, 653, 930, 149, 249, 890, 308,
|
| 41 |
-
881, 40, 835, 577, 422, 703, 813, 857, 995, 602, 583, 167, 670, 212, 751, 496, 608, 84, 639, 579, 178, 489,
|
| 42 |
-
37, 197, 789, 530, 111, 876, 570, 700, 444, 287, 366, 883, 385, 536, 460, 851, 81, 144, 60, 251, 13, 953,
|
| 43 |
-
270, 944, 319, 885, 710, 952, 517, 278, 656, 919, 377, 550, 207, 660, 984, 447, 553, 338, 234, 383, 749,
|
| 44 |
-
916, 626, 462, 788, 434, 714, 799, 821, 477, 549, 661, 206, 667, 541, 642, 689, 194, 152, 981, 938, 854,
|
| 45 |
-
483, 332, 280, 546, 389, 405, 545, 239, 896, 672, 923, 402, 423, 907, 888, 140, 870, 559, 756, 25, 211, 158,
|
| 46 |
-
723, 635, 302, 702, 453, 218, 164, 829, 247, 775, 191, 732, 115, 331, 901, 416, 873, 754, 900, 435, 762,
|
| 47 |
-
124, 304, 329, 349, 295, 95, 451, 285, 225, 945, 697, 417
|
| 48 |
-
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/class_orders/tinyimagenet.yaml
DELETED
|
@@ -1,17 +0,0 @@
|
|
| 1 |
-
class_order: [
|
| 2 |
-
131, 181, 22, 172, 144, 92, 97, 187, 58, 93, 6, 70, 106, 68,
|
| 3 |
-
153, 168, 179, 199, 29, 46, 9, 142, 134, 88, 193, 110, 26,
|
| 4 |
-
32, 117, 112, 17, 39, 166, 13, 94, 138, 109, 147, 51, 101,
|
| 5 |
-
59, 188, 116, 5, 170, 99, 100, 167, 180, 146, 65, 1, 104,
|
| 6 |
-
43, 38, 184, 123, 171, 137, 162, 71, 44, 95, 174, 12, 7,
|
| 7 |
-
54, 152, 21, 47, 28, 176, 34, 2, 132, 118, 42, 189, 150,
|
| 8 |
-
14, 165, 41, 192, 45, 82, 128, 63, 57, 197, 160, 53, 75,
|
| 9 |
-
108, 135, 121, 159, 183, 67, 169, 50, 87, 69, 89, 196,
|
| 10 |
-
115, 19, 148, 96, 86, 11, 8, 60, 33, 173, 78, 4, 119, 105,
|
| 11 |
-
182, 127, 177, 30, 186, 40, 49, 178, 76, 157, 161, 73, 164,
|
| 12 |
-
151, 31, 74, 191, 27, 125, 198, 81, 20, 155, 114, 139, 36,
|
| 13 |
-
61, 56, 145, 48, 16, 83, 62, 85, 126, 0, 102, 23, 3, 140,
|
| 14 |
-
15, 195, 133, 113, 190, 141, 52, 163, 156, 80, 111, 90, 175,
|
| 15 |
-
143, 120, 84, 18, 25, 79, 37, 154, 136, 64, 158, 24, 185,
|
| 16 |
-
72, 35, 129, 55, 149, 91, 122, 77, 103, 124, 130, 66, 10, 107, 194, 98
|
| 17 |
-
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/clip/README.md
DELETED
|
@@ -1 +0,0 @@
|
|
| 1 |
-
This folder is a lightly modified version of https://github.com/openai/CLIP.
|
|
|
|
|
|
DFA-MoE/clip/__init__.py
DELETED
|
@@ -1 +0,0 @@
|
|
| 1 |
-
from .clip import *
|
|
|
|
|
|
DFA-MoE/clip/adapter.py
DELETED
|
@@ -1,69 +0,0 @@
|
|
| 1 |
-
# --------------------------------------------------------
|
| 2 |
-
# References:
|
| 3 |
-
# https://github.com/jxhe/unify-parameter-efficient-tuning
|
| 4 |
-
# --------------------------------------------------------
|
| 5 |
-
|
| 6 |
-
import math
|
| 7 |
-
import torch
|
| 8 |
-
import torch.nn as nn
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
class Adapter(nn.Module):
|
| 12 |
-
def __init__(self,
|
| 13 |
-
d_model=None,
|
| 14 |
-
bottleneck=None,
|
| 15 |
-
dropout=0.0,
|
| 16 |
-
init_option="lora",
|
| 17 |
-
adapter_scalar="1.0",
|
| 18 |
-
adapter_layernorm_option="in"):
|
| 19 |
-
super().__init__()
|
| 20 |
-
self.n_embd = d_model if d_model is None else d_model
|
| 21 |
-
self.down_size = bottleneck
|
| 22 |
-
|
| 23 |
-
#_before
|
| 24 |
-
self.adapter_layernorm_option = adapter_layernorm_option
|
| 25 |
-
|
| 26 |
-
self.adapter_layer_norm_before = None
|
| 27 |
-
if adapter_layernorm_option == "in" or adapter_layernorm_option == "out":
|
| 28 |
-
self.adapter_layer_norm_before = nn.LayerNorm(self.n_embd)
|
| 29 |
-
|
| 30 |
-
if adapter_scalar == "learnable_scalar":
|
| 31 |
-
self.scale = nn.Parameter(torch.ones(1))
|
| 32 |
-
else:
|
| 33 |
-
self.scale = float(adapter_scalar)
|
| 34 |
-
|
| 35 |
-
self.down_proj = nn.Linear(self.n_embd, 64)
|
| 36 |
-
self.non_linear_func = nn.ReLU()
|
| 37 |
-
self.up_proj = nn.Linear(self.down_size, self.n_embd)
|
| 38 |
-
|
| 39 |
-
self.dropout = dropout
|
| 40 |
-
if init_option == "bert":
|
| 41 |
-
raise NotImplementedError
|
| 42 |
-
elif init_option == "lora":
|
| 43 |
-
with torch.no_grad():
|
| 44 |
-
nn.init.kaiming_uniform_(self.down_proj.weight, a=math.sqrt(5))
|
| 45 |
-
nn.init.zeros_(self.up_proj.weight)
|
| 46 |
-
nn.init.zeros_(self.down_proj.bias)
|
| 47 |
-
nn.init.zeros_(self.up_proj.bias)
|
| 48 |
-
|
| 49 |
-
def forward(self, x, add_residual=True, residual=None):
|
| 50 |
-
|
| 51 |
-
residual = x if residual is None else residual
|
| 52 |
-
if self.adapter_layernorm_option == 'in': # none
|
| 53 |
-
x = self.adapter_layer_norm_before(x)
|
| 54 |
-
|
| 55 |
-
down = self.down_proj(x)
|
| 56 |
-
down = self.non_linear_func(down)
|
| 57 |
-
down = nn.functional.dropout(down, p=self.dropout, training=self.training)
|
| 58 |
-
up = self.up_proj(down)
|
| 59 |
-
|
| 60 |
-
up = up * self.scale
|
| 61 |
-
|
| 62 |
-
if self.adapter_layernorm_option == 'out': # none
|
| 63 |
-
up = self.adapter_layer_norm_before(up)
|
| 64 |
-
|
| 65 |
-
if add_residual:
|
| 66 |
-
output = up + residual
|
| 67 |
-
else:
|
| 68 |
-
output = up
|
| 69 |
-
return output
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/clip/bpe_simple_vocab_16e6.txt.gz
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:924691ac288e54409236115652ad4aa250f48203de50a9e4722a6ecd48d6804a
|
| 3 |
-
size 1356917
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/clip/clip.py
DELETED
|
@@ -1,310 +0,0 @@
|
|
| 1 |
-
# Code ported from https://github.com/openai/CLIP
|
| 2 |
-
|
| 3 |
-
import hashlib
|
| 4 |
-
import os
|
| 5 |
-
import urllib
|
| 6 |
-
import warnings
|
| 7 |
-
from typing import Union, List
|
| 8 |
-
|
| 9 |
-
import torch
|
| 10 |
-
from torchvision.transforms import Compose, Resize, CenterCrop, ToTensor, Normalize, RandomResizedCrop, InterpolationMode
|
| 11 |
-
from tqdm import tqdm
|
| 12 |
-
|
| 13 |
-
from clip.model import build_model
|
| 14 |
-
from clip.tokenizer import SimpleTokenizer as _Tokenizer
|
| 15 |
-
|
| 16 |
-
__all__ = ["available_models", "load", "tokenize"]
|
| 17 |
-
_tokenizer = _Tokenizer()
|
| 18 |
-
|
| 19 |
-
_MODELS = {
|
| 20 |
-
"RN50": "https://openaipublic.azureedge.net/clip/models/afeb0e10f9e5a86da6080e35cf09123aca3b358a0c3e3b6c78a7b63bc04b6762/RN50.pt",
|
| 21 |
-
"RN101": "https://openaipublic.azureedge.net/clip/models/8fa8567bab74a42d41c5915025a8e4538c3bdbe8804a470a72f30b0d94fab599/RN101.pt",
|
| 22 |
-
"RN50x4": "https://openaipublic.azureedge.net/clip/models/7e526bd135e493cef0776de27d5f42653e6b4c8bf9e0f653bb11773263205fdd/RN50x4.pt",
|
| 23 |
-
"RN50x16": "https://openaipublic.azureedge.net/clip/models/52378b407f34354e150460fe41077663dd5b39c54cd0bfd2b27167a4a06ec9aa/RN50x16.pt",
|
| 24 |
-
"ViT-B/32": "https://openaipublic.azureedge.net/clip/models/40d365715913c9da98579312b702a82c18be219cc2a73407c4526f58eba950af/ViT-B-32.pt",
|
| 25 |
-
"ViT-B/16": "https://openaipublic.azureedge.net/clip/models/5806e77cd80f8b59890b7e101eabd078d9fb84e6937f9e85e4ecb61988df416f/ViT-B-16.pt",
|
| 26 |
-
}
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
def _download(url: str, root: str = os.path.expanduser("~/.cache/clip")):
|
| 30 |
-
os.makedirs(root, exist_ok=True)
|
| 31 |
-
filename = os.path.basename(url)
|
| 32 |
-
|
| 33 |
-
expected_sha256 = url.split("/")[-2]
|
| 34 |
-
download_target = os.path.join(root, filename)
|
| 35 |
-
|
| 36 |
-
if os.path.exists(download_target) and not os.path.isfile(download_target):
|
| 37 |
-
raise RuntimeError(f"{download_target} exists and is not a regular file")
|
| 38 |
-
|
| 39 |
-
if os.path.isfile(download_target):
|
| 40 |
-
if hashlib.sha256(open(download_target, "rb").read()).hexdigest() == expected_sha256:
|
| 41 |
-
return download_target
|
| 42 |
-
else:
|
| 43 |
-
warnings.warn(f"{download_target} exists, but the SHA256 checksum does not match; re-downloading the file")
|
| 44 |
-
|
| 45 |
-
with urllib.request.urlopen(url) as source, open(download_target, "wb") as output:
|
| 46 |
-
with tqdm(total=int(source.info().get("Content-Length")), ncols=80, unit='iB', unit_scale=True) as loop:
|
| 47 |
-
while True:
|
| 48 |
-
buffer = source.read(8192)
|
| 49 |
-
if not buffer:
|
| 50 |
-
break
|
| 51 |
-
|
| 52 |
-
output.write(buffer)
|
| 53 |
-
loop.update(len(buffer))
|
| 54 |
-
|
| 55 |
-
if hashlib.sha256(open(download_target, "rb").read()).hexdigest() != expected_sha256:
|
| 56 |
-
raise RuntimeError(f"Model has been downloaded but the SHA256 checksum does not not match")
|
| 57 |
-
|
| 58 |
-
return download_target
|
| 59 |
-
|
| 60 |
-
def _convert_to_rgb(image):
|
| 61 |
-
return image.convert('RGB')
|
| 62 |
-
|
| 63 |
-
def _transform(n_px: int, is_train: bool):
|
| 64 |
-
normalize = Normalize((0.48145466, 0.4578275, 0.40821073), (0.26862954, 0.26130258, 0.27577711))
|
| 65 |
-
if is_train:
|
| 66 |
-
return Compose([
|
| 67 |
-
RandomResizedCrop(n_px, scale=(0.9, 1.0), interpolation=InterpolationMode.BICUBIC),
|
| 68 |
-
_convert_to_rgb,
|
| 69 |
-
ToTensor(),
|
| 70 |
-
normalize,
|
| 71 |
-
])
|
| 72 |
-
else:
|
| 73 |
-
return Compose([
|
| 74 |
-
Resize(n_px, interpolation=InterpolationMode.BICUBIC),
|
| 75 |
-
CenterCrop(n_px),
|
| 76 |
-
_convert_to_rgb,
|
| 77 |
-
ToTensor(),
|
| 78 |
-
normalize,
|
| 79 |
-
])
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
def available_models() -> List[str]:
|
| 84 |
-
"""Returns the names of available CLIP models"""
|
| 85 |
-
return list(_MODELS.keys())
|
| 86 |
-
|
| 87 |
-
# def load(name: str, device: Union[str, torch.device] = "cuda" if torch.cuda.is_available() else "cpu", jit: bool = False, download_root: str = None):
|
| 88 |
-
# """Load a CLIP model
|
| 89 |
-
|
| 90 |
-
# Parameters
|
| 91 |
-
# ----------
|
| 92 |
-
# name : str
|
| 93 |
-
# A model name listed by `clip.available_models()`, or the path to a model checkpoint containing the state_dict
|
| 94 |
-
|
| 95 |
-
# device : Union[str, torch.device]
|
| 96 |
-
# The device to put the loaded model
|
| 97 |
-
|
| 98 |
-
# jit : bool
|
| 99 |
-
# Whether to load the optimized JIT model or more hackable non-JIT model (default).
|
| 100 |
-
|
| 101 |
-
# download_root: str
|
| 102 |
-
# path to download the model files; by default, it uses "~/.cache/clip"
|
| 103 |
-
|
| 104 |
-
# Returns
|
| 105 |
-
# -------
|
| 106 |
-
# model : torch.nn.Module
|
| 107 |
-
# The CLIP model
|
| 108 |
-
|
| 109 |
-
# preprocess : Callable[[PIL.Image], torch.Tensor]
|
| 110 |
-
# A torchvision transform that converts a PIL image into a tensor that the returned model can take as its input
|
| 111 |
-
# """
|
| 112 |
-
# if name in _MODELS:
|
| 113 |
-
# model_path = _download(_MODELS[name], download_root or os.path.expanduser("~/.cache/clip"))
|
| 114 |
-
# elif os.path.isfile(name):
|
| 115 |
-
# model_path = name
|
| 116 |
-
# else:
|
| 117 |
-
# raise RuntimeError(f"Model {name} not found; available models = {available_models()}")
|
| 118 |
-
|
| 119 |
-
# try:
|
| 120 |
-
# # loading JIT archive
|
| 121 |
-
# model = torch.jit.load(model_path, map_location=device if jit else "cpu").eval()
|
| 122 |
-
# state_dict = None
|
| 123 |
-
# except RuntimeError:
|
| 124 |
-
# # loading saved state dict
|
| 125 |
-
# if jit:
|
| 126 |
-
# warnings.warn(f"File {model_path} is not a JIT archive. Loading as a state dict instead")
|
| 127 |
-
# jit = False
|
| 128 |
-
# state_dict = torch.load(model_path, map_location="cpu")
|
| 129 |
-
|
| 130 |
-
# if not jit:
|
| 131 |
-
# model = build_model(state_dict or model.state_dict()).to(device)
|
| 132 |
-
# if str(device) == "cpu":
|
| 133 |
-
# model.float()
|
| 134 |
-
# return model, _transform(model.visual.input_resolution)
|
| 135 |
-
|
| 136 |
-
# # patch the device names
|
| 137 |
-
# device_holder = torch.jit.trace(lambda: torch.ones([]).to(torch.device(device)), example_inputs=[])
|
| 138 |
-
# device_node = [n for n in device_holder.graph.findAllNodes("prim::Constant") if "Device" in repr(n)][-1]
|
| 139 |
-
|
| 140 |
-
# def patch_device(module):
|
| 141 |
-
# try:
|
| 142 |
-
# graphs = [module.graph] if hasattr(module, "graph") else []
|
| 143 |
-
# except RuntimeError:
|
| 144 |
-
# graphs = []
|
| 145 |
-
|
| 146 |
-
# if hasattr(module, "forward1"):
|
| 147 |
-
# graphs.append(module.forward1.graph)
|
| 148 |
-
|
| 149 |
-
# for graph in graphs:
|
| 150 |
-
# for node in graph.findAllNodes("prim::Constant"):
|
| 151 |
-
# if "value" in node.attributeNames() and str(node["value"]).startswith("cuda"):
|
| 152 |
-
# node.copyAttributes(device_node)
|
| 153 |
-
|
| 154 |
-
# model.apply(patch_device)
|
| 155 |
-
# patch_device(model.encode_image)
|
| 156 |
-
# patch_device(model.encode_text)
|
| 157 |
-
|
| 158 |
-
# # patch dtype to float32 on CPU
|
| 159 |
-
# if str(device) == "cpu":
|
| 160 |
-
# float_holder = torch.jit.trace(lambda: torch.ones([]).float(), example_inputs=[])
|
| 161 |
-
# float_input = list(float_holder.graph.findNode("aten::to").inputs())[1]
|
| 162 |
-
# float_node = float_input.node()
|
| 163 |
-
|
| 164 |
-
# def patch_float(module):
|
| 165 |
-
# try:
|
| 166 |
-
# graphs = [module.graph] if hasattr(module, "graph") else []
|
| 167 |
-
# except RuntimeError:
|
| 168 |
-
# graphs = []
|
| 169 |
-
|
| 170 |
-
# if hasattr(module, "forward1"):
|
| 171 |
-
# graphs.append(module.forward1.graph)
|
| 172 |
-
|
| 173 |
-
# for graph in graphs:
|
| 174 |
-
# for node in graph.findAllNodes("aten::to"):
|
| 175 |
-
# inputs = list(node.inputs())
|
| 176 |
-
# for i in [1, 2]: # dtype can be the second or third argument to aten::to()
|
| 177 |
-
# if inputs[i].node()["value"] == 5:
|
| 178 |
-
# inputs[i].node().copyAttributes(float_node)
|
| 179 |
-
|
| 180 |
-
# model.apply(patch_float)
|
| 181 |
-
# patch_float(model.encode_image)
|
| 182 |
-
# patch_float(model.encode_text)
|
| 183 |
-
|
| 184 |
-
# model.float()
|
| 185 |
-
|
| 186 |
-
# return model, _transform(model.input_resolution.item())
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
def load(name: str, device: Union[str, torch.device] = "cuda" if torch.cuda.is_available() else "cpu", jit=True, is_train=False, pretrained=True):
|
| 190 |
-
"""Load a CLIP model
|
| 191 |
-
Parameters
|
| 192 |
-
----------
|
| 193 |
-
name : str
|
| 194 |
-
A model name listed by `clip.available_models()`, or the path to a model checkpoint containing the state_dict
|
| 195 |
-
device : Union[str, torch.device]
|
| 196 |
-
The device to put the loaded model
|
| 197 |
-
jit : bool
|
| 198 |
-
Whether to load the optimized JIT model (default) or more hackable non-JIT model.
|
| 199 |
-
Returns
|
| 200 |
-
-------
|
| 201 |
-
model : torch.nn.Module
|
| 202 |
-
The CLIP model
|
| 203 |
-
preprocess : Callable[[PIL.Image], torch.Tensor]
|
| 204 |
-
A torchvision transform that converts a PIL image into a tensor that the returned model can take as its input
|
| 205 |
-
"""
|
| 206 |
-
if name in _MODELS:
|
| 207 |
-
model_path = _download(_MODELS[name])
|
| 208 |
-
elif os.path.isfile(name):
|
| 209 |
-
model_path = name
|
| 210 |
-
else:
|
| 211 |
-
raise RuntimeError(f"Model {name} not found; available models = {available_models()}")
|
| 212 |
-
|
| 213 |
-
try:
|
| 214 |
-
# loading JIT archive
|
| 215 |
-
model = torch.jit.load(model_path, map_location=device if jit else "cpu").eval()
|
| 216 |
-
state_dict = None
|
| 217 |
-
except RuntimeError:
|
| 218 |
-
# loading saved state dict
|
| 219 |
-
if jit:
|
| 220 |
-
warnings.warn(f"File {model_path} is not a JIT archive. Loading as a state dict instead")
|
| 221 |
-
jit = False
|
| 222 |
-
state_dict = torch.load(model_path, map_location="cpu")
|
| 223 |
-
|
| 224 |
-
if not jit:
|
| 225 |
-
try:
|
| 226 |
-
model = build_model(state_dict or model.state_dict()).to(device)
|
| 227 |
-
except KeyError:
|
| 228 |
-
sd = {k[7:]: v for k,v in state_dict["state_dict"].items()}
|
| 229 |
-
model = build_model(sd).to(device)
|
| 230 |
-
|
| 231 |
-
if str(device) == "cpu":
|
| 232 |
-
model.float()
|
| 233 |
-
return model, \
|
| 234 |
-
_transform(model.visual.input_resolution, is_train=True), \
|
| 235 |
-
_transform(model.visual.input_resolution, is_train=False)
|
| 236 |
-
|
| 237 |
-
# patch the device names
|
| 238 |
-
device_holder = torch.jit.trace(lambda: torch.ones([]).to(torch.device(device)), example_inputs=[])
|
| 239 |
-
device_node = [n for n in device_holder.graph.findAllNodes("prim::Constant") if "Device" in repr(n)][-1]
|
| 240 |
-
|
| 241 |
-
def patch_device(module):
|
| 242 |
-
graphs = [module.graph] if hasattr(module, "graph") else []
|
| 243 |
-
if hasattr(module, "forward1"):
|
| 244 |
-
graphs.append(module.forward1.graph)
|
| 245 |
-
|
| 246 |
-
for graph in graphs:
|
| 247 |
-
for node in graph.findAllNodes("prim::Constant"):
|
| 248 |
-
if "value" in node.attributeNames() and str(node["value"]).startswith("cuda"):
|
| 249 |
-
node.copyAttributes(device_node)
|
| 250 |
-
|
| 251 |
-
model.apply(patch_device)
|
| 252 |
-
patch_device(model.encode_image)
|
| 253 |
-
patch_device(model.encode_text)
|
| 254 |
-
|
| 255 |
-
# patch dtype to float32 on CPU
|
| 256 |
-
if str(device) == "cpu":
|
| 257 |
-
float_holder = torch.jit.trace(lambda: torch.ones([]).float(), example_inputs=[])
|
| 258 |
-
float_input = list(float_holder.graph.findNode("aten::to").inputs())[1]
|
| 259 |
-
float_node = float_input.node()
|
| 260 |
-
|
| 261 |
-
def patch_float(module):
|
| 262 |
-
graphs = [module.graph] if hasattr(module, "graph") else []
|
| 263 |
-
if hasattr(module, "forward1"):
|
| 264 |
-
graphs.append(module.forward1.graph)
|
| 265 |
-
|
| 266 |
-
for graph in graphs:
|
| 267 |
-
for node in graph.findAllNodes("aten::to"):
|
| 268 |
-
inputs = list(node.inputs())
|
| 269 |
-
for i in [1, 2]: # dtype can be the second or third argument to aten::to()
|
| 270 |
-
if inputs[i].node()["value"] == 5:
|
| 271 |
-
inputs[i].node().copyAttributes(float_node)
|
| 272 |
-
|
| 273 |
-
model.apply(patch_float)
|
| 274 |
-
patch_float(model.encode_image)
|
| 275 |
-
patch_float(model.encode_text)
|
| 276 |
-
|
| 277 |
-
model.float()
|
| 278 |
-
|
| 279 |
-
return model, \
|
| 280 |
-
_transform(model.input_resolution.item(), is_train=True), \
|
| 281 |
-
_transform(model.input_resolution.item(), is_train=False)
|
| 282 |
-
|
| 283 |
-
|
| 284 |
-
def tokenize(texts: Union[str, List[str]], context_length: int = 77) -> torch.LongTensor:
|
| 285 |
-
"""
|
| 286 |
-
Returns the tokenized representation of given input string(s)
|
| 287 |
-
Parameters
|
| 288 |
-
----------
|
| 289 |
-
texts : Union[str, List[str]]
|
| 290 |
-
An input string or a list of input strings to tokenize
|
| 291 |
-
context_length : int
|
| 292 |
-
The context length to use; all CLIP models use 77 as the context length
|
| 293 |
-
Returns
|
| 294 |
-
-------
|
| 295 |
-
A two-dimensional tensor containing the resulting tokens, shape = [number of input strings, context_length]
|
| 296 |
-
"""
|
| 297 |
-
if isinstance(texts, str):
|
| 298 |
-
texts = [texts]
|
| 299 |
-
|
| 300 |
-
sot_token = _tokenizer.encoder["<start_of_text>"]
|
| 301 |
-
eot_token = _tokenizer.encoder["<end_of_text>"]
|
| 302 |
-
all_tokens = [[sot_token] + _tokenizer.encode(text) + [eot_token] for text in texts]
|
| 303 |
-
result = torch.zeros(len(all_tokens), context_length, dtype=torch.long)
|
| 304 |
-
|
| 305 |
-
for i, tokens in enumerate(all_tokens):
|
| 306 |
-
if len(tokens) > context_length: # Truncate
|
| 307 |
-
tokens = tokens[:context_length]
|
| 308 |
-
result[i, :len(tokens)] = torch.tensor(tokens)
|
| 309 |
-
|
| 310 |
-
return result
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/clip/model.py
DELETED
|
@@ -1,713 +0,0 @@
|
|
| 1 |
-
from collections import OrderedDict
|
| 2 |
-
from typing import Tuple, Union
|
| 3 |
-
|
| 4 |
-
import os
|
| 5 |
-
import json
|
| 6 |
-
import numpy as np
|
| 7 |
-
import torch
|
| 8 |
-
import torch.nn.functional as F
|
| 9 |
-
from torch import nn
|
| 10 |
-
from .adapter import Adapter
|
| 11 |
-
from torch.distributions.normal import Normal
|
| 12 |
-
from collections import Counter
|
| 13 |
-
|
| 14 |
-
global_taskid = 0
|
| 15 |
-
global_is_train=True
|
| 16 |
-
class SparseDispatcher(object):
|
| 17 |
-
"""Helper for implementing a mixture of experts.
|
| 18 |
-
The purpose of this class is to create input minibatches for the
|
| 19 |
-
experts and to combine the results of the experts to form a unified
|
| 20 |
-
output tensor.
|
| 21 |
-
There are two functions:
|
| 22 |
-
dispatch - take an input Tensor and create input Tensors for each expert.
|
| 23 |
-
combine - take output Tensors from each expert and form a combined output
|
| 24 |
-
Tensor. Outputs from different experts for the same batch element are
|
| 25 |
-
summed together, weighted by the provided "gates".
|
| 26 |
-
The class is initialized with a "gates" Tensor, which specifies which
|
| 27 |
-
batch elements go to which experts, and the weights to use when combining
|
| 28 |
-
the outputs. Batch element b is sent to expert e iff gates[b, e] != 0.
|
| 29 |
-
The inputs and outputs are all two-dimensional [batch, depth].
|
| 30 |
-
Caller is responsible for collapsing additional dimensions prior to
|
| 31 |
-
calling this class and reshaping the output to the original shape.
|
| 32 |
-
See common_layers.reshape_like().
|
| 33 |
-
Example use:
|
| 34 |
-
gates: a float32 `Tensor` with shape `[batch_size, num_experts]`
|
| 35 |
-
inputs: a float32 `Tensor` with shape `[batch_size, input_size]`
|
| 36 |
-
experts: a list of length `num_experts` containing sub-networks.
|
| 37 |
-
dispatcher = SparseDispatcher(num_experts, gates)
|
| 38 |
-
expert_inputs = dispatcher.dispatch(inputs)
|
| 39 |
-
expert_outputs = [experts[i](expert_inputs[i]) for i in range(num_experts)]
|
| 40 |
-
outputs = dispatcher.combine(expert_outputs)
|
| 41 |
-
The preceding code sets the output for a particular example b to:
|
| 42 |
-
output[b] = Sum_i(gates[b, i] * experts[i](inputs[b]))
|
| 43 |
-
This class takes advantage of sparsity in the gate matrix by including in the
|
| 44 |
-
`Tensor`s for expert i only the batch elements for which `gates[b, i] > 0`.
|
| 45 |
-
"""
|
| 46 |
-
|
| 47 |
-
def __init__(self, num_experts, gates):
|
| 48 |
-
"""Create a SparseDispatcher."""
|
| 49 |
-
|
| 50 |
-
self._gates = gates
|
| 51 |
-
self._num_experts = num_experts
|
| 52 |
-
|
| 53 |
-
sorted_experts, index_sorted_experts = torch.nonzero(gates).sort(0)
|
| 54 |
-
|
| 55 |
-
# drop indices
|
| 56 |
-
_, self._expert_index = sorted_experts.split(1, dim=1)
|
| 57 |
-
# get according batch index for each expert
|
| 58 |
-
self._batch_index = torch.nonzero(gates)[index_sorted_experts[:, 1], 0]
|
| 59 |
-
# calculate num samples that each expert gets
|
| 60 |
-
self._part_sizes = (gates > 0).sum(0).tolist()
|
| 61 |
-
# expand gates to match with self._batch_index
|
| 62 |
-
gates_exp = gates[self._batch_index.flatten()]
|
| 63 |
-
self._nonzero_gates = torch.gather(gates_exp, 1, self._expert_index)
|
| 64 |
-
|
| 65 |
-
def dispatch(self, inp):
|
| 66 |
-
"""Create one input Tensor for each expert.
|
| 67 |
-
The `Tensor` for a expert `i` contains the slices of `inp` corresponding
|
| 68 |
-
to the batch elements `b` where `gates[b, i] > 0`.
|
| 69 |
-
Args:
|
| 70 |
-
inp: a `Tensor` of shape "[batch_size, <extra_input_dims>]`
|
| 71 |
-
Returns:
|
| 72 |
-
a list of `num_experts` `Tensor`s with shapes
|
| 73 |
-
`[expert_batch_size_i, <extra_input_dims>]`.
|
| 74 |
-
"""
|
| 75 |
-
|
| 76 |
-
# assigns samples to experts whose gate is nonzero
|
| 77 |
-
|
| 78 |
-
inp_exp = inp[self._batch_index].squeeze(1)
|
| 79 |
-
return torch.split(inp_exp, self._part_sizes, dim=0)
|
| 80 |
-
|
| 81 |
-
def combine(self, expert_out, multiply_by_gates=True):
|
| 82 |
-
"""Sum together the expert output, weighted by the gates.
|
| 83 |
-
The slice corresponding to a particular batch element `b` is computed
|
| 84 |
-
as the sum over all experts `i` of the expert output, weighted by the
|
| 85 |
-
corresponding gate values. If `multiply_by_gates` is set to False, the
|
| 86 |
-
gate values are ignored.
|
| 87 |
-
Args:
|
| 88 |
-
expert_out: a list of `num_experts` `Tensor`s, each with shape
|
| 89 |
-
`[expert_batch_size_i, <extra_output_dims>]`.
|
| 90 |
-
multiply_by_gates: a boolean
|
| 91 |
-
Returns:
|
| 92 |
-
a `Tensor` with shape `[batch_size, <extra_output_dims>]`.
|
| 93 |
-
"""
|
| 94 |
-
# apply exp to expert outputs, so we are not longer in log space
|
| 95 |
-
|
| 96 |
-
stitched = torch.cat(expert_out, 0)
|
| 97 |
-
if multiply_by_gates:
|
| 98 |
-
stitched = stitched.mul(self._nonzero_gates) # 加权
|
| 99 |
-
|
| 100 |
-
zeros = torch.zeros(self._gates.size(0), expert_out[-1].size(1), device=stitched.device)
|
| 101 |
-
# combine samples that have been processed by the same k experts
|
| 102 |
-
|
| 103 |
-
combined = zeros.index_add(0, self._batch_index, stitched.float())
|
| 104 |
-
# add eps to all zero values in order to avoid nans when going back to log space
|
| 105 |
-
# back to log space
|
| 106 |
-
return combined
|
| 107 |
-
|
| 108 |
-
def expert_to_gates(self):
|
| 109 |
-
"""Gate values corresponding to the examples in the per-expert `Tensor`s.
|
| 110 |
-
Returns:
|
| 111 |
-
a list of `num_experts` one-dimensional `Tensor`s with type `tf.float32`
|
| 112 |
-
and shapes `[expert_batch_size_i]`
|
| 113 |
-
"""
|
| 114 |
-
# split nonzero gates for each expert
|
| 115 |
-
return torch.split(self._nonzero_gates, self._part_sizes, dim=0)
|
| 116 |
-
|
| 117 |
-
class Bottleneck(nn.Module):
|
| 118 |
-
expansion = 4
|
| 119 |
-
|
| 120 |
-
def __init__(self, inplanes, planes, stride=1):
|
| 121 |
-
super().__init__()
|
| 122 |
-
|
| 123 |
-
# all conv layers have stride 1. an avgpool is performed after the second convolution when stride > 1
|
| 124 |
-
self.conv1 = nn.Conv2d(inplanes, planes, 1, bias=False)
|
| 125 |
-
self.bn1 = nn.BatchNorm2d(planes)
|
| 126 |
-
|
| 127 |
-
self.conv2 = nn.Conv2d(planes, planes, 3, padding=1, bias=False)
|
| 128 |
-
self.bn2 = nn.BatchNorm2d(planes)
|
| 129 |
-
|
| 130 |
-
self.avgpool = nn.AvgPool2d(stride) if stride > 1 else nn.Identity()
|
| 131 |
-
|
| 132 |
-
self.conv3 = nn.Conv2d(planes, planes * self.expansion, 1, bias=False)
|
| 133 |
-
self.bn3 = nn.BatchNorm2d(planes * self.expansion)
|
| 134 |
-
|
| 135 |
-
self.relu = nn.ReLU(inplace=True)
|
| 136 |
-
self.downsample = None
|
| 137 |
-
self.stride = stride
|
| 138 |
-
|
| 139 |
-
if stride > 1 or inplanes != planes * Bottleneck.expansion:
|
| 140 |
-
# downsampling layer is prepended with an avgpool, and the subsequent convolution has stride 1
|
| 141 |
-
self.downsample = nn.Sequential(OrderedDict([
|
| 142 |
-
("-1", nn.AvgPool2d(stride)),
|
| 143 |
-
("0", nn.Conv2d(inplanes, planes * self.expansion, 1, stride=1, bias=False)),
|
| 144 |
-
("1", nn.BatchNorm2d(planes * self.expansion))
|
| 145 |
-
]))
|
| 146 |
-
|
| 147 |
-
def forward(self, x: torch.Tensor):
|
| 148 |
-
identity = x
|
| 149 |
-
|
| 150 |
-
out = self.relu(self.bn1(self.conv1(x)))
|
| 151 |
-
out = self.relu(self.bn2(self.conv2(out)))
|
| 152 |
-
out = self.avgpool(out)
|
| 153 |
-
out = self.bn3(self.conv3(out))
|
| 154 |
-
|
| 155 |
-
if self.downsample is not None:
|
| 156 |
-
identity = self.downsample(x)
|
| 157 |
-
|
| 158 |
-
out += identity
|
| 159 |
-
out = self.relu(out)
|
| 160 |
-
return out
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
class AttentionPool2d(nn.Module):
|
| 164 |
-
def __init__(self, spacial_dim: int, embed_dim: int, num_heads: int, output_dim: int = None):
|
| 165 |
-
super().__init__()
|
| 166 |
-
self.positional_embedding = nn.Parameter(torch.randn(spacial_dim ** 2 + 1, embed_dim) / embed_dim ** 0.5)
|
| 167 |
-
self.k_proj = nn.Linear(embed_dim, embed_dim)
|
| 168 |
-
self.q_proj = nn.Linear(embed_dim, embed_dim)
|
| 169 |
-
self.v_proj = nn.Linear(embed_dim, embed_dim)
|
| 170 |
-
self.c_proj = nn.Linear(embed_dim, output_dim or embed_dim)
|
| 171 |
-
self.num_heads = num_heads
|
| 172 |
-
|
| 173 |
-
def forward(self, x):
|
| 174 |
-
x = x.reshape(x.shape[0], x.shape[1], x.shape[2] * x.shape[3]).permute(2, 0, 1) # NCHW -> (HW)NC
|
| 175 |
-
x = torch.cat([x.mean(dim=0, keepdim=True), x], dim=0) # (HW+1)NC
|
| 176 |
-
x = x + self.positional_embedding[:, None, :].to(x.dtype) # (HW+1)NC
|
| 177 |
-
x, _ = F.multi_head_attention_forward(
|
| 178 |
-
query=x, key=x, value=x,
|
| 179 |
-
embed_dim_to_check=x.shape[-1],
|
| 180 |
-
num_heads=self.num_heads,
|
| 181 |
-
q_proj_weight=self.q_proj.weight,
|
| 182 |
-
k_proj_weight=self.k_proj.weight,
|
| 183 |
-
v_proj_weight=self.v_proj.weight,
|
| 184 |
-
in_proj_weight=None,
|
| 185 |
-
in_proj_bias=torch.cat([self.q_proj.bias, self.k_proj.bias, self.v_proj.bias]),
|
| 186 |
-
bias_k=None,
|
| 187 |
-
bias_v=None,
|
| 188 |
-
add_zero_attn=False,
|
| 189 |
-
dropout_p=0,
|
| 190 |
-
out_proj_weight=self.c_proj.weight,
|
| 191 |
-
out_proj_bias=self.c_proj.bias,
|
| 192 |
-
use_separate_proj_weight=True,
|
| 193 |
-
training=self.training,
|
| 194 |
-
need_weights=False
|
| 195 |
-
)
|
| 196 |
-
|
| 197 |
-
return x[0]
|
| 198 |
-
|
| 199 |
-
|
| 200 |
-
class ModifiedResNet(nn.Module):
|
| 201 |
-
"""
|
| 202 |
-
A ResNet class that is similar to torchvision's but contains the following changes:
|
| 203 |
-
- There are now 3 "stem" convolutions as opposed to 1, with an average pool instead of a max pool.
|
| 204 |
-
- Performs anti-aliasing strided convolutions, where an avgpool is prepended to convolutions with stride > 1
|
| 205 |
-
- The final pooling layer is a QKV attention instead of an average pool
|
| 206 |
-
"""
|
| 207 |
-
|
| 208 |
-
def __init__(self, layers, output_dim, heads, input_resolution=224, width=64):
|
| 209 |
-
super().__init__()
|
| 210 |
-
self.output_dim = output_dim
|
| 211 |
-
self.input_resolution = input_resolution
|
| 212 |
-
|
| 213 |
-
# the 3-layer stem
|
| 214 |
-
self.conv1 = nn.Conv2d(3, width // 2, kernel_size=3, stride=2, padding=1, bias=False)
|
| 215 |
-
self.bn1 = nn.BatchNorm2d(width // 2)
|
| 216 |
-
self.conv2 = nn.Conv2d(width // 2, width // 2, kernel_size=3, padding=1, bias=False)
|
| 217 |
-
self.bn2 = nn.BatchNorm2d(width // 2)
|
| 218 |
-
self.conv3 = nn.Conv2d(width // 2, width, kernel_size=3, padding=1, bias=False)
|
| 219 |
-
self.bn3 = nn.BatchNorm2d(width)
|
| 220 |
-
self.avgpool = nn.AvgPool2d(2)
|
| 221 |
-
self.relu = nn.ReLU(inplace=True)
|
| 222 |
-
|
| 223 |
-
# residual layers
|
| 224 |
-
self._inplanes = width # this is a *mutable* variable used during construction
|
| 225 |
-
self.layer1 = self._make_layer(width, layers[0])
|
| 226 |
-
self.layer2 = self._make_layer(width * 2, layers[1], stride=2)
|
| 227 |
-
self.layer3 = self._make_layer(width * 4, layers[2], stride=2)
|
| 228 |
-
self.layer4 = self._make_layer(width * 8, layers[3], stride=2)
|
| 229 |
-
|
| 230 |
-
embed_dim = width * 32 # the ResNet feature dimension
|
| 231 |
-
self.attnpool = AttentionPool2d(input_resolution // 32, embed_dim, heads, output_dim)
|
| 232 |
-
|
| 233 |
-
def _make_layer(self, planes, blocks, stride=1):
|
| 234 |
-
layers = [Bottleneck(self._inplanes, planes, stride)]
|
| 235 |
-
|
| 236 |
-
self._inplanes = planes * Bottleneck.expansion
|
| 237 |
-
for _ in range(1, blocks):
|
| 238 |
-
layers.append(Bottleneck(self._inplanes, planes))
|
| 239 |
-
|
| 240 |
-
return nn.Sequential(*layers)
|
| 241 |
-
|
| 242 |
-
def forward(self, x):
|
| 243 |
-
def stem(x):
|
| 244 |
-
for conv, bn in [(self.conv1, self.bn1), (self.conv2, self.bn2), (self.conv3, self.bn3)]:
|
| 245 |
-
x = self.relu(bn(conv(x)))
|
| 246 |
-
x = self.avgpool(x)
|
| 247 |
-
return x
|
| 248 |
-
|
| 249 |
-
x = x.type(self.conv1.weight.dtype)
|
| 250 |
-
x = stem(x)
|
| 251 |
-
x = self.layer1(x)
|
| 252 |
-
x = self.layer2(x)
|
| 253 |
-
x = self.layer3(x)
|
| 254 |
-
x = self.layer4(x)
|
| 255 |
-
x = self.attnpool(x)
|
| 256 |
-
|
| 257 |
-
return x
|
| 258 |
-
|
| 259 |
-
|
| 260 |
-
class LayerNorm(nn.LayerNorm):
|
| 261 |
-
"""Subclass torch's LayerNorm to handle fp16."""
|
| 262 |
-
|
| 263 |
-
def forward(self, x: torch.Tensor):
|
| 264 |
-
orig_type = x.dtype
|
| 265 |
-
ret = super().forward(x.type(torch.float32))
|
| 266 |
-
return ret.type(orig_type)
|
| 267 |
-
|
| 268 |
-
|
| 269 |
-
class QuickGELU(nn.Module):
|
| 270 |
-
def forward(self, x: torch.Tensor):
|
| 271 |
-
return x * torch.sigmoid(1.702 * x)
|
| 272 |
-
|
| 273 |
-
|
| 274 |
-
class ResidualAttentionBlock(nn.Module):
|
| 275 |
-
def __init__(self, d_model: int, n_head: int, attn_mask: torch.Tensor = None, text_or_image=None):
|
| 276 |
-
super().__init__()
|
| 277 |
-
self.register_buffer("mean", torch.tensor([0.0]))
|
| 278 |
-
self.register_buffer("std", torch.tensor([1.0]))
|
| 279 |
-
self.attn = nn.MultiheadAttention(d_model, n_head)
|
| 280 |
-
self.ln_1 = LayerNorm(d_model)
|
| 281 |
-
self.mlp = nn.Sequential(OrderedDict([
|
| 282 |
-
("c_fc", nn.Linear(d_model, d_model * 4)),
|
| 283 |
-
("gelu", QuickGELU()),
|
| 284 |
-
("c_proj", nn.Linear(d_model * 4, d_model))
|
| 285 |
-
]))
|
| 286 |
-
self.ln_2 = LayerNorm(d_model)
|
| 287 |
-
self.attn_mask = attn_mask
|
| 288 |
-
self.is_train = global_is_train
|
| 289 |
-
self.step = 1
|
| 290 |
-
self.top_k = 2
|
| 291 |
-
self.ffn_num = 64
|
| 292 |
-
self.experts_num = 2
|
| 293 |
-
self.softmax = nn.Softmax(1)
|
| 294 |
-
self.softplus = nn.Softplus()
|
| 295 |
-
self.noisy_gating = True
|
| 296 |
-
self.adaptmlp_list = nn.ModuleList()
|
| 297 |
-
self.text_or_image = text_or_image
|
| 298 |
-
if text_or_image == 'text':
|
| 299 |
-
# print('text transformer')
|
| 300 |
-
self.choose_map_text = torch.zeros([ self.experts_num])
|
| 301 |
-
else:
|
| 302 |
-
# print('image transformer')
|
| 303 |
-
self.choose_map_image = torch.zeros([ self.experts_num])
|
| 304 |
-
self.router_list = nn.ParameterList()
|
| 305 |
-
self.w_noise_list = nn.ParameterList()
|
| 306 |
-
for i in range(self.step):
|
| 307 |
-
self.router_list.append(nn.Parameter(torch.zeros(d_model, self.experts_num), requires_grad=True))
|
| 308 |
-
self.w_noise_list.append(nn.Parameter(torch.zeros(d_model, self.experts_num), requires_grad=True))
|
| 309 |
-
for i in range(self.experts_num): #
|
| 310 |
-
self.adaptmlp = Adapter(d_model=d_model, dropout=0.1, bottleneck=self.ffn_num,
|
| 311 |
-
init_option='lora',
|
| 312 |
-
adapter_scalar=0.1,
|
| 313 |
-
adapter_layernorm_option='none',
|
| 314 |
-
)
|
| 315 |
-
self.adaptmlp_list.append(self.adaptmlp)
|
| 316 |
-
|
| 317 |
-
# self.taskid = None
|
| 318 |
-
def attention(self, x: torch.Tensor):
|
| 319 |
-
self.attn_mask = self.attn_mask.to(dtype=x.dtype, device=x.device) if self.attn_mask is not None else None
|
| 320 |
-
return self.attn(x, x, x, need_weights=False, attn_mask=self.attn_mask)[0]
|
| 321 |
-
|
| 322 |
-
def cv_squared(self, x):
|
| 323 |
-
"""The squared coefficient of variation of a sample.
|
| 324 |
-
Useful as a loss to encourage a positive distribution to be more uniform.
|
| 325 |
-
Epsilons added for numerical stability.
|
| 326 |
-
Returns 0 for an empty Tensor.
|
| 327 |
-
Args:
|
| 328 |
-
x: a `Tensor`.
|
| 329 |
-
Returns:
|
| 330 |
-
a `Scalar`.
|
| 331 |
-
"""
|
| 332 |
-
eps = 1e-10
|
| 333 |
-
# if only num_experts = 1
|
| 334 |
-
|
| 335 |
-
if x.shape[0] == 1:
|
| 336 |
-
return torch.tensor([0], device=x.device, dtype=x.dtype)
|
| 337 |
-
return x.float().var() / (x.float().mean()**2 + eps)
|
| 338 |
-
|
| 339 |
-
def _gates_to_load(self, gates):
|
| 340 |
-
"""Compute the true load per expert, given the gates.
|
| 341 |
-
The load is the number of examples for which the corresponding gate is >0.
|
| 342 |
-
Args:
|
| 343 |
-
gates: a `Tensor` of shape [batch_size, n]
|
| 344 |
-
Returns:
|
| 345 |
-
a float32 `Tensor` of shape [n]
|
| 346 |
-
"""
|
| 347 |
-
return (gates > 0).sum(0)
|
| 348 |
-
|
| 349 |
-
def _prob_in_top_k(self, clean_values, noisy_values, noise_stddev, noisy_top_values):
|
| 350 |
-
"""Helper function to NoisyTopKGating.
|
| 351 |
-
Computes the probability that value is in top k, given different random noise.
|
| 352 |
-
This gives us a way of backpropagating from a loss that balances the number
|
| 353 |
-
of times each expert is in the top k experts per example.
|
| 354 |
-
In the case of no noise, pass in None for noise_stddev, and the result will
|
| 355 |
-
not be differentiable.
|
| 356 |
-
Args:
|
| 357 |
-
clean_values: a `Tensor` of shape [batch, n].
|
| 358 |
-
noisy_values: a `Tensor` of shape [batch, n]. Equal to clean values plus
|
| 359 |
-
normally distributed noise with standard deviation noise_stddev.
|
| 360 |
-
noise_stddev: a `Tensor` of shape [batch, n], or None
|
| 361 |
-
noisy_top_values: a `Tensor` of shape [batch, m].
|
| 362 |
-
"values" Output of tf.top_k(noisy_top_values, m). m >= k+1
|
| 363 |
-
Returns:
|
| 364 |
-
a `Tensor` of shape [batch, n].
|
| 365 |
-
"""
|
| 366 |
-
# print('1231',clean_values) # 全nan
|
| 367 |
-
batch = clean_values.size(0)
|
| 368 |
-
m = noisy_top_values.size(1)
|
| 369 |
-
top_values_flat = noisy_top_values.flatten()
|
| 370 |
-
|
| 371 |
-
threshold_positions_if_in = torch.arange(batch, device=clean_values.device) * m + self.top_k
|
| 372 |
-
threshold_if_in = torch.unsqueeze(torch.gather(top_values_flat, 0, threshold_positions_if_in), 1)
|
| 373 |
-
is_in = torch.gt(noisy_values, threshold_if_in)
|
| 374 |
-
threshold_positions_if_out = threshold_positions_if_in - 1
|
| 375 |
-
threshold_if_out = torch.unsqueeze(torch.gather(top_values_flat, 0, threshold_positions_if_out), 1)
|
| 376 |
-
# is each value currently in the top k.
|
| 377 |
-
normal = Normal(self.mean, self.std)
|
| 378 |
-
#
|
| 379 |
-
|
| 380 |
-
prob_if_in = normal.cdf((clean_values - threshold_if_in)/noise_stddev)
|
| 381 |
-
prob_if_out = normal.cdf((clean_values - threshold_if_out)/noise_stddev)
|
| 382 |
-
prob = torch.where(is_in, prob_if_in, prob_if_out)
|
| 383 |
-
return prob
|
| 384 |
-
|
| 385 |
-
def noisy_top_k_gating(self, x, train, w_gate, w_noise, noise_epsilon=1e-2):
|
| 386 |
-
"""Noisy top-k gating.
|
| 387 |
-
See paper: https://arxiv.org/abs/1701.06538.
|
| 388 |
-
Args:
|
| 389 |
-
x: input Tensor with shape [batch_size, input_size]
|
| 390 |
-
train: a boolean - we only add noise at training time.
|
| 391 |
-
noise_epsilon: a float
|
| 392 |
-
Returns:
|
| 393 |
-
gates: a Tensor with shape [batch_size, num_experts]
|
| 394 |
-
load: a Tensor with shape [num_experts]
|
| 395 |
-
"""
|
| 396 |
-
|
| 397 |
-
clean_logits = x @ w_gate.to(x)
|
| 398 |
-
if self.noisy_gating and train:
|
| 399 |
-
raw_noise_stddev = x @ w_noise.to(x)
|
| 400 |
-
noise_stddev = ((self.softplus(raw_noise_stddev) + noise_epsilon))
|
| 401 |
-
noisy_logits = clean_logits + (torch.randn_like(clean_logits) * noise_stddev)
|
| 402 |
-
logits = noisy_logits
|
| 403 |
-
else:
|
| 404 |
-
logits = clean_logits
|
| 405 |
-
# calculate topk + 1 that will be needed for the noisy gates
|
| 406 |
-
top_logits, top_indices = logits.topk(min(self.top_k + 1, self.experts_num), dim=1)
|
| 407 |
-
top_k_logits = top_logits[:, :self.top_k]
|
| 408 |
-
top_k_indices = top_indices[:, :self.top_k]
|
| 409 |
-
top_k_gates = self.softmax(top_k_logits)
|
| 410 |
-
zeros = torch.zeros_like(logits)
|
| 411 |
-
gates = zeros.scatter(1, top_k_indices, top_k_gates)
|
| 412 |
-
if self.noisy_gating and self.top_k < self.experts_num and train: # 目前未用上
|
| 413 |
-
load = (self._prob_in_top_k(clean_logits, noisy_logits, noise_stddev, top_logits)).sum(0)
|
| 414 |
-
else:
|
| 415 |
-
load = self._gates_to_load(gates)
|
| 416 |
-
return gates, load
|
| 417 |
-
|
| 418 |
-
def forward(self, x: torch.Tensor):
|
| 419 |
-
x = x + self.attention(self.ln_1(x))
|
| 420 |
-
if global_taskid is not None:
|
| 421 |
-
x_re = x.permute(1, 0, 2)[:, 0, :]
|
| 422 |
-
gates, load = self.noisy_top_k_gating(x_re, self.is_train, self.router_list[global_taskid],
|
| 423 |
-
self.w_noise_list[global_taskid])
|
| 424 |
-
importance = gates.sum(0)
|
| 425 |
-
|
| 426 |
-
nonzero_indices = torch.nonzero(gates)
|
| 427 |
-
counter = Counter(nonzero_indices[:, 1].tolist())
|
| 428 |
-
for number, count in counter.items():
|
| 429 |
-
if self.text_or_image == 'text':
|
| 430 |
-
self.choose_map_text[number] = self.choose_map_text[number] + count
|
| 431 |
-
else:
|
| 432 |
-
self.choose_map_image[number] = self.choose_map_image[number] + count
|
| 433 |
-
dispatcher = SparseDispatcher(self.experts_num, gates)
|
| 434 |
-
expert_inputs = dispatcher.dispatch(x.permute(1, 0, 2).view(x.shape[1], -1))
|
| 435 |
-
expert_outputs = [self.adaptmlp_list[i](expert_inputs[i].view(expert_inputs[i].shape[0],
|
| 436 |
-
x.shape[0], x.shape[2]).to(x), add_residual=False)
|
| 437 |
-
for i in range(self.experts_num)]
|
| 438 |
-
|
| 439 |
-
i = 0
|
| 440 |
-
while i < len(expert_outputs):
|
| 441 |
-
if expert_outputs[i].shape[0] == 0:
|
| 442 |
-
expert_outputs.pop(i)
|
| 443 |
-
else:
|
| 444 |
-
expert_outputs[i] = expert_outputs[i].view(expert_outputs[i].shape[0], -1)
|
| 445 |
-
i += 1
|
| 446 |
-
|
| 447 |
-
y = dispatcher.combine(expert_outputs)
|
| 448 |
-
y = y.view(x.shape[1], x.shape[0], x.shape[2])
|
| 449 |
-
x = x + self.mlp(self.ln_2(x)) + y.permute(1, 0, 2)
|
| 450 |
-
else:
|
| 451 |
-
x = x + self.mlp(self.ln_2(x))
|
| 452 |
-
return x
|
| 453 |
-
|
| 454 |
-
|
| 455 |
-
class Transformer(nn.Module):
|
| 456 |
-
def __init__(self, width: int, layers: int, heads: int, attn_mask: torch.Tensor = None, text_or_image=None):
|
| 457 |
-
super().__init__()
|
| 458 |
-
self.width = width
|
| 459 |
-
self.layers = layers
|
| 460 |
-
self.resblocks = nn.Sequential(*[ResidualAttentionBlock(width, heads, attn_mask, text_or_image) for _ in range(layers)])
|
| 461 |
-
|
| 462 |
-
def forward(self, x: torch.Tensor):
|
| 463 |
-
return self.resblocks(x)
|
| 464 |
-
|
| 465 |
-
|
| 466 |
-
class VisualTransformer(nn.Module):
|
| 467 |
-
def __init__(self, input_resolution: int, patch_size: int, width: int, layers: int, heads: int, output_dim: int, text_or_image=None):
|
| 468 |
-
super().__init__()
|
| 469 |
-
self.input_resolution = input_resolution
|
| 470 |
-
self.output_dim = output_dim
|
| 471 |
-
# Added so this info is available. should not change anything.
|
| 472 |
-
self.patch_size = patch_size
|
| 473 |
-
self.width = width
|
| 474 |
-
self.layers = layers
|
| 475 |
-
self.heads = heads
|
| 476 |
-
|
| 477 |
-
self.conv1 = nn.Conv2d(in_channels=3, out_channels=width, kernel_size=patch_size, stride=patch_size, bias=False)
|
| 478 |
-
|
| 479 |
-
scale = width ** -0.5
|
| 480 |
-
self.class_embedding = nn.Parameter(scale * torch.randn(width))
|
| 481 |
-
self.positional_embedding = nn.Parameter(scale * torch.randn((input_resolution // patch_size) ** 2 + 1, width))
|
| 482 |
-
self.ln_pre = LayerNorm(width)
|
| 483 |
-
|
| 484 |
-
self.transformer = Transformer(width, layers, heads, text_or_image=text_or_image)
|
| 485 |
-
|
| 486 |
-
self.ln_post = LayerNorm(width)
|
| 487 |
-
self.proj = nn.Parameter(scale * torch.randn(width, output_dim))
|
| 488 |
-
|
| 489 |
-
def forward(self, x: torch.Tensor):
|
| 490 |
-
x = self.conv1(x)
|
| 491 |
-
x = x.reshape(x.shape[0], x.shape[1], -1)
|
| 492 |
-
x = x.permute(0, 2, 1)
|
| 493 |
-
x = torch.cat([self.class_embedding.to(x.dtype) + torch.zeros(x.shape[0], 1, x.shape[-1], dtype=x.dtype, device=x.device), x], dim=1) # shape = [*, grid ** 2 + 1, width]
|
| 494 |
-
x = x + self.positional_embedding.to(x.dtype)
|
| 495 |
-
x = self.ln_pre(x)
|
| 496 |
-
|
| 497 |
-
x = x.permute(1, 0, 2) # NLD -> LND
|
| 498 |
-
x = self.transformer(x)
|
| 499 |
-
x = x.permute(1, 0, 2) # LND -> NLD
|
| 500 |
-
|
| 501 |
-
x = self.ln_post(x[:, 0, :])
|
| 502 |
-
|
| 503 |
-
if self.proj is not None:
|
| 504 |
-
x = x @ self.proj
|
| 505 |
-
|
| 506 |
-
return x
|
| 507 |
-
|
| 508 |
-
|
| 509 |
-
class CLIP(nn.Module):
|
| 510 |
-
def __init__(self,
|
| 511 |
-
embed_dim: int,
|
| 512 |
-
# vision
|
| 513 |
-
image_resolution: int,
|
| 514 |
-
vision_layers: Union[Tuple[int, int, int, int], int],
|
| 515 |
-
vision_width: int,
|
| 516 |
-
vision_patch_size: int,
|
| 517 |
-
# text
|
| 518 |
-
context_length: int,
|
| 519 |
-
vocab_size: int,
|
| 520 |
-
transformer_width: int,
|
| 521 |
-
transformer_heads: int,
|
| 522 |
-
transformer_layers: int,
|
| 523 |
-
baseline = False
|
| 524 |
-
):
|
| 525 |
-
super().__init__()
|
| 526 |
-
self.baseline = baseline
|
| 527 |
-
|
| 528 |
-
self.context_length = context_length
|
| 529 |
-
|
| 530 |
-
if isinstance(vision_layers, (tuple, list)):
|
| 531 |
-
vision_heads = vision_width * 32 // 64
|
| 532 |
-
self.visual = ModifiedResNet(
|
| 533 |
-
layers=vision_layers,
|
| 534 |
-
output_dim=embed_dim,
|
| 535 |
-
heads=vision_heads,
|
| 536 |
-
input_resolution=image_resolution,
|
| 537 |
-
width=vision_width
|
| 538 |
-
)
|
| 539 |
-
else:
|
| 540 |
-
vision_heads = vision_width // 64
|
| 541 |
-
self.visual = VisualTransformer(
|
| 542 |
-
input_resolution=image_resolution,
|
| 543 |
-
patch_size=vision_patch_size,
|
| 544 |
-
width=vision_width,
|
| 545 |
-
layers=vision_layers,
|
| 546 |
-
heads=vision_heads,
|
| 547 |
-
output_dim=embed_dim,
|
| 548 |
-
text_or_image='image'
|
| 549 |
-
)
|
| 550 |
-
|
| 551 |
-
self.transformer = Transformer(
|
| 552 |
-
width=transformer_width,
|
| 553 |
-
layers=transformer_layers,
|
| 554 |
-
heads=transformer_heads,
|
| 555 |
-
attn_mask=self.build_attention_mask(),
|
| 556 |
-
text_or_image='text'
|
| 557 |
-
)
|
| 558 |
-
|
| 559 |
-
self.vocab_size = vocab_size
|
| 560 |
-
self.token_embedding = nn.Embedding(vocab_size, transformer_width)
|
| 561 |
-
self.positional_embedding = nn.Parameter(torch.empty(self.context_length, transformer_width))
|
| 562 |
-
self.ln_final = LayerNorm(transformer_width)
|
| 563 |
-
|
| 564 |
-
self.text_projection = nn.Parameter(torch.empty(transformer_width, embed_dim))
|
| 565 |
-
self.logit_scale = nn.Parameter(torch.ones([]) * np.log(1 / 0.07))
|
| 566 |
-
|
| 567 |
-
self.initialize_parameters()
|
| 568 |
-
|
| 569 |
-
def initialize_parameters(self):
|
| 570 |
-
nn.init.normal_(self.token_embedding.weight, std=0.02)
|
| 571 |
-
nn.init.normal_(self.positional_embedding, std=0.01)
|
| 572 |
-
self.logit_scale = nn.Parameter(torch.ones([]) * np.log(1 / 0.07))
|
| 573 |
-
|
| 574 |
-
if isinstance(self.visual, ModifiedResNet):
|
| 575 |
-
if self.visual.attnpool is not None:
|
| 576 |
-
std = self.visual.attnpool.c_proj.in_features ** -0.5
|
| 577 |
-
nn.init.normal_(self.visual.attnpool.q_proj.weight, std=std)
|
| 578 |
-
nn.init.normal_(self.visual.attnpool.k_proj.weight, std=std)
|
| 579 |
-
nn.init.normal_(self.visual.attnpool.v_proj.weight, std=std)
|
| 580 |
-
nn.init.normal_(self.visual.attnpool.c_proj.weight, std=std)
|
| 581 |
-
|
| 582 |
-
for resnet_block in [self.visual.layer1, self.visual.layer2, self.visual.layer3, self.visual.layer4]:
|
| 583 |
-
for name, param in resnet_block.named_parameters():
|
| 584 |
-
if name.endswith("bn3.weight"):
|
| 585 |
-
nn.init.zeros_(param)
|
| 586 |
-
|
| 587 |
-
proj_std = (self.transformer.width ** -0.5) * ((2 * self.transformer.layers) ** -0.5)
|
| 588 |
-
attn_std = self.transformer.width ** -0.5
|
| 589 |
-
fc_std = (2 * self.transformer.width) ** -0.5
|
| 590 |
-
for block in self.transformer.resblocks:
|
| 591 |
-
nn.init.normal_(block.attn.in_proj_weight, std=attn_std)
|
| 592 |
-
nn.init.normal_(block.attn.out_proj.weight, std=proj_std)
|
| 593 |
-
nn.init.normal_(block.mlp.c_fc.weight, std=fc_std)
|
| 594 |
-
nn.init.normal_(block.mlp.c_proj.weight, std=proj_std)
|
| 595 |
-
|
| 596 |
-
if self.text_projection is not None:
|
| 597 |
-
nn.init.normal_(self.text_projection, std=self.transformer.width ** -0.5)
|
| 598 |
-
|
| 599 |
-
def build_attention_mask(self):
|
| 600 |
-
# lazily create causal attention mask, with full attention between the vision tokens
|
| 601 |
-
# pytorch uses additive attention mask; fill with -inf
|
| 602 |
-
mask = torch.empty(self.context_length, self.context_length)
|
| 603 |
-
mask.fill_(float("-inf"))
|
| 604 |
-
mask.triu_(1) # zero out the lower diagonal
|
| 605 |
-
return mask
|
| 606 |
-
|
| 607 |
-
@property
|
| 608 |
-
def dtype(self):
|
| 609 |
-
return self.visual.conv1.weight.dtype
|
| 610 |
-
|
| 611 |
-
def encode_image(self, image):
|
| 612 |
-
return self.visual(image.type(self.dtype))
|
| 613 |
-
|
| 614 |
-
def encode_text(self, text):
|
| 615 |
-
|
| 616 |
-
x = self.token_embedding(text).type(self.dtype) # [batch_size, n_ctx, d_model]
|
| 617 |
-
|
| 618 |
-
x = x + self.positional_embedding.type(self.dtype)
|
| 619 |
-
x = x.permute(1, 0, 2) # NLD -> LND
|
| 620 |
-
x = self.transformer(x)
|
| 621 |
-
x = x.permute(1, 0, 2) # LND -> NLD
|
| 622 |
-
x = self.ln_final(x).type(self.dtype)
|
| 623 |
-
|
| 624 |
-
# take features from the eot embedding (eot_token is the highest number in each sequence)
|
| 625 |
-
x = x[torch.arange(x.shape[0]), text.argmax(dim=-1)] @ self.text_projection
|
| 626 |
-
|
| 627 |
-
return x
|
| 628 |
-
|
| 629 |
-
def forward(self, image, text, taskid, is_train):
|
| 630 |
-
global global_taskid, global_is_train
|
| 631 |
-
global_taskid = taskid
|
| 632 |
-
global_is_train = is_train
|
| 633 |
-
if image is None:
|
| 634 |
-
return self.encode_text(text)
|
| 635 |
-
elif text is None:
|
| 636 |
-
return self.encode_image(image)
|
| 637 |
-
image_features = self.encode_image(image)
|
| 638 |
-
text_features = self.encode_text(text)
|
| 639 |
-
|
| 640 |
-
image_features = image_features / image_features.norm(dim=-1, keepdim=True)
|
| 641 |
-
text_features = text_features / text_features.norm(dim=-1, keepdim=True)
|
| 642 |
-
|
| 643 |
-
# if self.baseline:
|
| 644 |
-
logit_scale = self.logit_scale.exp()
|
| 645 |
-
logits_per_image = logit_scale * image_features @ text_features.t()
|
| 646 |
-
logits_per_text = logits_per_image.t()
|
| 647 |
-
return logits_per_image, logits_per_text
|
| 648 |
-
|
| 649 |
-
|
| 650 |
-
|
| 651 |
-
def convert_weights(model: nn.Module):
|
| 652 |
-
"""Convert applicable model parameters to fp16"""
|
| 653 |
-
|
| 654 |
-
def _convert_weights_to_fp16(l):
|
| 655 |
-
if isinstance(l, (nn.Conv1d, nn.Conv2d, nn.Linear)):
|
| 656 |
-
l.weight.data = l.weight.data.half()
|
| 657 |
-
if l.bias is not None:
|
| 658 |
-
l.bias.data = l.bias.data.half()
|
| 659 |
-
|
| 660 |
-
if isinstance(l, nn.MultiheadAttention):
|
| 661 |
-
for attr in [*[f"{s}_proj_weight" for s in ["in", "q", "k", "v"]], "in_proj_bias", "bias_k", "bias_v"]:
|
| 662 |
-
tensor = getattr(l, attr)
|
| 663 |
-
if tensor is not None:
|
| 664 |
-
tensor.data = tensor.data.half()
|
| 665 |
-
|
| 666 |
-
for name in ["text_projection", "proj"]:
|
| 667 |
-
if hasattr(l, name):
|
| 668 |
-
attr = getattr(l, name)
|
| 669 |
-
if attr is not None:
|
| 670 |
-
attr.data = attr.data.half()
|
| 671 |
-
|
| 672 |
-
model.apply(_convert_weights_to_fp16)
|
| 673 |
-
|
| 674 |
-
|
| 675 |
-
def build_model(state_dict: dict):
|
| 676 |
-
vit = "visual.proj" in state_dict
|
| 677 |
-
|
| 678 |
-
if vit:
|
| 679 |
-
vision_width = state_dict["visual.conv1.weight"].shape[0]
|
| 680 |
-
vision_layers = len([k for k in state_dict.keys() if k.startswith("visual.") and k.endswith(".attn.in_proj_weight")])
|
| 681 |
-
vision_patch_size = state_dict["visual.conv1.weight"].shape[-1]
|
| 682 |
-
grid_size = round((state_dict["visual.positional_embedding"].shape[0] - 1) ** 0.5)
|
| 683 |
-
image_resolution = vision_patch_size * grid_size
|
| 684 |
-
else:
|
| 685 |
-
counts: list = [len(set(k.split(".")[2] for k in state_dict if k.startswith(f"visual.layer{b}"))) for b in [1, 2, 3, 4]]
|
| 686 |
-
vision_layers = tuple(counts)
|
| 687 |
-
vision_width = state_dict["visual.layer1.0.conv1.weight"].shape[0]
|
| 688 |
-
output_width = round((state_dict["visual.attnpool.positional_embedding"].shape[0] - 1) ** 0.5)
|
| 689 |
-
vision_patch_size = None
|
| 690 |
-
assert output_width ** 2 + 1 == state_dict["visual.attnpool.positional_embedding"].shape[0]
|
| 691 |
-
image_resolution = output_width * 32
|
| 692 |
-
|
| 693 |
-
embed_dim = state_dict["text_projection"].shape[1]
|
| 694 |
-
context_length = state_dict["positional_embedding"].shape[0]
|
| 695 |
-
vocab_size = state_dict["token_embedding.weight"].shape[0]
|
| 696 |
-
transformer_width = state_dict["ln_final.weight"].shape[0]
|
| 697 |
-
transformer_heads = transformer_width // 64
|
| 698 |
-
transformer_layers = len(set(k.split(".")[2] for k in state_dict if k.startswith(f"transformer.resblocks")))
|
| 699 |
-
|
| 700 |
-
model = CLIP(
|
| 701 |
-
embed_dim,
|
| 702 |
-
image_resolution, vision_layers, vision_width, vision_patch_size,
|
| 703 |
-
context_length, vocab_size, transformer_width, transformer_heads, transformer_layers
|
| 704 |
-
)
|
| 705 |
-
|
| 706 |
-
for key in ["input_resolution", "context_length", "vocab_size"]:
|
| 707 |
-
if key in state_dict:
|
| 708 |
-
del state_dict[key]
|
| 709 |
-
|
| 710 |
-
model.load_state_dict(state_dict, strict=False)
|
| 711 |
-
for p in model.parameters():
|
| 712 |
-
p.data = p.data.float()
|
| 713 |
-
return model.eval()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/clip/tokenizer.py
DELETED
|
@@ -1,140 +0,0 @@
|
|
| 1 |
-
import gzip
|
| 2 |
-
import html
|
| 3 |
-
import os
|
| 4 |
-
from functools import lru_cache
|
| 5 |
-
|
| 6 |
-
import ftfy
|
| 7 |
-
import regex as re
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
@lru_cache()
|
| 11 |
-
def default_bpe():
|
| 12 |
-
return os.path.join(os.path.dirname(os.path.abspath(__file__)), "bpe_simple_vocab_16e6.txt.gz")
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
@lru_cache()
|
| 16 |
-
def bytes_to_unicode():
|
| 17 |
-
"""
|
| 18 |
-
Returns list of utf-8 byte and a corresponding list of unicode strings.
|
| 19 |
-
The reversible bpe codes work on unicode strings.
|
| 20 |
-
This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
|
| 21 |
-
When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
|
| 22 |
-
This is a signficant percentage of your normal, say, 32K bpe vocab.
|
| 23 |
-
To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
|
| 24 |
-
And avoids mapping to whitespace/control characters the bpe code barfs on.
|
| 25 |
-
"""
|
| 26 |
-
bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
|
| 27 |
-
cs = bs[:]
|
| 28 |
-
n = 0
|
| 29 |
-
for b in range(2**8):
|
| 30 |
-
if b not in bs:
|
| 31 |
-
bs.append(b)
|
| 32 |
-
cs.append(2**8+n)
|
| 33 |
-
n += 1
|
| 34 |
-
cs = [chr(n) for n in cs]
|
| 35 |
-
return dict(zip(bs, cs))
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
def get_pairs(word):
|
| 39 |
-
"""Return set of symbol pairs in a word.
|
| 40 |
-
Word is represented as tuple of symbols (symbols being variable-length strings).
|
| 41 |
-
"""
|
| 42 |
-
pairs = set()
|
| 43 |
-
prev_char = word[0]
|
| 44 |
-
for char in word[1:]:
|
| 45 |
-
pairs.add((prev_char, char))
|
| 46 |
-
prev_char = char
|
| 47 |
-
return pairs
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
def basic_clean(text):
|
| 51 |
-
text = ftfy.fix_text(text)
|
| 52 |
-
text = html.unescape(html.unescape(text))
|
| 53 |
-
return text.strip()
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
def whitespace_clean(text):
|
| 57 |
-
text = re.sub(r'\s+', ' ', text)
|
| 58 |
-
text = text.strip()
|
| 59 |
-
return text
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
class SimpleTokenizer(object):
|
| 63 |
-
def __init__(self, bpe_path: str = default_bpe(), special_tokens=None):
|
| 64 |
-
self.byte_encoder = bytes_to_unicode()
|
| 65 |
-
self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
|
| 66 |
-
merges = gzip.open(bpe_path).read().decode("utf-8").split('\n')
|
| 67 |
-
merges = merges[1:49152-256-2+1]
|
| 68 |
-
merges = [tuple(merge.split()) for merge in merges]
|
| 69 |
-
vocab = list(bytes_to_unicode().values())
|
| 70 |
-
vocab = vocab + [v+'</w>' for v in vocab]
|
| 71 |
-
for merge in merges:
|
| 72 |
-
vocab.append(''.join(merge))
|
| 73 |
-
if not special_tokens:
|
| 74 |
-
special_tokens = ['<start_of_text>', '<end_of_text>']
|
| 75 |
-
else:
|
| 76 |
-
special_tokens = ['<start_of_text>', '<end_of_text>'] + special_tokens
|
| 77 |
-
vocab.extend(special_tokens)
|
| 78 |
-
self.encoder = dict(zip(vocab, range(len(vocab))))
|
| 79 |
-
self.decoder = {v: k for k, v in self.encoder.items()}
|
| 80 |
-
self.bpe_ranks = dict(zip(merges, range(len(merges))))
|
| 81 |
-
self.cache = {t:t for t in special_tokens}
|
| 82 |
-
special = "|".join(special_tokens)
|
| 83 |
-
self.pat = re.compile(special + r"""|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""", re.IGNORECASE)
|
| 84 |
-
|
| 85 |
-
self.vocab_size = len(self.encoder)
|
| 86 |
-
self.all_special_ids = [self.encoder[t] for t in special_tokens]
|
| 87 |
-
|
| 88 |
-
def bpe(self, token):
|
| 89 |
-
if token in self.cache:
|
| 90 |
-
return self.cache[token]
|
| 91 |
-
word = tuple(token[:-1]) + ( token[-1] + '</w>',)
|
| 92 |
-
pairs = get_pairs(word)
|
| 93 |
-
|
| 94 |
-
if not pairs:
|
| 95 |
-
return token+'</w>'
|
| 96 |
-
|
| 97 |
-
while True:
|
| 98 |
-
bigram = min(pairs, key = lambda pair: self.bpe_ranks.get(pair, float('inf')))
|
| 99 |
-
if bigram not in self.bpe_ranks:
|
| 100 |
-
break
|
| 101 |
-
first, second = bigram
|
| 102 |
-
new_word = []
|
| 103 |
-
i = 0
|
| 104 |
-
while i < len(word):
|
| 105 |
-
try:
|
| 106 |
-
j = word.index(first, i)
|
| 107 |
-
new_word.extend(word[i:j])
|
| 108 |
-
i = j
|
| 109 |
-
except:
|
| 110 |
-
new_word.extend(word[i:])
|
| 111 |
-
break
|
| 112 |
-
|
| 113 |
-
if word[i] == first and i < len(word)-1 and word[i+1] == second:
|
| 114 |
-
new_word.append(first+second)
|
| 115 |
-
i += 2
|
| 116 |
-
else:
|
| 117 |
-
new_word.append(word[i])
|
| 118 |
-
i += 1
|
| 119 |
-
new_word = tuple(new_word)
|
| 120 |
-
word = new_word
|
| 121 |
-
if len(word) == 1:
|
| 122 |
-
break
|
| 123 |
-
else:
|
| 124 |
-
pairs = get_pairs(word)
|
| 125 |
-
word = ' '.join(word)
|
| 126 |
-
self.cache[token] = word
|
| 127 |
-
return word
|
| 128 |
-
|
| 129 |
-
def encode(self, text):
|
| 130 |
-
bpe_tokens = []
|
| 131 |
-
text = whitespace_clean(basic_clean(text)).lower()
|
| 132 |
-
for token in re.findall(self.pat, text):
|
| 133 |
-
token = ''.join(self.byte_encoder[b] for b in token.encode('utf-8'))
|
| 134 |
-
bpe_tokens.extend(self.encoder[bpe_token] for bpe_token in self.bpe(token).split(' '))
|
| 135 |
-
return bpe_tokens
|
| 136 |
-
|
| 137 |
-
def decode(self, tokens):
|
| 138 |
-
text = ''.join([self.decoder[token] for token in tokens])
|
| 139 |
-
text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors="replace").replace('</w>', ' ')
|
| 140 |
-
return text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/configs/class/aircraft.yaml
DELETED
|
@@ -1,76 +0,0 @@
|
|
| 1 |
-
hydra:
|
| 2 |
-
run:
|
| 3 |
-
dir: ./experiments/Aircraft
|
| 4 |
-
job:
|
| 5 |
-
chdir: true
|
| 6 |
-
|
| 7 |
-
job_logging:
|
| 8 |
-
version: 1
|
| 9 |
-
formatters:
|
| 10 |
-
simple:
|
| 11 |
-
format: '%(message)s'
|
| 12 |
-
|
| 13 |
-
class_order: ""
|
| 14 |
-
dataset_root: ""
|
| 15 |
-
workdir: ""
|
| 16 |
-
log_path: "metrics.json"
|
| 17 |
-
model_name: "ViT-B/16"
|
| 18 |
-
prompt_template: "a bad photo of a {}."
|
| 19 |
-
zero_shot_eval: true
|
| 20 |
-
pre_task_zero_shot_eval: False
|
| 21 |
-
# zs_mtil_indices: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24] # indices of MTIL datasets (0..24) to evaluate in zero-shot; excludes any selected as downstream train_dataset
|
| 22 |
-
zs_mtil_indices: [1] # indices of MTIL datasets (0..24) to evaluate in zero-shot; excludes any selected as downstream train_dataset
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
batch_size: 128
|
| 26 |
-
increment: ${initial_increment}
|
| 27 |
-
initial_increment: 10
|
| 28 |
-
scenario: "class"
|
| 29 |
-
dataset: "cifar100"
|
| 30 |
-
|
| 31 |
-
# method: "lwf"
|
| 32 |
-
# lr: 7.1e-6
|
| 33 |
-
# ls: 0.2
|
| 34 |
-
# we:
|
| 35 |
-
# avg_freq:
|
| 36 |
-
# ref_dataset:
|
| 37 |
-
# ref_sentences:
|
| 38 |
-
|
| 39 |
-
weight_decay: 0.0
|
| 40 |
-
l2: 0
|
| 41 |
-
ce_method: 0
|
| 42 |
-
|
| 43 |
-
method: "MoE-Adapters"
|
| 44 |
-
lr: 1e-3
|
| 45 |
-
ls: 0.0
|
| 46 |
-
#we:
|
| 47 |
-
#avg_freq:
|
| 48 |
-
#ref_dataset:
|
| 49 |
-
#ref_sentences: random
|
| 50 |
-
|
| 51 |
-
# =====================
|
| 52 |
-
# DFA (two-expert + router) hyperparameters
|
| 53 |
-
# =====================
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
# Training schedule
|
| 57 |
-
epochs_a: 1
|
| 58 |
-
epochs_b: 1
|
| 59 |
-
e2_top_k: 2 # top-k for E2 router
|
| 60 |
-
|
| 61 |
-
# Stage A (InfoNCE)
|
| 62 |
-
lr_e1: 1.0e-5
|
| 63 |
-
tau_con: 0.15 # contrastive temperature for InfoNCE
|
| 64 |
-
use_moco_queue: true # enable momentum queue for more negatives
|
| 65 |
-
moco_queue_size: 128 # queue size (default 4096, increases negative samples significantly)
|
| 66 |
-
|
| 67 |
-
# Stage B (Task expert + Router)
|
| 68 |
-
lr_e2: 5.0e-4
|
| 69 |
-
|
| 70 |
-
text_lr_e2: 1.0e-5
|
| 71 |
-
lr_e2_router: 1.0e-3
|
| 72 |
-
lr_top_router: 1.0e-5
|
| 73 |
-
lambda_b_con: 0.001
|
| 74 |
-
tau_b_con: 0.19
|
| 75 |
-
num_task_experts: 2
|
| 76 |
-
seed: 43
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/configs/class/caltech.yaml
DELETED
|
@@ -1,79 +0,0 @@
|
|
| 1 |
-
hydra:
|
| 2 |
-
run:
|
| 3 |
-
dir: ./experiments/Caltech101
|
| 4 |
-
job:
|
| 5 |
-
chdir: true
|
| 6 |
-
|
| 7 |
-
job_logging:
|
| 8 |
-
version: 1
|
| 9 |
-
formatters:
|
| 10 |
-
simple:
|
| 11 |
-
format: '%(message)s'
|
| 12 |
-
|
| 13 |
-
class_order: ""
|
| 14 |
-
dataset_root: ""
|
| 15 |
-
workdir: ""
|
| 16 |
-
log_path: "metrics.json"
|
| 17 |
-
model_name: "ViT-B/16"
|
| 18 |
-
prompt_template: "a bad photo of a {}."
|
| 19 |
-
zero_shot_eval: true
|
| 20 |
-
pre_task_zero_shot_eval: False
|
| 21 |
-
zs_mtil_indices: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24] # indices of MTIL datasets (0..24) to evaluate in zero-shot; excludes any selected as downstream train_dataset
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
batch_size: 128
|
| 26 |
-
increment: ${initial_increment}
|
| 27 |
-
initial_increment: 10
|
| 28 |
-
scenario: "class"
|
| 29 |
-
dataset: "cifar100"
|
| 30 |
-
|
| 31 |
-
# method: "lwf"
|
| 32 |
-
# lr: 7.1e-6
|
| 33 |
-
# ls: 0.2
|
| 34 |
-
# we:
|
| 35 |
-
# avg_freq:
|
| 36 |
-
# ref_dataset:
|
| 37 |
-
# ref_sentences:
|
| 38 |
-
|
| 39 |
-
weight_decay: 0.0
|
| 40 |
-
l2: 0
|
| 41 |
-
ce_method: 0
|
| 42 |
-
|
| 43 |
-
method: "MoE-Adapters"
|
| 44 |
-
lr: 1e-3
|
| 45 |
-
ls: 0.0
|
| 46 |
-
#we:
|
| 47 |
-
#avg_freq:
|
| 48 |
-
#ref_dataset:
|
| 49 |
-
#ref_sentences: random
|
| 50 |
-
|
| 51 |
-
# =====================
|
| 52 |
-
# DFA (two-expert + router) hyperparameters
|
| 53 |
-
# =====================
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
# Training schedule
|
| 57 |
-
epochs_a: 1
|
| 58 |
-
epochs_b: 1
|
| 59 |
-
e2_top_k: 2 # top-k for E2 router
|
| 60 |
-
|
| 61 |
-
# Stage A (InfoNCE)
|
| 62 |
-
lr_e1: 1.0e-5
|
| 63 |
-
tau_con: 0.15 # contrastive temperature for InfoNCE
|
| 64 |
-
use_moco_queue: true # enable momentum queue for more negatives
|
| 65 |
-
moco_queue_size: 128 # queue size (default 4096, increases negative samples significantly)
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
# Stage B (Task expert + Router)
|
| 71 |
-
lr_e2: 5.0e-4
|
| 72 |
-
|
| 73 |
-
text_lr_e2: 1.0e-5
|
| 74 |
-
lr_e2_router: 1.0e-3
|
| 75 |
-
lr_top_router: 1.0e-5
|
| 76 |
-
lambda_b_con: 0.001
|
| 77 |
-
tau_b_con: 0.19
|
| 78 |
-
num_task_experts: 2
|
| 79 |
-
seed: 43
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/configs/class/car.yaml
DELETED
|
@@ -1,79 +0,0 @@
|
|
| 1 |
-
hydra:
|
| 2 |
-
run:
|
| 3 |
-
dir: ./experiments/StanfordCars
|
| 4 |
-
job:
|
| 5 |
-
chdir: true
|
| 6 |
-
|
| 7 |
-
job_logging:
|
| 8 |
-
version: 1
|
| 9 |
-
formatters:
|
| 10 |
-
simple:
|
| 11 |
-
format: '%(message)s'
|
| 12 |
-
|
| 13 |
-
class_order: ""
|
| 14 |
-
dataset_root: ""
|
| 15 |
-
workdir: ""
|
| 16 |
-
log_path: "metrics.json"
|
| 17 |
-
model_name: "ViT-B/16"
|
| 18 |
-
prompt_template: "a bad photo of a {}."
|
| 19 |
-
zero_shot_eval: true
|
| 20 |
-
pre_task_zero_shot_eval: False
|
| 21 |
-
zs_mtil_indices: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24] # indices of MTIL datasets (0..24) to evaluate in zero-shot; excludes any selected as downstream train_dataset
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
batch_size: 128
|
| 26 |
-
increment: ${initial_increment}
|
| 27 |
-
initial_increment: 10
|
| 28 |
-
scenario: "class"
|
| 29 |
-
dataset: "cifar100"
|
| 30 |
-
|
| 31 |
-
# method: "lwf"
|
| 32 |
-
# lr: 7.1e-6
|
| 33 |
-
# ls: 0.2
|
| 34 |
-
# we:
|
| 35 |
-
# avg_freq:
|
| 36 |
-
# ref_dataset:
|
| 37 |
-
# ref_sentences:
|
| 38 |
-
|
| 39 |
-
weight_decay: 0.0
|
| 40 |
-
l2: 0
|
| 41 |
-
ce_method: 0
|
| 42 |
-
|
| 43 |
-
method: "MoE-Adapters"
|
| 44 |
-
lr: 1e-3
|
| 45 |
-
ls: 0.0
|
| 46 |
-
#we:
|
| 47 |
-
#avg_freq:
|
| 48 |
-
#ref_dataset:
|
| 49 |
-
#ref_sentences: random
|
| 50 |
-
|
| 51 |
-
# =====================
|
| 52 |
-
# DFA (two-expert + router) hyperparameters
|
| 53 |
-
# =====================
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
# Training schedule
|
| 57 |
-
epochs_a: 1
|
| 58 |
-
epochs_b: 1
|
| 59 |
-
e2_top_k: 2 # top-k for E2 router
|
| 60 |
-
|
| 61 |
-
# Stage A (InfoNCE)
|
| 62 |
-
lr_e1: 1.0e-5
|
| 63 |
-
tau_con: 0.15 # contrastive temperature for InfoNCE
|
| 64 |
-
use_moco_queue: true # enable momentum queue for more negatives
|
| 65 |
-
moco_queue_size: 128 # queue size (default 4096, increases negative samples significantly)
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
# Stage B (Task expert + Router)
|
| 71 |
-
lr_e2: 5.0e-4
|
| 72 |
-
|
| 73 |
-
text_lr_e2: 1.0e-5
|
| 74 |
-
lr_e2_router: 1.0e-3
|
| 75 |
-
lr_top_router: 1.0e-5
|
| 76 |
-
lambda_b_con: 0.001
|
| 77 |
-
tau_b_con: 0.17
|
| 78 |
-
num_task_experts: 2
|
| 79 |
-
seed: 43
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/configs/class/cifar10.yaml
DELETED
|
@@ -1,79 +0,0 @@
|
|
| 1 |
-
hydra:
|
| 2 |
-
run:
|
| 3 |
-
dir: ./experiments/Cifar10
|
| 4 |
-
job:
|
| 5 |
-
chdir: true
|
| 6 |
-
|
| 7 |
-
job_logging:
|
| 8 |
-
version: 1
|
| 9 |
-
formatters:
|
| 10 |
-
simple:
|
| 11 |
-
format: '%(message)s'
|
| 12 |
-
|
| 13 |
-
class_order: ""
|
| 14 |
-
dataset_root: ""
|
| 15 |
-
workdir: ""
|
| 16 |
-
log_path: "metrics.json"
|
| 17 |
-
model_name: "ViT-B/16"
|
| 18 |
-
prompt_template: "a bad photo of a {}."
|
| 19 |
-
zero_shot_eval: true
|
| 20 |
-
pre_task_zero_shot_eval: False
|
| 21 |
-
zs_mtil_indices: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24] # indices of MTIL datasets (0..24) to evaluate in zero-shot; excludes any selected as downstream train_dataset
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
batch_size: 128
|
| 26 |
-
increment: ${initial_increment}
|
| 27 |
-
initial_increment: 10
|
| 28 |
-
scenario: "class"
|
| 29 |
-
dataset: "cifar100"
|
| 30 |
-
|
| 31 |
-
# method: "lwf"
|
| 32 |
-
# lr: 7.1e-6
|
| 33 |
-
# ls: 0.2
|
| 34 |
-
# we:
|
| 35 |
-
# avg_freq:
|
| 36 |
-
# ref_dataset:
|
| 37 |
-
# ref_sentences:
|
| 38 |
-
|
| 39 |
-
weight_decay: 0.0
|
| 40 |
-
l2: 0
|
| 41 |
-
ce_method: 0
|
| 42 |
-
|
| 43 |
-
method: "MoE-Adapters"
|
| 44 |
-
lr: 1e-3
|
| 45 |
-
ls: 0.0
|
| 46 |
-
#we:
|
| 47 |
-
#avg_freq:
|
| 48 |
-
#ref_dataset:
|
| 49 |
-
#ref_sentences: random
|
| 50 |
-
|
| 51 |
-
# =====================
|
| 52 |
-
# DFA (two-expert + router) hyperparameters
|
| 53 |
-
# =====================
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
# Training schedule
|
| 57 |
-
epochs_a: 1
|
| 58 |
-
epochs_b: 1
|
| 59 |
-
e2_top_k: 2 # top-k for E2 router
|
| 60 |
-
|
| 61 |
-
# Stage A (InfoNCE)
|
| 62 |
-
lr_e1: 1.0e-5
|
| 63 |
-
tau_con: 0.15 # contrastive temperature for InfoNCE
|
| 64 |
-
use_moco_queue: true # enable momentum queue for more negatives
|
| 65 |
-
moco_queue_size: 128 # queue size (default 4096, increases negative samples significantly)
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
# Stage B (Task expert + Router)
|
| 71 |
-
lr_e2: 5.0e-4
|
| 72 |
-
|
| 73 |
-
text_lr_e2: 1.0e-5
|
| 74 |
-
lr_e2_router: 1.0e-3
|
| 75 |
-
lr_top_router: 1.0e-5
|
| 76 |
-
lambda_b_con: 0.001
|
| 77 |
-
tau_b_con: 0.19
|
| 78 |
-
num_task_experts: 2
|
| 79 |
-
seed: 43
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/configs/class/cifar100.yaml
DELETED
|
@@ -1,79 +0,0 @@
|
|
| 1 |
-
hydra:
|
| 2 |
-
run:
|
| 3 |
-
dir: ./experiments/CIFAR100
|
| 4 |
-
job:
|
| 5 |
-
chdir: true
|
| 6 |
-
|
| 7 |
-
job_logging:
|
| 8 |
-
version: 1
|
| 9 |
-
formatters:
|
| 10 |
-
simple:
|
| 11 |
-
format: '%(message)s'
|
| 12 |
-
|
| 13 |
-
class_order: ""
|
| 14 |
-
dataset_root: ""
|
| 15 |
-
workdir: ""
|
| 16 |
-
log_path: "metrics.json"
|
| 17 |
-
model_name: "ViT-B/16"
|
| 18 |
-
prompt_template: "a bad photo of a {}."
|
| 19 |
-
zero_shot_eval: true
|
| 20 |
-
pre_task_zero_shot_eval: False
|
| 21 |
-
zs_mtil_indices: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24] # indices of MTIL datasets (0..24) to evaluate in zero-shot; excludes any selected as downstream train_dataset
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
batch_size: 64
|
| 26 |
-
increment: ${initial_increment}
|
| 27 |
-
initial_increment: 10
|
| 28 |
-
scenario: "class"
|
| 29 |
-
dataset: "cifar100"
|
| 30 |
-
|
| 31 |
-
# method: "lwf"
|
| 32 |
-
# lr: 7.1e-6
|
| 33 |
-
# ls: 0.2
|
| 34 |
-
# we:
|
| 35 |
-
# avg_freq:
|
| 36 |
-
# ref_dataset:
|
| 37 |
-
# ref_sentences:
|
| 38 |
-
|
| 39 |
-
weight_decay: 0.0
|
| 40 |
-
l2: 0
|
| 41 |
-
ce_method: 0
|
| 42 |
-
|
| 43 |
-
method: "MoE-Adapters"
|
| 44 |
-
lr: 1e-3
|
| 45 |
-
ls: 0.0
|
| 46 |
-
#we:
|
| 47 |
-
#avg_freq:
|
| 48 |
-
#ref_dataset:
|
| 49 |
-
#ref_sentences: random
|
| 50 |
-
|
| 51 |
-
# =====================
|
| 52 |
-
# DFA (two-expert + router) hyperparameters
|
| 53 |
-
# =====================
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
# Training schedule
|
| 57 |
-
epochs_a: 1
|
| 58 |
-
epochs_b: 1
|
| 59 |
-
e2_top_k: 2 # top-k for E2 router
|
| 60 |
-
|
| 61 |
-
# Stage A (InfoNCE)
|
| 62 |
-
lr_e1: 1.0e-5
|
| 63 |
-
tau_con: 0.15 # contrastive temperature for InfoNCE
|
| 64 |
-
use_moco_queue: true # enable momentum queue for more negatives
|
| 65 |
-
moco_queue_size: 128 # queue size (default 4096, increases negative samples significantly)
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
# Stage B (Task expert + Router)
|
| 71 |
-
lr_e2: 5.0e-4
|
| 72 |
-
|
| 73 |
-
text_lr_e2: 1.0e-5
|
| 74 |
-
lr_e2_router: 1.0e-3
|
| 75 |
-
lr_top_router: 1.0e-5
|
| 76 |
-
lambda_b_con: 0.001
|
| 77 |
-
tau_b_con: 0.17
|
| 78 |
-
num_task_experts: 2
|
| 79 |
-
seed: 43
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/configs/class/clevr.yaml
DELETED
|
@@ -1,79 +0,0 @@
|
|
| 1 |
-
hydra:
|
| 2 |
-
run:
|
| 3 |
-
dir: ./experiments/CLEVRCount
|
| 4 |
-
job:
|
| 5 |
-
chdir: true
|
| 6 |
-
|
| 7 |
-
job_logging:
|
| 8 |
-
version: 1
|
| 9 |
-
formatters:
|
| 10 |
-
simple:
|
| 11 |
-
format: '%(message)s'
|
| 12 |
-
|
| 13 |
-
class_order: ""
|
| 14 |
-
dataset_root: ""
|
| 15 |
-
workdir: ""
|
| 16 |
-
log_path: "metrics.json"
|
| 17 |
-
model_name: "ViT-B/16"
|
| 18 |
-
prompt_template: "a bad photo of a {}."
|
| 19 |
-
zero_shot_eval: true
|
| 20 |
-
pre_task_zero_shot_eval: False
|
| 21 |
-
zs_mtil_indices: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24] # indices of MTIL datasets (0..24) to evaluate in zero-shot; excludes any selected as downstream train_dataset
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
batch_size: 128
|
| 26 |
-
increment: ${initial_increment}
|
| 27 |
-
initial_increment: 10
|
| 28 |
-
scenario: "class"
|
| 29 |
-
dataset: "cifar100"
|
| 30 |
-
|
| 31 |
-
# method: "lwf"
|
| 32 |
-
# lr: 7.1e-6
|
| 33 |
-
# ls: 0.2
|
| 34 |
-
# we:
|
| 35 |
-
# avg_freq:
|
| 36 |
-
# ref_dataset:
|
| 37 |
-
# ref_sentences:
|
| 38 |
-
|
| 39 |
-
weight_decay: 0.0
|
| 40 |
-
l2: 0
|
| 41 |
-
ce_method: 0
|
| 42 |
-
|
| 43 |
-
method: "MoE-Adapters"
|
| 44 |
-
lr: 1e-3
|
| 45 |
-
ls: 0.0
|
| 46 |
-
#we:
|
| 47 |
-
#avg_freq:
|
| 48 |
-
#ref_dataset:
|
| 49 |
-
#ref_sentences: random
|
| 50 |
-
|
| 51 |
-
# =====================
|
| 52 |
-
# DFA (two-expert + router) hyperparameters
|
| 53 |
-
# =====================
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
# Training schedule
|
| 57 |
-
epochs_a: 1
|
| 58 |
-
epochs_b: 1
|
| 59 |
-
e2_top_k: 2 # top-k for E2 router
|
| 60 |
-
|
| 61 |
-
# Stage A (InfoNCE)
|
| 62 |
-
lr_e1: 1.0e-5
|
| 63 |
-
tau_con: 0.15 # contrastive temperature for InfoNCE
|
| 64 |
-
use_moco_queue: true # enable momentum queue for more negatives
|
| 65 |
-
moco_queue_size: 128 # queue size (default 4096, increases negative samples significantly)
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
# Stage B (Task expert + Router)
|
| 71 |
-
lr_e2: 5.0e-4
|
| 72 |
-
|
| 73 |
-
text_lr_e2: 1.0e-5
|
| 74 |
-
lr_e2_router: 1.0e-3
|
| 75 |
-
lr_top_router: 1.0e-5
|
| 76 |
-
lambda_b_con: 0.001
|
| 77 |
-
tau_b_con: 0.19
|
| 78 |
-
num_task_experts: 2
|
| 79 |
-
seed: 43
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/configs/class/conuntry.yaml
DELETED
|
@@ -1,79 +0,0 @@
|
|
| 1 |
-
hydra:
|
| 2 |
-
run:
|
| 3 |
-
dir: ./experiments/Country211
|
| 4 |
-
job:
|
| 5 |
-
chdir: true
|
| 6 |
-
|
| 7 |
-
job_logging:
|
| 8 |
-
version: 1
|
| 9 |
-
formatters:
|
| 10 |
-
simple:
|
| 11 |
-
format: '%(message)s'
|
| 12 |
-
|
| 13 |
-
class_order: ""
|
| 14 |
-
dataset_root: ""
|
| 15 |
-
workdir: ""
|
| 16 |
-
log_path: "metrics.json"
|
| 17 |
-
model_name: "ViT-B/16"
|
| 18 |
-
prompt_template: "a bad photo of a {}."
|
| 19 |
-
zero_shot_eval: true
|
| 20 |
-
pre_task_zero_shot_eval: False
|
| 21 |
-
zs_mtil_indices: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24] # indices of MTIL datasets (0..24) to evaluate in zero-shot; excludes any selected as downstream train_dataset
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
batch_size: 128
|
| 26 |
-
increment: ${initial_increment}
|
| 27 |
-
initial_increment: 10
|
| 28 |
-
scenario: "class"
|
| 29 |
-
dataset: "cifar100"
|
| 30 |
-
|
| 31 |
-
# method: "lwf"
|
| 32 |
-
# lr: 7.1e-6
|
| 33 |
-
# ls: 0.2
|
| 34 |
-
# we:
|
| 35 |
-
# avg_freq:
|
| 36 |
-
# ref_dataset:
|
| 37 |
-
# ref_sentences:
|
| 38 |
-
|
| 39 |
-
weight_decay: 0.0
|
| 40 |
-
l2: 0
|
| 41 |
-
ce_method: 0
|
| 42 |
-
|
| 43 |
-
method: "MoE-Adapters"
|
| 44 |
-
lr: 1e-3
|
| 45 |
-
ls: 0.0
|
| 46 |
-
#we:
|
| 47 |
-
#avg_freq:
|
| 48 |
-
#ref_dataset:
|
| 49 |
-
#ref_sentences: random
|
| 50 |
-
|
| 51 |
-
# =====================
|
| 52 |
-
# DFA (two-expert + router) hyperparameters
|
| 53 |
-
# =====================
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
# Training schedule
|
| 57 |
-
epochs_a: 1
|
| 58 |
-
epochs_b: 1
|
| 59 |
-
e2_top_k: 2 # top-k for E2 router
|
| 60 |
-
|
| 61 |
-
# Stage A (InfoNCE)
|
| 62 |
-
lr_e1: 1.0e-5
|
| 63 |
-
tau_con: 0.15 # contrastive temperature for InfoNCE
|
| 64 |
-
use_moco_queue: true # enable momentum queue for more negatives
|
| 65 |
-
moco_queue_size: 128 # queue size (default 4096, increases negative samples significantly)
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
# Stage B (Task expert + Router)
|
| 71 |
-
lr_e2: 5.0e-4
|
| 72 |
-
|
| 73 |
-
text_lr_e2: 1.0e-5
|
| 74 |
-
lr_e2_router: 1.0e-3
|
| 75 |
-
lr_top_router: 1.0e-5
|
| 76 |
-
lambda_b_con: 0.001
|
| 77 |
-
tau_b_con: 0.19
|
| 78 |
-
num_task_experts: 2
|
| 79 |
-
seed: 43
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/configs/class/dtd.yaml
DELETED
|
@@ -1,79 +0,0 @@
|
|
| 1 |
-
hydra:
|
| 2 |
-
run:
|
| 3 |
-
dir: ./experiments/DTD
|
| 4 |
-
job:
|
| 5 |
-
chdir: true
|
| 6 |
-
|
| 7 |
-
job_logging:
|
| 8 |
-
version: 1
|
| 9 |
-
formatters:
|
| 10 |
-
simple:
|
| 11 |
-
format: '%(message)s'
|
| 12 |
-
|
| 13 |
-
class_order: ""
|
| 14 |
-
dataset_root: ""
|
| 15 |
-
workdir: ""
|
| 16 |
-
log_path: "metrics.json"
|
| 17 |
-
model_name: "ViT-B/16"
|
| 18 |
-
prompt_template: "a bad photo of a {}."
|
| 19 |
-
zero_shot_eval: true
|
| 20 |
-
pre_task_zero_shot_eval: False
|
| 21 |
-
zs_mtil_indices: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24] # indices of MTIL datasets (0..24) to evaluate in zero-shot; excludes any selected as downstream train_dataset
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
batch_size: 128
|
| 26 |
-
increment: ${initial_increment}
|
| 27 |
-
initial_increment: 10
|
| 28 |
-
scenario: "class"
|
| 29 |
-
dataset: "cifar100"
|
| 30 |
-
|
| 31 |
-
# method: "lwf"
|
| 32 |
-
# lr: 7.1e-6
|
| 33 |
-
# ls: 0.2
|
| 34 |
-
# we:
|
| 35 |
-
# avg_freq:
|
| 36 |
-
# ref_dataset:
|
| 37 |
-
# ref_sentences:
|
| 38 |
-
|
| 39 |
-
weight_decay: 0.0
|
| 40 |
-
l2: 0
|
| 41 |
-
ce_method: 0
|
| 42 |
-
|
| 43 |
-
method: "MoE-Adapters"
|
| 44 |
-
lr: 1e-3
|
| 45 |
-
ls: 0.0
|
| 46 |
-
#we:
|
| 47 |
-
#avg_freq:
|
| 48 |
-
#ref_dataset:
|
| 49 |
-
#ref_sentences: random
|
| 50 |
-
|
| 51 |
-
# =====================
|
| 52 |
-
# DFA (two-expert + router) hyperparameters
|
| 53 |
-
# =====================
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
# Training schedule
|
| 57 |
-
epochs_a: 1
|
| 58 |
-
epochs_b: 1
|
| 59 |
-
e2_top_k: 2 # top-k for E2 router
|
| 60 |
-
|
| 61 |
-
# Stage A (InfoNCE)
|
| 62 |
-
lr_e1: 1.0e-5
|
| 63 |
-
tau_con: 0.15 # contrastive temperature for InfoNCE
|
| 64 |
-
use_moco_queue: true # enable momentum queue for more negatives
|
| 65 |
-
moco_queue_size: 128 # queue size (default 4096, increases negative samples significantly)
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
# Stage B (Task expert + Router)
|
| 71 |
-
lr_e2: 5.0e-4
|
| 72 |
-
|
| 73 |
-
text_lr_e2: 1.0e-5
|
| 74 |
-
lr_e2_router: 1.0e-3
|
| 75 |
-
lr_top_router: 1.0e-5
|
| 76 |
-
lambda_b_con: 0.001
|
| 77 |
-
tau_b_con: 0.19
|
| 78 |
-
num_task_experts: 2
|
| 79 |
-
seed: 43
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DFA-MoE/configs/class/eurosat.yaml
DELETED
|
@@ -1,79 +0,0 @@
|
|
| 1 |
-
hydra:
|
| 2 |
-
run:
|
| 3 |
-
dir: ./experiments/Eurosat
|
| 4 |
-
job:
|
| 5 |
-
chdir: true
|
| 6 |
-
|
| 7 |
-
job_logging:
|
| 8 |
-
version: 1
|
| 9 |
-
formatters:
|
| 10 |
-
simple:
|
| 11 |
-
format: '%(message)s'
|
| 12 |
-
|
| 13 |
-
class_order: ""
|
| 14 |
-
dataset_root: ""
|
| 15 |
-
workdir: ""
|
| 16 |
-
log_path: "metrics.json"
|
| 17 |
-
model_name: "ViT-B/16"
|
| 18 |
-
prompt_template: "a bad photo of a {}."
|
| 19 |
-
zero_shot_eval: true
|
| 20 |
-
pre_task_zero_shot_eval: False
|
| 21 |
-
zs_mtil_indices: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24] # indices of MTIL datasets (0..24) to evaluate in zero-shot; excludes any selected as downstream train_dataset
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
batch_size: 128
|
| 25 |
-
increment: ${initial_increment}
|
| 26 |
-
initial_increment: 10
|
| 27 |
-
scenario: "class"
|
| 28 |
-
dataset: "cifar100"
|
| 29 |
-
|
| 30 |
-
# method: "lwf"
|
| 31 |
-
# lr: 7.1e-6
|
| 32 |
-
# ls: 0.2
|
| 33 |
-
# we:
|
| 34 |
-
# avg_freq:
|
| 35 |
-
# ref_dataset:
|
| 36 |
-
# ref_sentences:
|
| 37 |
-
|
| 38 |
-
weight_decay: 0.0
|
| 39 |
-
l2: 0
|
| 40 |
-
ce_method: 0
|
| 41 |
-
|
| 42 |
-
method: "MoE-Adapters"
|
| 43 |
-
lr: 1e-3
|
| 44 |
-
ls: 0.0
|
| 45 |
-
#we:
|
| 46 |
-
#avg_freq:
|
| 47 |
-
#ref_dataset:
|
| 48 |
-
#ref_sentences: random
|
| 49 |
-
|
| 50 |
-
# =====================
|
| 51 |
-
# DFA (two-expert + router) hyperparameters
|
| 52 |
-
# =====================
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
# Training schedule
|
| 56 |
-
epochs_a: 1
|
| 57 |
-
epochs_b: 1
|
| 58 |
-
e2_top_k: 2 # top-k for E2 router
|
| 59 |
-
|
| 60 |
-
# Stage A (InfoNCE)
|
| 61 |
-
lr_e1: 1.0e-5
|
| 62 |
-
tau_con: 0.15 # contrastive temperature for InfoNCE
|
| 63 |
-
use_moco_queue: true # enable momentum queue for more negatives
|
| 64 |
-
moco_queue_size: 128 # queue size (default 4096, increases negative samples significantly)
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
# Stage B (Task expert + Router)
|
| 71 |
-
lr_e2: 5.0e-4
|
| 72 |
-
|
| 73 |
-
text_lr_e2: 1.0e-5
|
| 74 |
-
lr_e2_router: 1.0e-3
|
| 75 |
-
lr_top_router: 1.0e-5
|
| 76 |
-
lambda_b_con: 0.001
|
| 77 |
-
tau_b_con: 0.17
|
| 78 |
-
num_task_experts: 2
|
| 79 |
-
seed: 43
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|