fork and bug fix from https://github.com/AkaliKong/iLoRA

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.df filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,65 @@
+# iLoRA
+
+#### Preparation
+
+1. Prepare the environment:
+
+```python
+git clone
+cd iLoRA
+pip install -r requirements.txt
+```
+
+2. Prepare the pre-trained huggingface model of Llama2-7B (https://huggingface.co/meta-llama/Llama-2-7b-hf).
+3. Modify the paths inside the .sh file.
+
+
+#### Train iLoRA
+
+Train iLoRA with a single A100 GPU on MovieLens dataset:
+
+```python
+sh train_movielens.sh
+```
+
+Train iLoRA with a single A100 GPU on Steam dataset:
+
+```
+sh train_steam.sh
+```
+
+Train iLoRA with a single A100 GPU on LastFM dataset:
+
+```
+sh train_lastfm.sh
+```
+
+Note that: set the `llm_path` argument with your own directory path of the Llama2 model.
+
+##### For the environmental issues mentioned by everyone during the reproduction process, we have attempted to help resolve them and have listed some solutions:
+
+If you encounter an error with your transformers/generation/utils.py file, please replace it with the debug/utils.py file we have provided in your environment.
+
+If you encounter an error with your transformers/models/llama/modeling_llama.py file, please replace it with the debug/modeling_llama.py file.
+
+Thank you all for your attention to our work! Wishing you success in your research.
+
+##### Evaluate iLoRA
+
+Test iLoRA with a single A100 GPU on MovieLens dataset:
+
+```
+sh test_movielens.sh
+```
+
+Test iLoRA with a single A100 GPU on Steam dataset:
+
+```
+sh test_steam.sh
+```
+
+Test iLoRA with a single A100 GPU on LastFM dataset:
+
+```
+sh test_lastfm.sh
+```

SASRecModules_ori.py

@@ -0,0 +1,95 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+
+class PositionwiseFeedForward(nn.Module):
+    def __init__(self, d_in, d_hid, dropout=0.1):
+        super().__init__()
+        self.w_1 = nn.Conv1d(d_in, d_hid, 1)
+        self.w_2 = nn.Conv1d(d_hid, d_in, 1)
+        self.layer_norm = nn.LayerNorm(d_in)
+        self.dropout = nn.Dropout(dropout)
+
+    def forward(self, x):
+        residual = x
+        output = x.transpose(1, 2)
+        output = self.w_2(F.relu(self.w_1(output)))
+        output = output.transpose(1, 2)
+        output = self.dropout(output)
+        output = self.layer_norm(output + residual)
+        return output
+
+
+
+class MultiHeadAttention(nn.Module):
+    def __init__(self, hidden_size, num_units, num_heads, dropout_rate):
+        super().__init__()
+        self.hidden_size = hidden_size
+        self.num_heads = num_heads
+        assert hidden_size % num_heads == 0
+        
+        self.linear_q = nn.Linear(hidden_size, num_units)
+        self.linear_k = nn.Linear(hidden_size, num_units)
+        self.linear_v = nn.Linear(hidden_size, num_units)
+        self.dropout = nn.Dropout(dropout_rate)
+        self.softmax = nn.Softmax(dim=-1)
+
+
+    def forward(self, queries, keys):
+        """
+        :param queries: A 3d tensor with shape of [N, T_q, C_q]
+        :param keys: A 3d tensor with shape of [N, T_k, C_k]
+        
+        :return: A 3d tensor with shape of (N, T_q, C)
+        
+        """
+        Q = self.linear_q(queries)  # (N, T_q, C)
+        K = self.linear_k(keys)  # (N, T_k, C)
+        V = self.linear_v(keys)  # (N, T_k, C)
+        
+        # Split and Concat
+        split_size = self.hidden_size // self.num_heads
+        Q_ = torch.cat(torch.split(Q, split_size, dim=2), dim=0)  # (h*N, T_q, C/h)
+        K_ = torch.cat(torch.split(K, split_size, dim=2), dim=0)  # (h*N, T_k, C/h)
+        V_ = torch.cat(torch.split(V, split_size, dim=2), dim=0)  # (h*N, T_k, C/h)
+        
+        # Multiplication
+        matmul_output = torch.bmm(Q_, K_.transpose(1, 2)) / self.hidden_size ** 0.5  # (h*N, T_q, T_k)
+        
+        # Key Masking
+        key_mask = torch.sign(torch.abs(keys.sum(dim=-1))).repeat(self.num_heads, 1)  # (h*N, T_k)
+        key_mask_reshaped = key_mask.unsqueeze(1).repeat(1, queries.shape[1], 1)  # (h*N, T_q, T_k)
+        key_paddings = torch.ones_like(matmul_output) * (-2 ** 32 + 1)
+        matmul_output_m1 = torch.where(torch.eq(key_mask_reshaped, 0), key_paddings, matmul_output)  # (h*N, T_q, T_k)
+        
+        # Causality - Future Blinding
+        diag_vals = torch.ones_like(matmul_output[0, :, :])   # (T_q, T_k)
+        tril = torch.tril(diag_vals)  # (T_q, T_k)
+        causality_mask = tril.unsqueeze(0).repeat(matmul_output.shape[0], 1, 1)  # (h*N, T_q, T_k)
+        causality_paddings = torch.ones_like(causality_mask) * (-2 ** 32 + 1)
+        matmul_output_m2 = torch.where(torch.eq(causality_mask, 0), causality_paddings, matmul_output_m1)  # (h*N, T_q, T_k)
+        
+        # Activation
+        matmul_output_sm = self.softmax(matmul_output_m2)  # (h*N, T_q, T_k)
+        
+        # Query Masking
+        query_mask = torch.sign(torch.abs(queries.sum(dim=-1))).repeat(self.num_heads, 1)  # (h*N, T_q)
+        query_mask = query_mask.unsqueeze(-1).repeat(1, 1, keys.shape[1])  # (h*N, T_q, T_k)
+        matmul_output_qm = matmul_output_sm * query_mask
+        
+        # Dropout
+        matmul_output_dropout = self.dropout(matmul_output_qm)
+        
+        # Weighted Sum
+        output_ws = torch.bmm(matmul_output_dropout, V_)  # ( h*N, T_q, C/h)
+        
+        # Restore Shape
+        output = torch.cat(torch.split(output_ws, output_ws.shape[0] // self.num_heads, dim=0), dim=2)  # (N, T_q, C)
+        
+        # Residual Connection
+        output_res = output + queries
+        
+        return output_res
+        

data/_init_.py

@@ -0,0 +1 @@
+

data/data_interface.py

@@ -0,0 +1,190 @@
+import inspect
+import importlib
+import pickle as pkl
+import pytorch_lightning as pl
+from torch.utils.data import DataLoader
+from torch.utils.data.sampler import WeightedRandomSampler
+
+import random
+import torch
+import argparse
+from transformers import LlamaForCausalLM, LlamaTokenizer
+import os
+
+
+
+class TrainCollater:
+    def __init__(self,
+                 prompt_list=None,
+                 llm_tokenizer=None,
+                 train=False,
+                 terminator="\n",
+                 max_step=1):
+        self.prompt_list = prompt_list
+        self.llm_tokenizer = llm_tokenizer
+        self.train=train
+        self.terminator = terminator
+        self.max_step = max_step
+        self.cur_step = 1
+
+    def __call__(self, batch):
+        if isinstance(self.prompt_list,list):
+            instruction = random.choice(self.prompt_list)
+            inputs_text = instruction if isinstance(instruction, list) else [instruction] * len(batch)
+        else:
+            instruction = sample["instruction_input"] if "instruction_input" in sample else None
+            inputs_text = instruction if isinstance(instruction, list) else [instruction] * len(batch)
+ 
+        thresh_hold = self.cur_step/self.max_step
+        p = random.random()
+        if p < thresh_hold or not self.train:
+            for i, sample in enumerate(batch):
+                input_text=inputs_text[i]
+                if '[HistoryHere]' in input_text:
+                    insert_prompt=", ".join([seq_title+' [HistoryEmb]' for seq_title in sample['seq_name']])
+                    input_text=input_text.replace('[HistoryHere]',insert_prompt)
+                if '[CansHere]' in input_text:
+                    insert_prompt=", ".join([can_title+' [CansEmb]' for can_title in sample['cans_name']])
+                    input_text=input_text.replace('[CansHere]',insert_prompt)   
+                if '[TargetHere]' in input_text:
+                    insert_prompt=insert_prompt=", ".join([sample['correct_answer']+' [ItemEmb]'])
+                    input_text=input_text.replace('[TargetHere]',insert_prompt)  
+                
+                inputs_text[i]=input_text
+            flag = False
+        else:
+            for i, sample in enumerate(batch):
+                input_text=inputs_text[i]
+                if '[HistoryHere]' in input_text:
+                    insert_prompt=", ".join([seq_title+' [PH]' for seq_title in sample['seq_name']])
+                    input_text=input_text.replace('[HistoryHere]',insert_prompt)
+                if '[CansHere]' in input_text:
+                    insert_prompt=", ".join([can_title+' [PH]' for can_title in sample['cans_name']])
+                    input_text=input_text.replace('[CansHere]',insert_prompt)   
+                
+                inputs_text[i]=input_text
+            flag = True
+        self.cur_step += 1
+        
+        targets_text = [sample['correct_answer'] for sample in batch]
+
+        if self.train:
+            targets_text=[target_text+self.terminator for target_text in targets_text]
+            inputs_pair = [[p, t] for p, t in zip(inputs_text, targets_text)]
+
+            batch_tokens = self.llm_tokenizer(
+                inputs_pair,
+                return_tensors="pt",
+                padding="longest",
+                truncation=False,
+                add_special_tokens=True,
+                return_attention_mask=True,
+                return_token_type_ids=True)
+            new_batch={"tokens":batch_tokens,
+                       "seq":torch.stack([torch.tensor(sample['seq']) for sample in batch], dim=0),
+                       "cans":torch.stack([torch.tensor(sample['cans']) for sample in batch], dim=0),
+                       "len_seq":torch.stack([torch.tensor(sample['len_seq']) for sample in batch], dim=0),
+                       "len_cans":torch.stack([torch.tensor(sample['len_cans']) for sample in batch], dim=0),
+                       "item_id": torch.stack([torch.tensor(sample['item_id']) for sample in batch], dim=0),
+                       "flag":flag,
+                       }
+            
+            
+        else:
+            batch_tokens = self.llm_tokenizer(
+                inputs_text,
+                return_tensors="pt",
+                padding="longest",
+                truncation=False,
+                add_special_tokens=True,
+                return_token_type_ids=True)
+            cans_name=[sample['cans_name'] for sample in batch]
+            new_batch={"tokens":batch_tokens,
+                       "seq":torch.stack([torch.tensor(sample['seq']) for sample in batch], dim=0),
+                       "cans":torch.stack([torch.tensor(sample['cans']) for sample in batch], dim=0),
+                       "len_seq":torch.stack([torch.tensor(sample['len_seq']) for sample in batch], dim=0),
+                       "len_cans":torch.stack([torch.tensor(sample['len_cans']) for sample in batch], dim=0),
+                       "item_id": torch.stack([torch.tensor(sample['item_id']) for sample in batch], dim=0),
+                       "correct_answer": targets_text,
+                       "cans_name": cans_name,
+            
+                       }
+            
+        return new_batch
+
+class DInterface(pl.LightningDataModule):
+
+    def __init__(self, 
+                 llm_tokenizer=None,
+                 num_workers=8,
+                 dataset='',
+                 **kwargs):
+        super().__init__()
+        self.num_workers = num_workers
+        self.llm_tokenizer=llm_tokenizer
+        self.dataset = dataset
+        self.kwargs = kwargs
+        self.batch_size = kwargs['batch_size']
+        self.max_epochs = kwargs['max_epochs']
+        self.load_data_module()
+        self.load_prompt(kwargs['prompt_path'])
+
+        self.trainset = self.instancialize(stage='train')
+        self.valset = self.instancialize(stage='val')
+        self.testset = self.instancialize(stage='test')
+        self.max_steps = self.max_epochs*(len(self.trainset)//self.batch_size)//self.num_workers
+
+    def train_dataloader(self):
+        return DataLoader(self.trainset,
+                          batch_size=self.batch_size, 
+                          num_workers=self.num_workers, 
+                          shuffle=True,
+                          drop_last=True,
+                          collate_fn=TrainCollater(prompt_list=self.prompt_list,llm_tokenizer=self.llm_tokenizer,train=True, max_step=self.max_steps))
+
+    def val_dataloader(self):
+        return DataLoader(self.valset, 
+                          batch_size=self.batch_size, 
+                          num_workers=self.num_workers, 
+                          shuffle=False,
+                          collate_fn=TrainCollater(prompt_list=self.prompt_list,llm_tokenizer=self.llm_tokenizer,train=False))
+
+    def test_dataloader(self):
+        return DataLoader(self.testset, 
+                          batch_size=self.batch_size, 
+                          num_workers=self.num_workers, 
+                          shuffle=False,
+                          
+                          drop_last=True,
+                          
+                          collate_fn=TrainCollater(prompt_list=self.prompt_list,llm_tokenizer=self.llm_tokenizer,train=False))
+
+    def load_data_module(self):
+        name = self.dataset
+        camel_name = ''.join([i.capitalize() for i in name.split('_')])
+        try:
+            self.data_module = getattr(importlib.import_module(
+                '.'+name, package=__package__), camel_name)
+        except:
+            raise ValueError(
+                f'Invalid Dataset File Name or Invalid Class Name data.{name}.{camel_name}')
+
+    def instancialize(self, **other_args):
+        class_args = inspect.getargspec(self.data_module.__init__).args[1:]
+        inkeys = self.kwargs.keys()
+        args1 = {}
+        for arg in class_args:
+            if arg in inkeys:
+                args1[arg] = self.kwargs[arg]
+        args1.update(other_args)
+        return self.data_module(**args1)
+    
+    def load_prompt(self,prompt_path):
+        if os.path.isfile(prompt_path):
+            with open(prompt_path, 'r') as f:
+                raw_prompts = f.read().splitlines()
+            self.prompt_list = [p.strip() for p in raw_prompts]
+            print('Load {} training prompts'.format(len(self.prompt_list)))
+            print('Prompt Example \n{}'.format(random.choice(self.prompt_list)))
+        else:
+            self.prompt_list = []

data/lastfm_data.py

@@ -0,0 +1,87 @@
+import torch
+import os.path as op
+import numpy as np
+import pickle as pkl
+import torch.utils.data as data
+
+import pandas as pd
+import random
+
+class LastfmData(data.Dataset):
+    def __init__(self, data_dir=r'data/ref/lastfm_ctr',
+                 stage=None,
+                 cans_num=10,
+                 sep=", ",
+                 no_augment=True):
+        self.__dict__.update(locals())
+        self.aug = (stage=='train') and not no_augment
+        self.padding_item_id=4606
+        self.check_files()
+
+    def __len__(self):
+        return len(self.session_data['seq'])
+
+    def __getitem__(self, i):
+        temp = self.session_data.iloc[i]
+        candidates = self.negative_sampling(temp['seq_unpad'],temp['next'])
+        cans_name=[self.item_id2name[can] for can in candidates]
+        sample = {
+            'seq': temp['seq'],
+            'seq_name': temp['seq_title'],
+            'len_seq': temp['len_seq'],
+            'seq_str': self.sep.join(temp['seq_title']),
+            'cans': candidates,
+            'cans_name': cans_name,
+            'cans_str': self.sep.join(cans_name),
+            'len_cans': self.cans_num,
+            'item_id': temp['next'],
+            'item_name': temp['next_item_name'],
+            'correct_answer': temp['next_item_name']
+        }
+        return sample
+
+    def negative_sampling(self,seq_unpad,next_item):
+        canset=[i for i in list(self.item_id2name.keys()) if i not in seq_unpad and i!=next_item]
+        candidates=random.sample(canset, 1)
+        return candidates
+
+    def check_files(self):
+        self.item_id2name=self.get_music_id2name()
+        if self.stage=='train':
+            filename="train_data.df"
+        elif self.stage=='val':
+            filename="Val_data.df"
+        elif self.stage=='test':
+            filename="Test_data.df"
+        data_path=op.join(self.data_dir, filename)
+        self.session_data = self.session_data4frame(data_path, self.item_id2name)  
+
+
+    def get_music_id2name(self):
+        music_id2name = dict()
+        item_path=op.join(self.data_dir, 'id2name.txt')
+        with open(item_path, 'r') as f:
+            for l in f.readlines():
+                ll = l.strip('\n').split('::')
+                music_id2name[int(ll[0])] = ll[1].strip()
+        return music_id2name
+
+    def session_data4frame(self, datapath, music_id2name):
+        train_data = pd.read_pickle(datapath)
+        train_data = train_data[train_data['len_seq'] >= 3]
+        def remove_padding(xx):
+            x = xx[:]
+            for i in range(10):
+                try:
+                    x.remove(self.padding_item_id)
+                except:
+                    break
+            return x
+        train_data['seq_unpad'] = train_data['seq'].apply(remove_padding)
+        def seq_to_title(x): 
+            return [music_id2name[x_i] for x_i in x]
+        train_data['seq_title'] = train_data['seq_unpad'].apply(seq_to_title)
+        def next_item_title(x): 
+            return music_id2name[x]
+        train_data['next_item_name'] = train_data['next'].apply(next_item_title)
+        return train_data

data/movielens_data.py

@@ -0,0 +1,102 @@
+import torch
+import os.path as op
+import numpy as np
+import pickle as pkl
+import torch.utils.data as data
+
+import pandas as pd
+import random
+
+class MovielensData(data.Dataset):
+    def __init__(self, data_dir=r'data/ref/movielens',
+                 stage=None,
+                 cans_num=10,
+                 sep=", ",
+                 no_augment=True):
+        self.__dict__.update(locals())
+        self.aug = (stage=='train') and not no_augment
+        self.padding_item_id=1682
+        self.padding_rating=0
+        self.check_files()
+
+    def __len__(self):
+        return len(self.session_data['seq'])
+
+    def __getitem__(self, i):
+        temp = self.session_data.iloc[i]
+        candidates = self.negative_sampling(temp['seq_unpad'],temp['next'])
+        cans_name=[self.item_id2name[can] for can in candidates]
+        sample = {
+            'seq': temp['seq'],
+            'seq_name': temp['seq_title'],
+            'len_seq': temp['len_seq'],
+            'seq_str': self.sep.join(temp['seq_title']),
+            'cans': candidates,
+            'cans_name': cans_name,
+            'cans_str': self.sep.join(cans_name),
+            'len_cans': self.cans_num,
+            'item_id': temp['next'],
+            'item_name': temp['next_item_name'],
+            'correct_answer': temp['next_item_name']
+        }
+        return sample
+    
+    def negative_sampling(self,seq_unpad,next_item):
+        canset=[i for i in list(self.item_id2name.keys()) if i not in seq_unpad and i!=next_item]
+        candidates=random.sample(canset, self.cans_num-1)+[next_item]
+        random.shuffle(candidates)
+        return candidates  
+
+    def check_files(self):
+        self.item_id2name=self.get_movie_id2name()
+        if self.stage=='train':
+            filename="train_data.df"
+        elif self.stage=='val':
+            filename="Val_data.df"
+        elif self.stage=='test':
+            filename="Test_data.df"
+        data_path=op.join(self.data_dir, filename)
+        self.session_data = self.session_data4frame(data_path, self.item_id2name)  
+    
+    def get_mv_title(self,s):
+        sub_list=[", The", ", A", ", An"]
+        for sub_s in sub_list:
+            if sub_s in s:
+                return sub_s[2:]+" "+s.replace(sub_s,"")
+        return s
+
+    def get_movie_id2name(self):
+        movie_id2name = dict()
+        item_path=op.join(self.data_dir, 'u.item')
+        with open(item_path, 'r', encoding = "ISO-8859-1") as f:
+            for l in f.readlines():
+                ll = l.strip('\n').split('|')
+                movie_id2name[int(ll[0]) - 1] = self.get_mv_title(ll[1][:-7])
+        return movie_id2name
+    
+    def session_data4frame(self, datapath, movie_id2name):
+        train_data = pd.read_pickle(datapath)
+        train_data = train_data[train_data['len_seq'] >= 3]
+        def remove_padding(xx):
+            x = xx[:]
+            for i in range(10):
+                try:
+                    x.remove((self.padding_item_id,self.padding_rating))
+                except:
+                    break
+            return x
+        train_data['seq_unpad'] = train_data['seq'].apply(remove_padding)
+        def seq_to_title(x): 
+            return [movie_id2name[x_i[0]] for x_i in x]
+        train_data['seq_title'] = train_data['seq_unpad'].apply(seq_to_title)
+        def next_item_title(x): 
+            return movie_id2name[x[0]]
+        train_data['next_item_name'] = train_data['next'].apply(next_item_title)
+        def get_id_from_tumple(x):
+            return x[0]
+        def get_id_from_list(x):
+            return [i[0] for i in x]
+        train_data['next'] = train_data['next'].apply(get_id_from_tumple)
+        train_data['seq'] = train_data['seq'].apply(get_id_from_list)
+        train_data['seq_unpad']=train_data['seq_unpad'].apply(get_id_from_list)
+        return train_data

data/ref/lastfm/Test_data.df

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cac058a2842b2a3ed4d39ed1fa0a742f5034a23c9851a2ffc157b4845d703570
+size 6673

data/ref/lastfm/Val_data.df

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92a1ddb65ebdeff999f910b65660eb3d8c7cd85e6a19308d80c429481839f8b3
+size 6690

data/ref/lastfm/id2name.txt

@@ -0,0 +1,4606 @@
+0::Morcheeba
+1::Enigma
+2::Café Del Mar
+3::Fleetwood Mac
+4::China Crisis
+5::Loscil
+6::Chicane
+7::Sigue Sigue Sputnik
+8::Duran Duran
+9::Air
+10::Röyksopp
+11::Moby
+12::Depeche Mode
+13::Groove Armada
+14::INXS
+15::Deep Forest
+16::Porcupine Tree
+17::De/Vision
+18::Radiohead
+19::VAST
+20::Michael Jackson
+21::God Is an Astronaut
+22::Pink Floyd
+23::Planet Funk
+24::Scissor Sisters
+25::Mew
+26::Stereophonics
+27::Placebo
+28::Infected Mushroom
+29::Delerium
+30::Roxette
+31::Paradise Lost
+32::Jamiroquai
+33::James Blunt
+34::Reamonn
+35::Opeth
+36::Elton John
+37::Tears for Fears
+38::Death Cab for Cutie
+39::Savage Garden
+40::Darren Hayes
+41::Explosions in the Sky
+42::AC/DC
+43::Apocalyptica
+44::Megadeth
+45::Dream Theater
+46::Tiësto
+47::The Darkness
+48::VNV Nation
+49::Thirteen Senses
+50::Dave Gahan
+51::Apparat
+52::Def Leppard
+53::Zero 7
+54::Dope
+55::Audioslave
+56::Charon
+57::Amorphis
+58::Pain
+59::Sentenced
+60::Blank & Jones
+61::Mesh
+62::Keane
+63::Muse
+64::Bright Eyes
+65::Tom Waits
+66::Gogol Bordello
+67::CAKE
+68::Jeff Buckley
+69::Sparklehorse
+70::Modest Mouse
+71::Sunset Rubdown
+72::Nine Inch Nails
+73::Clap Your Hands Say Yeah
+74::Manu Chao
+75::Belle and Sebastian
+76::John Lennon
+77::Primal Scream
+78::The Velvet Underground
+79::Bon Jovi
+80::Johann Sebastian Bach
+81::Weezer
+82::Imogen Heap
+83::Janis Joplin
+84::Oh No Oh My
+85::The Cramps
+86::Panda Bear
+87::Gavin DeGraw
+88::Wilco
+89::Ben Kweller
+90::Everlast
+91::Menomena
+92::Turin Brakes
+93::DIR EN GREY
+94::My Chemical Romance
+95::Travis
+96::The Killers
+97::Tina Turner
+98::Katatonia
+99::Skillet
+100::宇多田ヒカル
+101::浜崎あゆみ
+102::UVERworld
+103::Linkin Park
+104::中島美嘉
+105::GACKT
+106::L'Arc~en~Ciel
+107::雅-MIYAVI-
+108::The All-American Rejects
+109::The Rasmus
+110::HYDE
+111::Simple Plan
+112::30 Seconds to Mars
+113::Three Days Grace
+114::Seether
+115::Lostprophets
+116::HIM
+117::Korn
+118::Within Temptation
+119::Shakira
+120::Epica
+121::Nickelback
+122::Noir Désir
+123::Tool
+124::Hawthorne Heights
+125::Snow Patrol
+126::Jimmy Eat World
+127::Ricky Martin
+128::Ryan Adams
+129::Shayne Ward
+130::Sting
+131::Sex Pistols
+132::Hoobastank
+133::They Might Be Giants
+134::Rie fu
+135::ギルガメッシュ
+136::ムック
+137::Il Divo
+138::Josh Groban
+139::倖田來未
+140::Type O Negative
+141::Lonestar
+142::KT Tunstall
+143::Vanessa Carlton
+144::Hinder
+145::Phil Collins
+146::Melt-Banana
+147::Goo Goo Dolls
+148::Krypteria
+149::Paula DeAnda
+150::Saliva
+151::Seether (Feat. Amy Lee)
+152::Puddle of Mudd
+153::Staind
+154::Royal Hunt
+155::Dream Evil
+156::Olivia
+157::Juno Reactor
+158::Polysics
+159::O-Zone
+160::Mark Lanegan
+161::12012
+162::Sadie
+163::Sami Yusuf
+164::Massari
+165::Saybia
+166::Beirut
+167::Arcade Fire
+168::Babyshambles
+169::The Decemberists
+170::The National
+171::Sigur Rós
+172::The Libertines
+173::Kate Nash
+174::Wolf Parade
+175::Animal Collective
+176::The Fray
+177::The Shins
+178::Lady Gaga
+179::Green Day
+180::Nirvana
+181::Mariah Carey
+182::JoJo
+183::Backstreet Boys
+184::Jesse McCartney
+185::Justin Bieber
+186::Usher
+187::Nick Carter
+188::Metro Station
+189::Jeremih
+190::Fall Out Boy
+191::Kristinia DeBarge
+192::Paris Hilton
+193::50 Cent
+194::New Kids on the Block
+195::Дима Билан
+196::London After Midnight
+197::Psyclon Nine
+198::The Crüxshadows
+199::KMFDM
+200::Mindless Self Indulgence
+201::Daft Punk
+202::Goldfrapp
+203::Madonna
+204::Dido
+205::Gorillaz
+206::The Cure
+207::Poets of the Fall
+208::OneRepublic
+209::System of a Down
+210::The Beatles
+211::Thom Yorke
+212::Massive Attack
+213::2Pac
+214::Rihanna
+215::Britney Spears
+216::Jennifer Lopez
+217::Katy Perry
+218::P!nk
+219::Black Eyed Peas
+220::Hilary Duff
+221::Fergie
+222::Ashley Tisdale
+223::Chris Brown
+224::Kanye West
+225::Avril Lavigne
+226::Taylor Swift
+227::The Pussycat Dolls
+228::Evanescence
+229::Akira Yamaoka
+230::blink-182
+231::Bat for Lashes
+232::The Smashing Pumpkins
+233::3OH!3
+234::Ke$ha
+235::Craig David
+236::Eminem
+237::Ne-Yo
+238::The Birthday Massacre
+239::Limp Bizkit
+240::Fort Minor
+241::Funeral for a Friend
+242::Finch
+243::Cold
+244::Papa Roach
+245::Paramore
+246::Flyleaf
+247::Bullet for My Valentine
+248::Angels & Airwaves
+249::LaFee
+250::In Flames
+251::Anathema
+252::Saosin
+253::Tracktor Bowling
+254::Pitchshifter
+255::P.O.D.
+256::U2
+257::Deftones
+258::Ill Niño
+259::Clan of Xymox
+260::Celldweller
+261::Static-X
+262::Fluke
+263::happysad
+264::Lacrimosa
+265::Lindsay Lohan
+266::Good Charlotte
+267::The Used
+268::Maroon 5
+269::Adam Lambert
+270::Kid Cudi
+271::Owl City
+272::The Pretty Reckless
+273::Metallica
+274::#####
+275::Fear Factory
+276::Rise Against
+277::Killswitch Engage
+278::Jane's Addiction
+279::Tiamat
+280::Nightwish
+281::Sixpence None the Richer
+282::t.A.T.u.
+283::Disturbed
+284::Jem
+285::Kerli
+286::Bruno Mars
+287::Slipknot
+288::Myslovitz
+289::Björk
+290::Rammstein
+291::A Perfect Circle
+292::Joe Satriani
+293::Stone Sour
+294::Editors
+295::Billy Talent
+296::Silverstein
+297::The Red Jumpsuit Apparatus
+298::Alesana
+299::Senses Fail
+300::Story of the Year
+301::Tactical Sekt
+302::Sepultura
+303::The Offspring
+304::Akon
+305::B.o.B
+306::Black Rebel Motorcycle Club
+307::Lykke Li
+308::Jay-Z
+309::Black Veil Brides
+310::Tokio Hotel
+311::Oomph!
+312::Guano Apes
+313::Red
+314::Breaking Benjamin
+315::Thousand Foot Krutch
+316::Jay-Z and Linkin Park
+317::Hollywood Undead
+318::Cypress Hill
+319::Lacuna Coil
+320::September
+321::Gregorian
+322::OK Go
+323::Fightstar
+324::Godsmack
+325::Keith Urban
+326::Rascal Flatts
+327::Brooks & Dunn
+328::Scooter
+329::Thursday
+330::Bloodhound Gang
+331::Robert Miles
+332::Automatic Loveletter
+333::Underoath
+334::Lupe Fiasco
+335::Dead by April
+336::Adema
+337::Dead by Sunrise
+338::Faktion
+339::We Are the Fallen
+340::Coma
+341::Drowning Pool
+342::Drake
+343::Sky Ferreira
+344::The Magic Numbers
+345::Mary Elizabeth McGlynn
+346::Brand New
+347::Will Smith
+348::Daughtry
+349::12 Stones
+350::Ferry Corsten
+351::Silversun Pickups
+352::Scary Kids Scaring Kids
+353::Xandria
+354::Kill Hannah
+355::Taking Back Sunday
+356::Hayden Panettiere
+357::Линда
+358::Smile Empty Soul
+359::Jason Derülo
+360::O.N.A.
+361::Chylińska
+362::Alexandre Desplat
+363::Emilie Autumn
+364::Fuel
+365::Stateless
+366::Robert Pattinson
+367::Trading Yesterday
+368::Armor for Sleep
+369::Bear McCreary
+370::I Am Ghost
+371::Xzibit
+372::Helios
+373::Emery
+374::Alexi Murdoch
+375::Anya Marina
+376::Mudvayne
+377::Chingy
+378::Plumb
+379::Taproot
+380::Jason Walker
+381::Mark Ronson
+382::Gavin Rossdale
+383::Blindside
+384::Amy Lee
+385::James Newton Howard
+386::Band of Skulls
+387::Sea Wolf
+388::The Letter Black
+389::Ja Rule
+390::Holly Brook
+391::Hurricane Bells
+392::Kylie Minogue
+393::Ashlee Simpson
+394::Lily Allen
+395::Heidi Montag
+396::New Order
+397::Japan
+398::The Smiths
+399::Joy Division
+400::The Sonics
+401::Southern Culture on the Skids
+402::Small Faces
+403::The Detroit Cobras
+404::The Pretty Things
+405::Talking Heads
+406::The Stranglers
+407::The Clash
+408::Roxy Music
+409::Alphaville
+410::Blondie
+411::Frank Sinatra
+412::The Human League
+413::Information Society
+414::Pet Shop Boys
+415::Ultravox
+416::Yazoo
+417::Heaven 17
+418::Siouxsie and the Banshees
+419::Misfits
+420::Dean Martin
+421::Dick Dale
+422::Motörhead
+423::Devo
+424::Split Enz
+425::The Cars
+426::Dead or Alive
+427::Oingo Boingo
+428::Eurythmics
+429::Visage
+430::Magazine
+431::Madness
+432::Ska-P
+433::The Specials
+434::Culture Club
+435::Elvis Costello
+436::Sublime
+437::A Flock of Seagulls
+438::Echo & The Bunnymen
+439::Cab Calloway
+440::The B-52's
+441::Butthole Surfers
+442::Nekromantix
+443::The Byrds
+444::The Mighty Mighty Bosstones
+445::Peggy Lee
+446::Corona
+447::The Jam
+448::Joe Jackson
+449::Messer Chups
+450::The Meteors
+451::Adam and the Ants
+452::The Zombies
+453::Barón Rojo
+454::Missing Persons
+455::Squeeze
+456::Akurat
+457::The Brian Setzer Orchestra
+458::Stray Cats
+459::Goldfinger
+460::Mecano
+461::Men at Work
+462::Squirrel Nut Zippers
+463::Harry Connick, Jr.
+464::Dick Dale and His Del-Tones
+465::The Kings of Nuthin'
+466::B-Movie
+467::Anne Clark
+468::Mad Sin
+469::Calabrese
+470::The Seeds
+471::The Litter
+472::Big Bad Voodoo Daddy
+473::13th Floor Elevators
+474::Holly Golightly
+475::IRA
+476::The Troggs
+477::Duffy
+478::Allison Iraheta
+479::Lena
+480::Jónsi
+481::Kalomoira
+482::Άννα Βισση
+483::Inna
+484::Jewel
+485::Fresno
+486::Hurts
+487::Miranda Cosgrove
+488::Wanessa
+489::The Script
+490::Alexander Rybak
+491::Cinema Bizarre
+492::Edyta Górniak
+493::ZAZ
+494::Cary Brothers
+495::Karl Wolf
+496::Boris
+497::Fever Ray
+498::Fennesz
+499::I Set My Friends On Fire
+500::Blue
+501::Jessica Simpson
+502::RBD
+503::Mandy Moore
+504::Paulina Rubio
+505::Fey
+506::Dannii Minogue
+507::Thalía
+508::A*Teens
+509::Geri Halliwell
+510::Atomic Kitten
+511::Andrés Calamaro
+512::Laibach
+513::David Bowie
+514::Einstürzende Neubauten
+515::Jesu
+516::Soundgarden
+517::Stone Temple Pilots
+518::Killing Joke
+519::Bauhaus
+520::Nitzer Ebb
+521::Ladytron
+522::Suede
+523::My Bloody Valentine
+524::Fugazi
+525::Godflesh
+526::Prong
+527::Filter
+528::Lard
+529::Noisettes
+530::VV Brown
+531::Sonique
+532::Robbie Williams
+533::Christina Aguilera
+534::Leona Lewis
+535::Beyoncé
+536::Justin Timberlake
+537::Monrose
+538::Cheryl Cole
+539::Sarah Connor
+540::M. Pokora
+541::Taio Cruz
+542::Girls Aloud
+543::Rachel Stevens
+544::Take That
+545::Iron Maiden
+546::Savatage
+547::Bruce Dickinson
+548::Children of Bodom
+549::Andre Matos
+550::Viper
+551::Matanza
+552::Nevermore
+553::Enya
+554::The Ting Tings
+555::Pearl Jam
+556::Isis
+557::Tori Amos
+558::The Mars Volta
+559::Panic! At the Disco
+560::The Fall of Troy
+561::Dżem
+562::Hard-Fi
+563::Bond
+564::Sophie Ellis-Bextor
+565::Rick Astley
+566::Solar Stone
+567::Ian Van Dahl
+568::Guru Josh Project
+569::Morrissey
+570::Frank Zappa
+571::Plaid
+572::Underworld
+573::Thomas Newman
+574::Junkie XL
+575::Sasha
+576::Glee Cast
+577::All Time Low
+578::Runner Runner
+579::And One
+580::Crystal Castles
+581::Anything Box
+582::Recoil
+583::Hubert Kah
+584::Sally Shapiro
+585::Ayria
+586::Red Flag
+587::The Frozen Autumn
+588::Cause & Effect
+589::Wham!
+590::Billy Idol
+591::Provision
+592::Peter Murphy
+593::Kaizers Orchestra
+594::Kent
+595::Iggy Pop
+596::Yann Tiersen
+597::Rush
+598::Frédéric Chopin
+599::Incubus
+600::Claude Debussy
+601::Faith No More
+602::Wolfgang Amadeus Mozart
+603::John Petrucci & Jordan Rudess
+604::Ludwig van Beethoven
+605::The String Quartet
+606::Yes
+607::Mike Oldfield
+608::Zu
+609::Temple of the Dog
+610::Bob Dylan
+611::Miles Davis
+612::John Coltrane
+613::Thelonious Monk
+614::Florence + the Machine
+615::The Cinematic Orchestra
+616::Yeasayer
+617::Tera Melos
+618::Bill Evans
+619::Animals as Leaders
+620::Arcturus
+621::Clutch
+622::Chet Baker
+623::Madeleine Peyroux
+624::Charlie Parker
+625::Charles Mingus
+626::Herbie Hancock
+627::Pat Metheny
+628::Bring Me The Horizon
+629::Escape The Fate
+630::MALICE MIZER
+631::Marilyn Manson
+632::Coldplay
+633::Nas
+634::Mos Def
+635::Gang Starr
+636::The Roots
+637::A Tribe Called Quest
+638::Sufjan Stevens
+639::Saul Williams
+640::Sunn O)))
+641::Vampire Weekend
+642::Sonic Youth
+643::Masta Ace
+644::De La Soul
+645::Jedi Mind Tricks
+646::ASIAN KUNG-FU GENERATION
+647::MF DOOM
+648::N.W.A
+649::Mike Patton
+650::Mr. Bungle
+651::Public Enemy
+652::Kool G Rap
+653::Talib Kweli
+654::Wu-Tang Clan
+655::Black Kids
+656::Merzbow
+657::Why?
+658::Method Man
+659::Drudkh
+660::Bubba Sparxxx
+661::Del tha Funkee Homosapien
+662::Wolves in the Throne Room
+663::Orgy
+664::Foo Fighters
+665::Combichrist
+666::Grendel
+667::Agonoize
+668::Hocico
+669::Feindflug
+670::Amduscia
+671::Dawn of Ashes
+672::She Wants Revenge
+673::Headscan
+674::Rotersand
+675::Blutengel
+676::Wynardtage
+677::Unheilig
+678::[:SITD:]
+679::L'Âme Immortelle
+680::Terminal Choice
+681::Suicidal Romance
+682::Liquid Divine
+683::Melotron
+684::Evil's Toy
+685::Vibrasphere
+686::Bamboo Forest
+687::Miss Construction
+688::Samsas Traum
+689::Zombie Girl
+690::Nurzery [Rhymes]
+691::Nachtmahr
+692::ASP
+693::Subway to Sally
+694::Caustic
+695::Diary of Dreams
+696::xotox
+697::Tamtrum
+698::Covenant
+699::Icon of Coil
+700::Funker Vogt
+701::God Module
+702::Noisuf-X
+703::Reaper
+704::Portishead
+705::David Guetta
+706::Cascada
+707::Paul van Dyk
+708::ATB
+709::Skeletal Family
+710::Dragonette
+711::Armin van Buuren
+712::Yelle
+713::Kyau vs. Albert
+714::Novaspace
+715::OceanLab
+716::Groove Coverage
+717::Fragma
+718::Sylver
+719::Markus Schulz
+720::IAMX
+721::Hot Chip
+722::Cryo
+723::Front Line Assembly
+724::Haujobb
+725::mind.in.a.box
+726::Absurd Minds
+727::Seabound
+728::Colony 5
+729::Apoptygma Berzerk
+730::Suicide Commando
+731::Neuroticfish
+732::Assemblage 23
+733::Edge of Dawn
+734::Cosmic Gate
+735::Above & Beyond
+736::Pride and Fall
+737::E-Craft
+738::Code 64
+739::Angels & Agony
+740::4 Strings
+741::XP8
+742::Bruderschaft
+743::Syrian
+744::Digitalism
+745::Justice
+746::Armand van Helden
+747::Simian Mobile Disco
+748::MSTRKRFT
+749::The Sisters of Mercy
+750::Holly Valance
+751::The Knife
+752::Porcelain and the Tramps
+753::Dope Stars Inc.
+754::Ashbury Heights
+755::Vive la Fête
+756::Diorama
+757::Belanova
+758::Alice in Videoland
+759::Theatre of Tragedy
+760::ADULT.
+761::Annie
+762::Front 242
+763::Robots in Disguise
+764::Alien Sex Fiend
+765::Christian Death
+766::Helalyn Flowers
+767::Shiny Toy Guns
+768::Claire Voyant
+769::Hungry Lucy
+770::Minerve
+771::Client
+772::Frozen Plasma
+773::Sohodolls
+774::Tiga
+775::Vitalic
+776::Fischerspooner
+777::Sebastian
+778::Project Pitchfork
+779::Deine Lakaien
+780::Panzer AG
+781::Imperative Reaction
+782::In Strict Confidence
+783::Decoded Feedback
+784::Le Tigre
+785::Paul Oakenfold
+786::Cinema Strange
+787::Uffie
+788::Aesthetic Perfection
+789::Safri Duo
+790::Peaches
+791::Blaqk Audio
+792::Alien Vampires
+793::Heimataerde
+794::David Vendetta
+795::Rank 1
+796::Faderhead
+797::Destroid
+798::Blind Passengers
+799::Beborn Beton
+800::Faith and the Muse
+801::Miss Kittin
+802::Chicks on Speed
+803::BT
+804::Darude
+805::Switchblade Symphony
+806::Lasgo
+807::Kittie
+808::Michigan
+809::Felix da Housecat
+810::Pulsedriver
+811::Lo-Fi-Fnk
+812::Welle:Erdball
+813::T.O.Y.
+814::Electrocute
+815::Pzychobitch
+816::Hallucinogen
+817::Astral Projection
+818::Razed in Black
+819::Dismantled
+820::Amon Amarth
+821::Kreator
+822::Queen
+823::Sodom
+824::Flotsam and Jetsam
+825::Sabaton
+826::Hirax
+827::Venom
+828::Biomechanical
+829::Turisas
+830::Warbringer
+831::Dismember
+832::Cut Copy
+833::Katie Melua
+834::Garbage
+835::Scorpions
+836::a-ha
+837::Blur
+838::Arctic Monkeys
+839::Bloc Party
+840::Interpol
+841::Toni Braxton
+842::Bob Marley
+843::Whitney Houston
+844::Mary J. Blige
+845::Brandy
+846::Monica
+847::Kelly Clarkson
+848::Alanis Morissette
+849::Cobra Starship
+850::Cher
+851::Wir sind Helden
+852::Tokyo Police Club
+853::The Kills
+854::The Strokes
+855::Yeah Yeah Yeahs
+856::She & Him
+857::Ra Ra Riot
+858::Local Natives
+859::3 Doors Down
+860::Gwen Stefani
+861::La Roux
+862::Alphabeat
+863::Oasis
+864::Space Cowboy
+865::Pitty
+866::Ellie Goulding
+867::The Kinks
+868::Chuck Berry
+869::Ennio Morricone
+870::Kings of Convenience
+871::Elliott Smith
+872::Squarepusher
+873::of Montreal
+874::Neil Young
+875::The Veronicas
+876::Marina & the Diamonds
+877::The xx
+878::Kaiser Chiefs
+879::Electric Light Orchestra
+880::Johnny Cash
+881::Buena Vista Social Club
+882::Ozzy Osbourne
+883::1200 Micrograms
+884::Autechre
+885::Carbon Based Lifeforms
+886::Casino Versus Japan
+887::Arovane
+888::Proem
+889::The Crystal Method
+890::Ochre
+891::Belinda
+892::August Burns Red
+893::Slayer
+894::Alice in Chains
+895::Pixies
+896::The Doors
+897::Tegan and Sara
+898::Amy Winehouse
+899::Kamelot
+900::The Cardigans
+901::Yanni
+902::At the Drive-In
+903::Simple Minds
+904::The Kooks
+905::Alesha Dixon
+906::Alan Silvestri
+907::Franz Ferdinand
+908::Patrick Wolf
+909::Phoenix
+910::The Police
+911::Roy Orbison
+912::Venetian Snares
+913::Shpongle
+914::Skank
+915::The Beach Boys
+916::Samael
+917::Black Sabbath
+918::White Lies
+919::Freemasons
+920::Calvin Harris
+921::Pendulum
+922::Genesis
+923::Paul McCartney
+924::The Jimi Hendrix Experience
+925::ABBA
+926::Matt & Kim
+927::Antonio Vivaldi
+928::Just Jack
+929::Natalie Imbruglia
+930::The Last Shadow Puppets
+931::Morbid Angel
+932::Fleet Foxes
+933::Jimi Hendrix
+934::Motel
+935::Clint Mansell
+936::Brian Eno
+937::Hammock
+938::Soundtrack
+939::Little Joy
+940::UNKLE
+941::5'nizza
+942::Thrice
+943::Gnarls Barkley
+944::Flying Lotus
+945::Raekwon
+946::Four Tet
+947::Prefuse 73
+948::The Avalanches
+949::The Black Keys
+950::Metronomy
+951::Coil
+952::Steve Jablonsky
+953::Lara Fabian
+954::Yiruma
+955::M.I.A.
+956::Brown Eyed Girls
+957::Brandon Flowers
+958::Serge Gainsbourg
+959::Beach House
+960::AFX
+961::Jane Air
+962::Wisp
+963::Dr. Dre
+964::Hande Yener
+965::Aerosmith
+966::Monster Magnet
+967::Destroyer
+968::Danny Elfman
+969::Basshunter
+970::Eric Prydz
+971::Spice Girls
+972::The Drums
+973::Idiot Pilot
+974::Biffy Clyro
+975::Foals
+976::Slash
+977::will.i.am
+978::City and Colour
+979::VersaEmerge
+980::Ella Fitzgerald
+981::The Flashbulb
+982::Amon Tobin
+983::µ-Ziq
+984::Chris Clark
+985::Kettel
+986::Clark
+987::Tycho
+988::Bola
+989::The Future Sound of London
+990::Wagon Christ
+991::Telefon Tel Aviv
+992::DJ Shadow
+993::Lenny Kravitz
+994::Distance
+995::The Dead Weather
+996::The Dust Brothers
+997::Serj Tankian
+998::Wintersun
+999::The Whitest Boy Alive
+1000::Colby O'Donis
+1001::DJ Krush
+1002::Sébastien Tellier
+1003::Flunk
+1004::DJ Vadim
+1005::Paradiso Girls
+1006::Cansei de Ser Sexy
+1007::Kate Bush
+1008::Jon Brion
+1009::Slagsmålsklubben
+1010::Blonde Redhead
+1011::Steven Wilson
+1012::Jet
+1013::M83
+1014::No-Man
+1015::Sheryl Crow
+1016::[unknown]
+1017::Carole King
+1018::Rainbow
+1019::Róisín Murphy
+1020::Biosphere
+1021::Unter Null
+1022::Jack Off Jill
+1023::Tim Hecker
+1024::Broken Bells
+1025::Ratatat
+1026::Soulfly
+1027::The Horrors
+1028::Marilyn Monroe
+1029::Lumen
+1030::Танцы Минус
+1031::CocoRosie
+1032::Wings
+1033::Alcazar
+1034::The Presets
+1035::Mr. Oizo
+1036::Chromeo
+1037::Freddie Mercury
+1038::Morphine
+1039::The Dandy Warhols
+1040::Raul Seixas
+1041::The Cinematics
+1042::The Postal Service
+1043::UFO
+1044::Die Ärzte
+1045::Switchfoot
+1046::Europe
+1047::The Thermals
+1048::Audio Bullys
+1049::The Yardbirds
+1050::Tricky
+1051::Gin Blossoms
+1052::Junior Senior
+1053::Dusty Springfield
+1054::Jamie Cullum
+1055::Junior Boys
+1056::Cujo
+1057::ISAN
+1058::Sandra
+1059::Гражданская Оборона
+1060::Nicholas Hooper
+1061::Charlotte Sometimes
+1062::ILS
+1063::Survivor
+1064::StoneBridge
+1065::Kid Loco
+1066::Nancy Sinatra
+1067::System 7
+1068::Yo La Tengo
+1069::Subheim
+1070::Arcana
+1071::If These Trees Could Talk
+1072::Peter Bjorn and John
+1073::LMFAO
+1074::Birdy Nam Nam
+1075::Godspeed You! Black Emperor
+1076::Stars of the Lid
+1077::Christ.
+1078::Bibio
+1079::N*E*R*D
+1080::Theory of a Deadman
+1081::Tyler Bates
+1082::Straylight Run
+1083::Fanfarlo
+1084::Something Corporate
+1085::Nick Lachey
+1086::Black Moth Super Rainbow
+1087::Mirah
+1088::Kate Ryan
+1089::Manchester Orchestra
+1090::Hans Zimmer & James Newton Howard
+1091::RJD2
+1092::Ulrich Schnauss
+1093::D12
+1094::Young Buck
+1095::Albert Hammond, Jr.
+1096::Aztec Camera
+1097::Clinic
+1098::Twista
+1099::The Books
+1100::Hercules and Love Affair
+1101::B.B. King
+1102::Moderat
+1103::Eva Cassidy
+1104::Jerry Goldsmith
+1105::Axwell
+1106::Empire of the Sun
+1107::Plan B
+1108::The Black Ghosts
+1109::Suicidal Tendencies
+1110::Hugh Laurie
+1111::Detektivbyrån
+1112::Gescom
+1113::Polygon Window
+1114::The Tuss
+1115::Luke Vibert
+1116::Clueso
+1117::I Blame Coco
+1118::Tunng
+1119::Xavier Naidoo
+1120::The Dø
+1121::The Allman Brothers Band
+1122::The Golden Filter
+1123::Lil' Kim
+1124::m-flo
+1125::Kleerup
+1126::Neon Indian
+1127::Cinnamon Chasers
+1128::Garou
+1129::The Coral
+1130::Secret Garden
+1131::Trent Reznor
+1132::Nancy Ajram
+1133::James Taylor
+1134::Lightning Bolt
+1135::iiO
+1136::Miss Kittin & The Hacker
+1137::David Arnold
+1138::Desmond Dekker
+1139::Jan Delay
+1140::Meat Beat Manifesto
+1141::Die Fantastischen Vier
+1142::Young Jeezy
+1143::Amorphous Androgynous
+1144::Steve Winwood
+1145::Skalpel
+1146::Death in Vegas
+1147::Wavves
+1148::Bear in Heaven
+1149::Embrace
+1150::Xavier Rudd
+1151::Monolake
+1152::John Powell
+1153::Kevin Devine
+1154::I Am Kloot
+1155::Klaus Schulze
+1156::Paul Desmond
+1157::Dexter Gordon
+1158::Dan Black
+1159::Azure Ray
+1160::Tiger Lou
+1161::Trevor Rabin
+1162::Sub Focus
+1163::Bag Raiders
+1164::Silver Jews
+1165::India.Arie
+1166::Adorable
+1167::Fettes Brot
+1168::Olly Murs
+1169::Discovery
+1170::Skrillex
+1171::Dexys Midnight Runners
+1172::Swayzak
+1173::TERIYAKI BOYZ
+1174::Leftfield
+1175::ASHES dIVIDE
+1176::John B
+1177::Marco Beltrami
+1178::Remy Ma
+1179::Trina
+1180::Shawnna
+1181::Sixtoo
+1182::Plastikman
+1183::Dirty Vegas
+1184::Benassi Bros.
+1185::Agoria
+1186::Green Sun
+1187::The KLF
+1188::Anahí
+1189::Dulce María
+1190::Christian Chávez
+1191::Nicki Minaj
+1192::Enrique Iglesias
+1193::Red Hot Chili Peppers
+1194::Bryan Adams
+1195::Kid Abelha
+1196::Jordin Sparks
+1197::Alicia Keys
+1198::Katharine McPhee
+1199::Kat DeLuna
+1200::Nelly Furtado
+1201::Ciara
+1202::Keri Hilson
+1203::Jonas Brothers
+1204::T.I.
+1205::Colbie Caillat
+1206::Jason Mraz
+1207::Brad Paisley
+1208::Sugarland
+1209::Aaron Carter
+1210::*NSYNC
+1211::Miley Cyrus
+1212::No Doubt
+1213::Flo Rida
+1214::Demi Lovato
+1215::Selena Gomez & the Scene
+1216::R.E.M.
+1217::Emily Osment
+1218::Cyndi Lauper
+1219::Kelly Osbourne
+1220::Dixie Chicks
+1221::Timbaland
+1222::Vanessa Hudgens
+1223::Boys Like Girls
+1224::Camp Rock
+1225::Ace of Base
+1226::Lady Antebellum
+1227::Pitbull
+1228::Selena Gomez
+1229::Carrie Underwood
+1230::Rednex
+1231::Big Time Rush
+1232::Gloriana
+1233::Amy Macdonald
+1234::Metric
+1235::Jennette McCurdy
+1236::Keke Palmer
+1237::Dolly Parton
+1238::Billy Ray Cyrus
+1239::Pato Fu
+1240::Cássia Eller
+1241::Yellowcard
+1242::Adele
+1243::Leighton Meester
+1244::Simon Curtis
+1245::Two Door Cinema Club
+1246::Jessie James
+1247::Dierks Bentley
+1248::High School Musical
+1249::Kris Allen
+1250::Christina Perri
+1251::James Morrison
+1252::Plain White T's
+1253::Aqua
+1254::Jason Aldean
+1255::Blake Shelton
+1256::Gary Allan
+1257::Kenny Chesney
+1258::Lee Ann Womack
+1259::Matisyahu
+1260::The Lonely Island
+1261::Hanson
+1262::Tila Tequila
+1263::The Spill Canvas
+1264::Drake Bell
+1265::Edward Maya
+1266::Ladyhawke
+1267::Gabriella Cilmi
+1268::Nando Reis
+1269::TLC
+1270::Paula Abdul
+1271::Kellie Pickler
+1272::Nelly
+1273::John Denver
+1274::Miranda!
+1275::Boney M.
+1276::Matt Nathanson
+1277::Jay Sean
+1278::Charice
+1279::Flora
+1280::Sean Kingston
+1281::The Outfield
+1282::Iyaz
+1283::Fat Joe
+1284::Céline Dion
+1285::Ana Carolina
+1286::The Faint
+1287::The Cranberries
+1288::Beck
+1289::Cassie
+1290::Ashanti
+1291::Nicole Scherzinger
+1292::Destiny's Child
+1293::The Asteroids Galaxy Tour
+1294::Los Campesinos!
+1295::Mika
+1296::MGMT
+1297::Stevie Wonder
+1298::Tiziano Ferro
+1299::David Bisbal
+1300::Wisin & Yandel
+1301::Cherish
+1302::Santigold
+1303::Reba McEntire
+1304::S Club 7
+1305::Fool's Garden
+1306::Gloria Estefan
+1307::Aaliyah
+1308::Celia Cruz
+1309::Extreme
+1310::Bananarama
+1311::UB40
+1312::Mr. Big
+1313::Barry White
+1314::Sean Paul
+1315::KC and the Sunshine Band
+1316::Phantom Planet
+1317::Counting Crows
+1318::Violent Femmes
+1319::A.R. Rahman
+1320::Soda Stereo
+1321::DJ BoBo
+1322::Modern Talking
+1323::Gigi D'Agostino
+1324::Lila Downs
+1325::Tom Petty and the Heartbreakers
+1326::Does It Offend You, Yeah?
+1327::The Bangles
+1328::Fito Páez
+1329::Calle 13
+1330::Missy Elliott
+1331::Selena
+1332::Estopa
+1333::Fabolous
+1334::Los Rodríguez
+1335::The Weepies
+1336::Lissy Trullie
+1337::Bobby Valentino
+1338::Fugees
+1339::Kimya Dawson
+1340::The Little Ones
+1341::Bob Sinclar
+1342::Kabah
+1343::Don Omar
+1344::Henry Purcell
+1345::White Rabbits
+1346::Marc-Antoine Charpentier
+1347::Luigi Boccherini
+1348::Tomaso Giovanni Albinoni
+1349::Giovanni Battista Pergolesi
+1350::Hot Chocolate
+1351::Culture Beat
+1352::The Chemical Brothers
+1353::Led Zeppelin
+1354::Sam Cooke
+1355::Deltron 3030
+1356::Too $hort
+1357::Mudhoney
+1358::nevershoutnever!
+1359::David Cook
+1360::David Archuleta
+1361::Shania Twain
+1362::The Who
+1363::The Rolling Stones
+1364::Enter Shikari
+1365::Hannah Montana
+1366::Scar Symmetry
+1367::Patsy Cline
+1368::Elvis Presley
+1369::The Cheetah Girls
+1370::Aly & AJ
+1371::Everlife
+1372::Weird Al Yankovic
+1373::Liz Phair
+1374::Nat King Cole
+1375::Cryptopsy
+1376::Louis Armstrong
+1377::Relient K
+1378::Wale
+1379::Pink Martini
+1380::Air Supply
+1381::Judy Garland
+1382::Bing Crosby
+1383::Allstar Weekend
+1384::Andy Williams
+1385::Dinah Washington
+1386::Jimmy Buffett
+1387::Bobby Darin
+1388::Eartha Kitt
+1389::Martin L. Gore
+1390::Judas Priest
+1391::Tarot
+1392::Deep Purple
+1393::Mogwai
+1394::Puscifer
+1395::Eicca Toppinen
+1396::The Cult
+1397::Therion
+1398::Eisbrecher
+1399::Craig Armstrong
+1400::Steppenwolf
+1401::Vangelis
+1402::Agua de Annique
+1403::Rob Dougan
+1404::Skunk Anansie
+1405::John Murphy
+1406::A Silver Mt. Zion
+1407::Joanna Newsom
+1408::Bonobo
+1409::Saxon Shore
+1410::Yndi Halda
+1411::Jeniferever
+1412::Móveis Coloniais de Acaju
+1413::Era
+1414::Skid Row
+1415::Tarja
+1416::菅野よう子
+1417::Tristania
+1418::Queensrÿche
+1419::Creedence Clearwater Revival
+1420::George Harrison
+1421::Chico Buarque
+1422::Maria Rita
+1423::Rita Lee
+1424::Eddie Vedder
+1425::Esbjörn Svensson Trio
+1426::Andrew Lloyd Webber
+1427::Feist
+1428::Billie Holiday
+1429::Meiko
+1430::Mallu Magalhães
+1431::Cazuza
+1432::Vanguart
+1433::Amiina
+1434::Joe Cocker
+1435::William Fitzsimmons
+1436::Joshua Radin
+1437::Zé Ramalho
+1438::Firehouse
+1439::Tom Zé
+1440::Gregory and the Hawk
+1441::Melody Gardot
+1442::Richard Marx
+1443::Eric Clapton
+1444::Maurice Ravel
+1445::The Verve
+1446::Сплин
+1447::Archive
+1448::Indochine
+1449::Nightmares on Wax
+1450::The Shadows
+1451::Simon & Garfunkel
+1452::Peter Gabriel
+1453::Diana Krall
+1454::Arvo Pärt
+1455::Мумий Тролль
+1456::Belinda Carlisle
+1457::Gabriel Fauré
+1458::Crustation
+1459::Mandalay
+1460::Sergei Prokofiev
+1461::Dmitri Shostakovich
+1462::Dinosaur Pile-Up
+1463::Robert Schumann
+1464::Edvard Grieg
+1465::Carpathian Forest
+1466::George Michael
+1467::Spandau Ballet
+1468::Nik Kershaw
+1469::ABC
+1470::Damien Rice
+1471::Queens of the Stone Age
+1472::Bob Marley & The Wailers
+1473::Klaxons
+1474::Howlin' Wolf
+1475::Ray Charles
+1476::Muddy Waters
+1477::Prince
+1478::Beastie Boys
+1479::The White Stripes
+1480::Dire Straits
+1481::Gotthard
+1482::Eagles
+1483::Alice Cooper
+1484::Dark Tranquillity
+1485::At the Gates
+1486::Atreyu
+1487::Pantera
+1488::Lamb of God
+1489::Kyuss
+1490::John Williams
+1491::Helloween
+1492::Stratovarius
+1493::Journey
+1494::Testament
+1495::Death
+1496::Yngwie Malmsteen
+1497::Blind Guardian
+1498::Kasabian
+1499::The Donnas
+1500::Iron Butterfly
+1501::Orchestral Manoeuvres in the Dark
+1502::Kraftwerk
+1503::Melvins
+1504::Guns N' Roses
+1505::Mayhem
+1506::Carcass
+1507::Gamma Ray
+1508::Annihilator
+1509::Tenacious D
+1510::W.A.S.P.
+1511::Kansas
+1512::The Mamas & The Papas
+1513::Ramones
+1514::Anthrax
+1515::The Baseballs
+1516::Blind Melon
+1517::Avenged Sevenfold
+1518::Hans Zimmer
+1519::Mötley Crüe
+1520::T. Rex
+1521::Syd Barrett
+1522::The Runaways
+1523::Ratt
+1524::Stryper
+1525::Patti Smith
+1526::Jerry Cantrell
+1527::The Black Crowes
+1528::L7
+1529::Gary Numan
+1530::Thomas Dolby
+1531::Rage Against the Machine
+1532::Blue Öyster Cult
+1533::Frankie Goes to Hollywood
+1534::Heart
+1535::Twisted Sister
+1536::Dio
+1537::KISS
+1538::Van Halen
+1539::Joan Jett
+1540::Airbourne
+1541::D-A-D
+1542::Howard Shore
+1543::Boston
+1544::Fantômas
+1545::Kajagoogoo
+1546::Lita Ford
+1547::Vixen
+1548::Hanoi Rocks
+1549::Whitesnake
+1550::Joan Jett and the Blackhearts
+1551::Velvet Revolver
+1552::CRASHDÏET
+1553::Cheap Trick
+1554::Sweet
+1555::Lynyrd Skynyrd
+1556::Manowar
+1557::Thin Lizzy
+1558::Cream
+1559::Warrant
+1560::Stevie Ray Vaughan and Double Trouble
+1561::Buddy Guy
+1562::The Animals
+1563::ZZ Top
+1564::Buddy Holly
+1565::Dokken
+1566::The Stooges
+1567::Marillion
+1568::The Fall
+1569::Black Flag
+1570::Dead Kennedys
+1571::The Adicts
+1572::Supergrass
+1573::Quiet Riot
+1574::David Lee Roth
+1575::Saxon
+1576::Ugly Kid Joe
+1577::L.A. Guns
+1578::Hole
+1579::Bikini Kill
+1580::The Corrs
+1581::Uriah Heep
+1582::Asia
+1583::Dinosaur Jr.
+1584::The Vines
+1585::Status Quo
+1586::Foreigner
+1587::Poison
+1588::Burzum
+1589::Darkthrone
+1590::Girlschool
+1591::Cathedral
+1592::Doro
+1593::Cinderella
+1594::Lou Reed
+1595::Busted
+1596::New York Dolls
+1597::Kalmah
+1598::Cutting Crew
+1599::Faster Pussycat
+1600::Accept
+1601::White Lion
+1602::Dropkick Murphys
+1603::Diamond Head
+1604::Robert Johnson
+1605::Eliza Doolittle
+1606::Anvil
+1607::Winger
+1608::Jefferson Airplane
+1609::Nena
+1610::Men Without Hats
+1611::Animotion
+1612::Rob Zombie
+1613::Jeff Beck
+1614::Big Country
+1615::Modern English
+1616::Gary Moore
+1617::The Alan Parsons Project
+1618::Tomahawk
+1619::Eternal Tears of Sorrow
+1620::Sebastian Bach
+1621::Loreena McKennitt
+1622::Wall of Voodoo
+1623::Bow Wow Wow
+1624::Agent Orange
+1625::Little Walter
+1626::George Thorogood & The Destroyers
+1627::Great White
+1628::Jason Becker
+1629::The Buggles
+1630::Tesla
+1631::Screaming Trees
+1632::Bad Company
+1633::Steel Panther
+1634::Babes in Toyland
+1635::Joan Baez
+1636::Mother Love Bone
+1637::Triumph
+1638::Ted Nugent
+1639::Mad Season
+1640::Voivod
+1641::Woody Guthrie
+1642::Generation X
+1643::Bad English
+1644::Impellitteri
+1645::Johnny Thunders
+1646::Don McLean
+1647::Brian May
+1648::The Knack
+1649::Orbital
+1650::Luther Vandross
+1651::Quincy Jones
+1652::Grace Jones
+1653::Boz Scaggs
+1654::Brainstorm
+1655::The Dining Rooms
+1656::Sérgio Mendes
+1657::Al Green
+1658::A Day to Remember
+1659::Pillar
+1660::Disciple
+1661::Family Force 5
+1662::Fireflight
+1663::Talk Talk
+1664::Prefab Sprout
+1665::Icehouse
+1666::Fiction Factory
+1667::Jean-Michel Jarre
+1668::Howard Jones
+1669::Thompson Twins
+1670::The Fixx
+1671::Level 42
+1672::Erasure
+1673::Berlin
+1674::The Psychedelic Furs
+1675::Parliament
+1676::Art of Noise
+1677::XTC
+1678::Pseudo Echo
+1679::Midnight Oil
+1680::Shriekback
+1681::Wire
+1682::Sparks
+1683::Yellow Magic Orchestra
+1684::John Foxx
+1685::Landscape
+1686::Scritti Politti
+1687::Yello
+1688::New Musik
+1689::Cabaret Voltaire
+1690::LCD Soundsystem
+1691::Hall & Oates
+1692::Arcadia
+1693::The Damned
+1694::Soft Cell
+1695::The Associates
+1696::Propaganda
+1697::Go West
+1698::Gang of Four
+1699::Fad Gadget
+1700::Boytronic
+1701::Blancmange
+1702::Classix Nouveaux
+1703::Wang Chung
+1704::The The
+1705::The Communards
+1706::Saga
+1707::Telex
+1708::!!!
+1709::Blue Peter
+1710::O.S.T.R.
+1711::Kult
+1712::Pidżama Porno
+1713::Symphony X
+1714::Alestorm
+1715::Falconer
+1716::Star One
+1717:::wumpscut:
+1718::Electronic
+1719::Marc Almond
+1720::Pulp
+1721::Manic Street Preachers
+1722::Wolfsheim
+1723::The Good, the Bad & the Queen
+1724::Rooney
+1725::The Residents
+1726::Bee Gees
+1727::Tuxedomoon
+1728::In Extremo
+1729::Deutsch Amerikanische Freundschaft
+1730::Dagoba
+1731::Throbbing Gristle
+1732::Captain Beefheart & His Magic Band
+1733::The Legendary Pink Dots
+1734::The Power Station
+1735::Minus the Bear
+1736::The Stone Roses
+1737::Angelo Badalamenti
+1738::At Vance
+1739::The Pipettes
+1740::Tocotronic
+1741::Datarock
+1742::Astor Piazzolla
+1743::Oi Va Voi
+1744::Electric Six
+1745::Jarvis Cocker
+1746::Neurotic Outsiders
+1747::The Devils
+1748::Gloria Gaynor
+1749::Flowing Tears
+1750::Perfume
+1751::Versailles
+1752::ルルティア
+1753::múm
+1754::Aphex Twin
+1755::Boards of Canada
+1756::Ellen Allien
+1757::Daedelus
+1758::B. Fleischmann
+1759::Booka Shade
+1760::Mujuice
+1761::edIT
+1762::Ellen Allien & Apparat
+1763::Frog Pocket
+1764::Rotator
+1765::Spor
+1766::Noisia
+1767::Black Sun Empire
+1768::The Orb
+1769::Ricardo Villalobos
+1770::William Basinski
+1771::Pole
+1772::Deadbeat
+1773::Dntel
+1774::Add N to (X)
+1775::Brothomstates
+1776::Dominik Eulberg
+1777::Richie Hawtin
+1778::Mathew Jonson
+1779::Hexstatic
+1780::Alva Noto + Ryuichi Sakamoto
+1781::Kings of Leon
+1782::Joss Stone
+1783::Yeni Türkü
+1784::Barış Manço
+1785::Mor ve Ötesi
+1786::Düş Sokağı Sakinleri
+1787::Cradle of Filth
+1788::Little Richard
+1789::HammerFall
+1790::Massacration
+1791::Creed
+1792::The Hellacopters
+1793::Backyard Babies
+1794::Kelly Rowland
+1795::Gloria Trevi
+1796::Ludacris
+1797::Jennifer Hudson
+1798::Michelle Williams
+1799::Trey Songz
+1800::Pyotr Ilyich Tchaikovsky
+1801::Sade
+1802::Max Richter
+1803::Jack Johnson
+1804::Jethro Tull
+1805::The Album Leaf
+1806::John Mayer
+1807::Bruce Springsteen
+1808::Trans-Siberian Orchestra
+1809::Bill Withers
+1810::Five Finger Death Punch
+1811::Santana
+1812::Smash Mouth
+1813::Mike & The Mechanics
+1814::Kim Wilde
+1815::New Found Glory
+1816::Georg Friedrich Händel
+1817::Secondhand Serenade
+1818::Starsailor
+1819::Daddy Yankee
+1820::B.J. Thomas
+1821::OutKast
+1822::Ludovico Einaudi
+1823::Norah Jones
+1824::Amethystium
+1825::Diana Ross
+1826::Chris Cornell
+1827::Luciano Pavarotti
+1828::Sarah Brightman
+1829::Andrea Bocelli
+1830::Vanessa-Mae
+1831::The Rascals
+1832::k-os
+1833::Islands
+1834::Martina McBride
+1835::Juanes
+1836::Garth Brooks
+1837::Little Big Town
+1838::Tim McGraw
+1839::Faith Hill
+1840::Glenn Miller
+1841::Reik
+1842::George Strait
+1843::Alan Jackson
+1844::Paul Young
+1845::The Doobie Brothers
+1846::Van Morrison
+1847::LL Cool J
+1848::Naked Eyes
+1849::Sin Bandera
+1850::Ringo Starr
+1851::Carpenters
+1852::Paul McCartney & Wings
+1853::Matchbox Twenty
+1854::The Moody Blues
+1855::Barbra Streisand
+1856::Supertramp
+1857::The Jackson 5
+1858::Sarah McLachlan
+1859::America
+1860::Jordan Pruitt
+1861::Tarkan
+1862::Alabama
+1863::Montgomery Gentry
+1864::Ronnie Milsap
+1865::Sara Evans
+1866::SHeDaisy
+1867::Coheed and Cambria
+1868::Big & Rich
+1869::Sly & The Family Stone
+1870::Chris Isaak
+1871::Eiffel 65
+1872::Tom Jones
+1873::Leaves' Eyes
+1874::Lionel Richie
+1875::Jars of Clay
+1876::The Ataris
+1877::Hank Williams
+1878::Willie Nelson
+1879::Emmylou Harris
+1880::All Saints
+1881::Jim Croce
+1882::The Ventures
+1883::Kenny Loggins
+1884::Grand Funk Railroad
+1885::Alejandro Sanz
+1886::The Go-Go's
+1887::Adam Ant
+1888::Quarterflash
+1889::Rick Springfield
+1890::Miranda Lambert
+1891::Ben Harper
+1892::Chicago
+1893::LeAnn Rimes
+1894::The Ronettes
+1895::Jon Secada
+1896::Mae
+1897::Kellplanet
+1898::Afro Celt Sound System
+1899::The Presidents of the United States of America
+1900::Procol Harum
+1901::The Monkees
+1902::Jónsi & Alex
+1903::Terri Clark
+1904::The Avett Brothers
+1905::John Fogerty
+1906::The Everly Brothers
+1907::Ricardo Arjona
+1908::The Hollies
+1909::The Association
+1910::Herman's Hermits
+1911::Joni Mitchell
+1912::REO Speedwagon
+1913::Tina Arena
+1914::Kitaro
+1915::The Platters
+1916::The Drifters
+1917::Boyz II Men
+1918::Kool & The Gang
+1919::Stevie Nicks
+1920::Don Henley
+1921::The Lovin' Spoonful
+1922::Chad & Jeremy
+1923::Israel Kamakawiwo'ole
+1924::Lou Rawls
+1925::Dionne Warwick
+1926::Lorrie Morgan
+1927::Neil Sedaka
+1928::Connie Francis
+1929::Marc Anthony
+1930::Richard Hawley
+1931::Billy Ocean
+1932::Zucchero
+1933::The Pogues
+1934::Conway Twitty
+1935::The Oak Ridge Boys
+1936::Randy Travis
+1937::Blackhawk
+1938::Michael McDonald
+1939::Barry Manilow
+1940::Harry Nilsson
+1941::Peter, Paul & Mary
+1942::The Temptations
+1943::Tony Bennett
+1944::Nickel Creek
+1945::George Winston
+1946::Dwight Yoakam
+1947::Shivaree
+1948::Sheena Easton
+1949::Perry Como
+1950::Xela
+1951::Los Lobos
+1952::The Guess Who
+1953::The Boo Radleys
+1954::Nick Lowe
+1955::MC Hammer
+1956::Melissa Etheridge
+1957::Scott McKenzie
+1958::Cassandra Wilson
+1959::Steel Magnolia
+1960::The Jesus Lizard
+1961::Foetus
+1962::The Birthday Party
+1963::The Jesus and Mary Chain
+1964::Big Black
+1965::Shellac
+1966::Default
+1967::The Brand New Heavies
+1968::Jim Morrison
+1969::65daysofstatic
+1970::Late of the Pier
+1971::We Are Scientists
+1972::Good Shoes
+1973::Mystery Jets
+1974::Maximum the Hormone
+1975::Gloria
+1976::McFly
+1977::Asking Alexandria
+1978::The Devil Wears Prada
+1979::blessthefall
+1980::YUI
+1981::A Skylit Drive
+1982::Flow
+1983::the GazettE
+1984::HIGH and MIGHTY COLOR
+1985::Miss May I
+1986::Attack Attack!
+1987::The Word Alive
+1988::From First to Last
+1989::Pierce the Veil
+1990::Before Their Eyes
+1991::We Came As Romans
+1992::Eyes Set to Kill
+1993::Woe, Is Me
+1994::X JAPAN
+1995::アンティック-珈琲店-
+1996::Drop Dead, Gorgeous
+1997::ORANGE RANGE
+1998::Andrew Bird
+1999::Spoon
+2000::Iron & Wine
+2001::Dave Matthews Band
+2002::The New Pornographers
+2003::Billy Joel
+2004::Barenaked Ladies
+2005::Randy Newman
+2006::Ben Folds
+2007::Paul Simon
+2008::Rufus Wainwright
+2009::Ben Folds Five
+2010::Neko Case
+2011::Calexico
+2012::Dashboard Confessional
+2013::Regina Spektor
+2014::Reel Big Fish
+2015::Foxboro Hot Tubs
+2016::Alien Ant Farm
+2017::Screamin' Jay Hawkins
+2018::A Fine Frenzy
+2019::Bad Religion
+2020::Mombojó
+2021::Silverchair
+2022::Little Boots
+2023::Hey Monday
+2024::Engenheiros do Hawaii
+2025::Capital Inicial
+2026::Legião Urbana
+2027::Titãs
+2028::HORSE the band
+2029::As I Lay Dying
+2030::Millencolin
+2031::Breathe Carolina
+2032::The Maine
+2033::Forever the Sickest Kids
+2034::Sum 41
+2035::Mayday Parade
+2036::We The Kings
+2037::Cute Is What We Aim For
+2038::Sonata Arctica
+2039::The Fratellis
+2040::Raimundos
+2041::Faichecleres
+2042::Charlie Brown Jr.
+2043::Caetano Veloso
+2044::Tiê
+2045::Shinedown
+2046::The Raconteurs
+2047::Vasco Rossi
+2048::Vega 4
+2049::The Turtles
+2050::Trivium
+2051::Kid Rock
+2052::The Classic Crime
+2053::A Rocket to the Moon
+2054::Bidê ou Balde
+2055::Anberlin
+2056::Lordi
+2057::The Cab
+2058::This Providence
+2059::Slash's Snakepit
+2060::Gym Class Heroes
+2061::Murderdolls
+2062::Rosa De Saron
+2063::Eros Ramazzotti
+2064::Chimarruts
+2065::Daniel Powter
+2066::Walls of Jericho
+2067::Barão Vermelho
+2068::MxPx
+2069::Forgotten Boys
+2070::Ingrid Michaelson
+2071::Tiago Iorc
+2072::Cachorro Grande
+2073::Graforréia Xilarmônica
+2074::Forfun
+2075::Ira!
+2076::Zebrahead
+2077::Collide
+2078::Mamonas Assassinas
+2079::4 Non Blondes
+2080::Bonde do Rolê
+2081::Cash Cash
+2082::CPM 22
+2083::Beeshop
+2084::The Academy Is...
+2085::The Rocket Summer
+2086::Artist Vs. Poet
+2087::Drive
+2088::Mushroomhead
+2089::Alice Nine
+2090::Fu Manchu
+2091::Autoramas
+2092::Rock Rocket
+2093::Milton Nascimento
+2094::The Friday Night Boys
+2095::LM.C
+2096::Gilberto Gil
+2097::The Juliana Theory
+2098::Bonde das Impostora
+2099::Júpiter Maçã
+2100::Deborah Blando
+2101::Lipstick
+2102::Faber Drive
+2103::AFI
+2104::Buckcherry
+2105::Hardcore Superstar
+2106::Vains of Jenna
+2107::Sixx:A.M.
+2108::Concrete Blonde
+2109::Mott the Hoople
+2110::鷺巣詩郎
+2111::Burial
+2112::Mono
+2113::Gimmik
+2114::36 Crazyfists
+2115::Haste the Day
+2116::Hadouken!
+2117::Skinny Puppy
+2118::The Enemy
+2119::Boxcutter
+2120::Дельфин
+2121::Xploding Plastix
+2122::Maxïmo Park
+2123::Cult of Luna
+2124::Sakura
+2125::Flobots
+2126::Neurosis
+2127::Maybeshewill
+2128::Dolphin
+2129::Stigmata
+2130::Everything Is Made in China
+2131::Ocean Colour Scene
+2132::The Pigeon Detectives
+2133::The Streets
+2134::Roadrunner United
+2135::Pelican
+2136::Rashamba
+2137::Hol Baumann
+2138::Jesus on Extasy
+2139::Ef
+2140::Rosetta
+2141::Shitdisco
+2142::Benga
+2143::Moondog
+2144::Glen Hansard
+2145::Camouflage
+2146::Cock Robin
+2147::Simply Red
+2148::Janet Jackson
+2149::Gabrielle
+2150::Saint Etienne
+2151::Annie Lennox
+2152::Crowded House
+2153::Zoé
+2154::Chris Rea
+2155::DJ Sammy
+2156::Inspiral Carpets
+2157::Donna Summer
+2158::The Jacksons
+2159::Chaka Khan
+2160::Chic
+2161::Teena Marie
+2162::Happy Mondays
+2163::Toto
+2164::Shakin' Stevens
+2165::Milk Inc.
+2166::Iris
+2167::JLS
+2168::Bonnie Tyler
+2169::Seal
+2170::Julio Iglesias
+2171::Chris de Burgh
+2172::Haddaway
+2173::Dr. Alban
+2174::2 Unlimited
+2175::Black
+2176::Basement Jaxx
+2177::Shapeshifters
+2178::The Searchers
+2179::The Pretenders
+2180::When in Rome
+2181::James
+2182::T'Pau
+2183::Wet Wet Wet
+2184::The Charlatans
+2185::Alison Moyet
+2186::The Blow Monkeys
+2187::Mylo
+2188::Band Aid
+2189::The Beloved
+2190::Texas
+2191::Dubstar
+2192::EMF
+2193::Sash!
+2194::Pat Benatar
+2195::The Archies
+2196::Commodores
+2197::Loverboy
+2198::Manfred Mann
+2199::Del Amitri
+2200::The Shamen
+2201::Roger Sanchez
+2202::R. Kelly
+2203::Bronski Beat
+2204::The Four Tops
+2205::Andy Bell
+2206::Cliff Richard
+2207::Alexander O'Neal
+2208::Tammy Wynette
+2209::Lisa Stansfield
+2210::Snap!
+2211::Glen Campbell
+2212::The Alarm
+2213::Chumbawamba
+2214::The Bluetones
+2215::F.R. David
+2216::Bros
+2217::Sister Sledge
+2218::Soul II Soul
+2219::The Trammps
+2220::Petula Clark
+2221::Shannon
+2222::Technotronic
+2223::Alice DeeJay
+2224::Ensiferum
+2225::Angra
+2226::Norther
+2227::Insomnium
+2228::Skyfire
+2229::James LaBrie
+2230::Lil' Wayne
+2231::Blue October
+2232::The Prodigy
+2233::Bombay Bicycle Club
+2234::The Scene Aesthetic
+2235::Dolores O'Riordan
+2236::Crossfade
+2237::Noize MC
+2238::Pink
+2239::Jamelia
+2240::Агата Кристи
+2241::Amatory
+2242::Sunrise Avenue
+2243::Manga
+2244::Evergreen Terrace
+2245::Би-2
+2246::Biohazard
+2247::Кино
+2248::Benny Benassi
+2249::KoЯn
+2250::Timo Maas
+2251::Слот
+2252::Novembre
+2253::Unwritten Law
+2254::Crazy Town
+2255::DMX
+2256::Пилот
+2257::Evans Blue
+2258::Propellerheads
+2259::South Park
+2260::Bomfunk MC's
+2261::Кирпичи
+2262::Король и Шут
+2263::Lil Jon & The East Side Boyz
+2264::Hush
+2265::Lifehouse
+2266::Bush
+2267::Lovex
+2268::In This Moment
+2269::Black Tide
+2270::Slaughter
+2271::Pixie Lott
+2272::Claudia Leitte
+2273::Mylène Farmer
+2274::8mm
+2275::KLOQ
+2276::Hooverphonic
+2277::Kelis
+2278::Melanie C
+2279::Ivete Sangalo
+2280::Babado Novo
+2281::Rob Thomas
+2282::Elliott Yamin
+2283::Adriana Calcanhotto
+2284::Camille
+2285::Ani DiFranco
+2286::Emma Bunton
+2287::Danni Carlos
+2288::The Notwist
+2289::The Subways
+2290::Mando Diao
+2291::Kashmir
+2292::Blue Foundation
+2293::Lali Puna
+2294::Broadcast
+2295::Jackie Wilson
+2296::Marvin Gaye
+2297::The Isley Brothers
+2298::Earth, Wind & Fire
+2299::Sissel
+2300::Alejandra Guzmán
+2301::Giuseppe Verdi
+2302::Miguel Bosé
+2303::Johannes Brahms
+2304::Mijares
+2305::Charlotte Church
+2306::Curtis Mayfield
+2307::Antonín Dvořák
+2308::Fatback Band
+2309::Felix Mendelssohn
+2310::Johann Pachelbel
+2311::Diana Vickers
+2312::Parachute
+2313::Teddy Geiger
+2314::Brie Larson
+2315::HorrorPops
+2316::Sandy e Junior
+2317::The Distillers
+2318::Kevin Federline
+2319::Marisa Monte
+2320::Erykah Badu
+2321::Maria Bethânia
+2322::Kate Voegele
+2323::Los Hermanos
+2324::Jeffree Star
+2325::Sia
+2326::Психея
+2327::Emilíana Torrini
+2328::The Calling
+2329::Fatboy Slim
+2330::Omarion
+2331::Paolo Nutini
+2332::The Hives
+2333::Serebro
+2334::Skye Sweetnam
+2335::Anastacia
+2336::The Medic Droid
+2337::Schiller
+2338::NX Zero
+2339::LeToya
+2340::Stars
+2341::Kyo
+2342::Nouvelle Vague
+2343::Detonautas Roque Clube
+2344::Fiona Apple
+2345::Kudai
+2346::Corinne Bailey Rae
+2347::Bryan Ferry
+2348::Julieta Venegas
+2349::МакSим
+2350::Vanessa da Mata
+2351::Marcelo D2
+2352::DJ Cam
+2353::O Rappa
+2354::Os Mutantes
+2355::Lenine
+2356::SR-71
+2357::Antônio Carlos Jobim
+2358::Plazma
+2359::Freezepop
+2360::American Hi-Fi
+2361::Diddy
+2362::Cartel
+2363::Babasónicos
+2364::Erin McCarley
+2365::Lillix
+2366::The Almost
+2367::Сергей Лазарев
+2368::Queen Latifah
+2369::Ludov
+2370::Gram
+2371::Babyface
+2372::Tribalistas
+2373::Юлия Савичева
+2374::Cibelle
+2375::Bliss
+2376::Vinicius de Moraes
+2377::(hed) Planet Earth
+2378::In-Grid
+2379::Lemongrass
+2380::Global Communication
+2381::Rod Stewart
+2382::John Legend
+2383::Wyclef Jean
+2384::Band of Horses
+2385::Cat Power
+2386::BrokeNCYDE
+2387::Aloha From Hell
+2388::The Pains of Being Pure at Heart
+2389::+44
+2390::Kevin Rudolf
+2391::Joey Ramone
+2392::Emperor
+2393::Spock's Beard
+2394::Pain of Salvation
+2395::Meshuggah
+2396::Red Sparowes
+2397::Strapping Young Lad
+2398::Antimatter
+2399::Ulver
+2400::Bathory
+2401::Textures
+2402::The Ocean
+2403::Riverside
+2404::Moonsorrow
+2405::My Dying Bride
+2406::Ozric Tentacles
+2407::Younger Brother
+2408::Fair to Midland
+2409::Oceansize
+2410::Enslaved
+2411::Anna Ternheim
+2412::Lake of Tears
+2413::Aereogramme
+2414::Dimmu Borgir
+2415::Arch Enemy
+2416::Virgin Prunes
+2417::Fields of the Nephilim
+2418::Ween
+2419::Specimen
+2420::White Zombie
+2421::Plasmatics
+2422::Ten Years After
+2423::Faithless
+2424::Kosheen
+2425::Sugababes
+2426::Laura Pausini
+2427::Darin
+2428::Nick Cave and the Bad Seeds
+2429::Siobhan Donaghy
+2430::Delta Goodrem
+2431::Edyta Bartosiewicz
+2432::Far East Movement
+2433::Solange
+2434::Mónica Naranjo
+2435::Morandi
+2436::Robyn
+2437::Matt Wertz
+2438::Courtney Love
+2439::Lee Ryan
+2440::Five for Fighting
+2441::No Mercy
+2442::Everything but the Girl
+2443::Lamb
+2444::Jon McLaughlin
+2445::Amerie
+2446::Jazmine Sullivan
+2447::Sam Sparro
+2448::Kasia Kowalska
+2449::Elisa
+2450::Kurt Nilsen
+2451::Lucie Silvas
+2452::Howie Day
+2453::Beverley Knight
+2454::Live
+2455::Moloko
+2456::Joe Purdy
+2457::Melanie Fiona
+2458::Varius Manx
+2459::AaRON
+2460::Pati Yang
+2461::Jamie Foxx
+2462::Meredith Brooks
+2463::Paola & Chiara
+2464::Husky Rescue
+2465::Babylon Zoo
+2466::Kate Havnevik
+2467::Joan Osborne
+2468::Skin
+2469::La Bouche
+2470::Mutya Buena
+2471::All That Remains
+2472::Eths
+2473::Blood Stain Child
+2474::Light This City
+2475::Mnemic
+2476::Delain
+2477::The Agonist
+2478::Finger Eleven
+2479::Victoria Beckham
+2480::Alison Krauss
+2481::Jessica Andrews
+2482::Emerson Drive
+2483::Gretchen Wilson
+2484::The Frames
+2485::Lisa Hannigan
+2486::Willow Smith
+2487::Amanda Palmer
+2488::Baths
+2489::Lil B
+2490::Behemoth
+2491::Whitechapel
+2492::All Shall Perish
+2493::Despised Icon
+2494::Suicide Silence
+2495::Nile
+2496::Cannibal Corpse
+2497::Hatebreed
+2498::Job for a Cowboy
+2499::Carnifex
+2500::Beneath the Massacre
+2501::Annotations of an Autopsy
+2502::Оригами
+2503::DevilDriver
+2504::2H Company
+2505::Enduser
+2506::Kid606
+2507::SikTh
+2508::Shitmat
+2509::Machine Head
+2510::Decapitated
+2511::Russian Circles
+2512::Otep
+2513::Wednesday 13
+2514::Dry Kill Logic
+2515::Chimaira
+2516::Glassjaw
+2517::Aborted
+2518::Madball
+2519::Unleashed
+2520::Ion Dissonance
+2521::7раса
+2522::Deicide
+2523::Obituary
+2524::The Exploited
+2525::Shadows Fall
+2526::Cavalera Conspiracy
+2527::Bluetech
+2528::7000$
+2529::Kataklysm
+2530::Clawfinger
+2531::10 Years
+2532::The Haunted
+2533::Six Feet Under
+2534::Damageplan
+2535::Korea
+2536::Modeselektor
+2537::Crossbreed
+2538::Pan Sonic
+2539::Vital Remains
+2540::Nasum
+2541::Brujeria
+2542::Atari Teenage Riot
+2543::Rusko
+2544::Animosity
+2545::The Number Twelve Looks Like You
+2546::The Bug
+2547::Superjoint Ritual
+2548::Bon Iver
+2549::These New Puritans
+2550::Alexisonfire
+2551::Mumford & Sons
+2552::K'naan
+2553::Snoop Dogg
+2554::Death from Above 1979
+2555::Delphic
+2556::Wolfmother
+2557::Friendly Fires
+2558::Everything Everything
+2559::311
+2560::Warpaint
+2561::Passion Pit
+2562::Starfucker
+2563::Robin Thicke
+2564::jj
+2565::Nosaj Thing
+2566::Washed Out
+2567::Gossip
+2568::Razorlight
+2569::The Ordinary Boys
+2570::Eisley
+2571::The Flaming Lips
+2572::Pretty Girls Make Graves
+2573::The Slits
+2574::Guster
+2575::Augustana
+2576::The Kovenant
+2577::Black Label Society
+2578::Soilwork
+2579::Iced Earth
+2580::Ayreon
+2581::Agalloch
+2582::Edguy
+2583::Running Wild
+2584::Demons & Wizards
+2585::King Crimson
+2586::DragonForce
+2587::Dethklok
+2588::Styx
+2589::Korpiklaani
+2590::Firewind
+2591::Finntroll
+2592::Funkadelic
+2593::Bad Manners
+2594::Bad Brains
+2595::The Gathering
+2596::Labyrinth
+2597::Elvenking
+2598::Candlemass
+2599::Persuader
+2600::Metal Church
+2601::Cameo
+2602::Windir
+2603::Steve Miller Band
+2604::Andre 3000
+2605::The Sword
+2606::Ohio Players
+2607::Alcatrazz
+2608::Mountain
+2609::King Diamond
+2610::Devin Townsend
+2611::Bloodbath
+2612::Cynic
+2613::Mustard Plug
+2614::CéU
+2615::Nitin Sawhney
+2616::Minutemen
+2617::Antony and the Johnsons
+2618::Emilie Simon
+2619::Cocteau Twins
+2620::After Forever
+2621::Devil Doll
+2622::Violeta Parra
+2623::Porter
+2624::Gerry Rafferty
+2625::Steel Pulse
+2626::Bowling for Soup
+2627::Chevelle
+2628::Wheatus
+2629::Tamia
+2630::Westlife
+2631::Five
+2632::Jamal
+2633::Eva Simons
+2634::Kelly Key
+2635::Mr. President
+2636::Whigfield
+2637::Ice MC
+2638::Arash
+2639::Daniela Mercury
+2640::Lawrence
+2641::LS Jack
+2642::Steps
+2643::E-Type
+2644::Between the Buried and Me
+2645::Born of Osiris
+2646::Periphery
+2647::Veil of Maya
+2648::The Contortionist
+2649::Within The Ruins
+2650::After the Burial
+2651::Moving Mountains
+2652::Real Life
+2653::John Taylor
+2654::The Stills
+2655::Orianthi
+2656::Heaven Shall Burn
+2657::Austrian Death Machine
+2658::Stray From the Path
+2659::Ana Cañas
+2660::Silbermond
+2661::Eluveitie
+2662::Die Toten Hosen
+2663::Hot Water Music
+2664::Caliban
+2665::It Prevails
+2666::I Killed the Prom Queen
+2667::Lisa Marie Presley
+2668::Abandon All Ships
+2669::The Crimson Armada
+2670::Architects
+2671::Alter Bridge
+2672::Kutless
+2673::Maylene and the Sons of Disaster
+2674::Confide
+2675::Demon Hunter
+2676::Disarmonia Mundi
+2677::Kyte
+2678::Nightrage
+2679::Misery Signals
+2680::House vs. Hurricane
+2681::One Morning Left
+2682::Kids In Glass Houses
+2683::FM Static
+2684::Head
+2685::Threat Signal
+2686::Becoming the Archetype
+2687::Against Me!
+2688::Lucero
+2689::Mooncake
+2690::Of Mice & Men
+2691::Sabrepulse
+2692::Norma Jean
+2693::Blockhead
+2694::Emancipator
+2695::Мои Ракеты Вверх
+2696::The American Dollar
+2697::Caspian
+2698::NEVERSMILE
+2699::Xe-NONE
+2700::Asobi Seksu
+2701::Lydia
+2702::Bassnectar
+2703::The Amity Affliction
+2704::Maria Mena
+2705::Gorgoroth
+2706::Marduk
+2707::Immortal
+2708::Destruction
+2709::Entombed
+2710::Borknagar
+2711::Nocturnal Depression
+2712::Zемфира
+2713::Adriano Celentano
+2714::Alex Gaudino
+2715::Shaggy
+2716::Eels
+2717::Tortoise
+2718::Lights
+2719::Nadja
+2720::PJ Harvey
+2721::Leonard Cohen
+2722::Architecture in Helsinki
+2723::The Sound of Animals Fighting
+2724::Starlight Mints
+2725::Mercyful Fate
+2726::Anti-Flag
+2727::Isaac Hayes
+2728::The 69 Eyes
+2729::The Von Bondies
+2730::Deerhoof
+2731::Apostle of Hustle
+2732::Primus
+2733::Xiu Xiu
+2734::Built to Spill
+2735::The Fiery Furnaces
+2736::Peeping Tom
+2737::Public Image Ltd.
+2738::Eagles of Death Metal
+2739::Charlotte Gainsbourg
+2740::John Cage
+2741::The Sugarcubes
+2742::Danzig
+2743::John Zorn
+2744::Nurse With Wound
+2745::Scarling.
+2746::Caribou
+2747::Liars
+2748::GWAR
+2749::Sun City Girls
+2750::Secret Chiefs 3
+2751::Celtic Frost
+2752::Into Eternity
+2753::Funki Porcini
+2754::Leæther Strip
+2755::Auf der Maur
+2756::Living Colour
+2757::Lovage
+2758::Esthero
+2759::Faust
+2760::The Go! Team
+2761::Camera Obscura
+2762::The 5.6.7.8's
+2763::Exciter
+2764::Don Caballero
+2765::Helmet
+2766::Eluvium
+2767::Pinback
+2768::Angelspit
+2769::Bif Naked
+2770::John Frusciante
+2771::Harold Budd
+2772::Sloan
+2773::Daniel Johnston
+2774::Meat Puppets
+2775::Beulah
+2776::The Creepshow
+2777::Matthew Herbert
+2778::Tosca
+2779::Afterlife
+2780::El Perro del Mar
+2781::Gravity Kills
+2782::The Postmarks
+2783::Minor Threat
+2784::Hella
+2785::Veruca Salt
+2786::Love Is All
+2787::Buckethead
+2788::Richard Cheese
+2789::Mouse on Mars
+2790::MC5
+2791::My Ruin
+2792::Hanzel und Gretyl
+2793::Gang Gang Dance
+2794::The Concretes
+2795::The Long Blondes
+2796::OOIOO
+2797::Shannon Wright
+2798::Marcomé
+2799::Terry Riley
+2800::Tristan Feldbauer
+2801::Warren Zevon
+2802::Daniel Lanois
+2803::The Tragically Hip
+2804::Karlheinz Stockhausen
+2805::Hello Saferide
+2806::Unexpect
+2807::This Heat
+2808::Bill Laswell
+2809::Art Ensemble of Chicago
+2810::Milivoj Culibrk
+2811::Estradasphere
+2812::The Boy Least Likely To
+2813::The Wimshurst's Machine
+2814::Wintersleep
+2815::Ministry
+2816::New Model Army
+2817::Johnny Hates Jazz
+2818::Robert Palmer
+2819::The Skatalites
+2820::Lush
+2821::Dead Can Dance
+2822::Pavement
+2823::Suicide
+2824::Elvis Costello & The Attractions
+2825::The Beat
+2826::The Rapture
+2827::Suzanne Vega
+2828::The Sundays
+2829::Neil Diamond
+2830::Crass
+2831::Clock DVA
+2832::Toad the Wet Sprocket
+2833::GBH
+2834::Slowdive
+2835::Mazzy Star
+2836::The Rutles
+2837::Brendan Benson
+2838::Badly Drawn Boy
+2839::The Undertones
+2840::The Band
+2841::The La's
+2842::The Modern Lovers
+2843::Mr. Mister
+2844::Autograph
+2845::Scandal
+2846::Falco
+2847::Soft Machine
+2848::Black Tape for a Blue Girl
+2849::The Rembrandts
+2850::Buzzcocks
+2851::Aimee Mann
+2852::The Comsat Angels
+2853::The Danse Society
+2854::The Sounds
+2855::Sebadoh
+2856::Klaus Nomi
+2857::The Lemonheads
+2858::Laura Branigan
+2859::Book of Love
+2860::Tim Finn
+2861::Camper Van Beethoven
+2862::Pete Shelley
+2863::The Icicle Works
+2864::Buffalo Springfield
+2865::Badfinger
+2866::The Mission
+2867::Ride
+2868::The House of Love
+2869::France Gall
+2870::And Also the Trees
+2871::The Church
+2872::.38 Special
+2873::X-Ray Spex
+2874::Prince Buster
+2875::Teenage Fanclub
+2876::Big Star
+2877::The Korgis
+2878::The Raincoats
+2879::Severed Heads
+2880::Hüsker Dü
+2881::The Germs
+2882::Sham 69
+2883::Television Personalities
+2884::The Adverts
+2885::X-Mal Deutschland
+2886::The Replacements
+2887::IMA Robot
+2888::Freur
+2889::Youth Group
+2890::Love Spirals Downwards
+2891::Low
+2892::Todd Rundgren
+2893::Stereo Total
+2894::Television
+2895::The Blue Nile
+2896::Hothouse Flowers
+2897::Kirsty MacColl
+2898::Billy Bragg
+2899::Josef K
+2900::Sid Vicious
+2901::The Lightning Seeds
+2902::UK Subs
+2903::YACHT
+2904::Young Marble Giants
+2905::Talulah Gosh
+2906::Faces
+2907::The Teardrop Explodes
+2908::Pere Ubu
+2909::The Boomtown Rats
+2910::Pop Will Eat Itself
+2911::Das Ich
+2912::Stromae
+2913::Social Distortion
+2914::Salem
+2915::The Black Dahlia Murder
+2916::Sonic Syndicate
+2917::Emmure
+2918::The Dillinger Escape Plan
+2919::Dance Gavin Dance
+2920::Agoraphobic Nosebleed
+2921::Mastodon
+2922::Naked City
+2923::Rhapsody of Fire
+2924::Exodus
+2925::Primal Fear
+2926::Tankard
+2927::Artillery
+2928::Toxic Holocaust
+2929::Municipal Waste
+2930::Napalm Death
+2931::Bleeding Through
+2932::Waking The Cadaver
+2933::Impending Doom
+2934::See You Next Tuesday
+2935::Megaherz
+2936::Architect
+2937::ВFI
+2938::Blind Witness
+2939::We Butter The Bread With Butter
+2940::It Dies Today
+2941::Danko Jones
+2942::Best Coast
+2943::Protest The Hero
+2944::Ляпис Трубецкой
+2945::1349
+2946::Vader
+2947::Dark Funeral
+2948::Joe Bonamassa
+2949::Machinae Supremacy
+2950::Hypocrisy
+2951::Converge
+2952::The Ghost Inside
+2953::Comeback Kid
+2954::Have Heart
+2955::New Years Day
+2956::iwrestledabearonce
+2957::Ektomorf
+2958::Blood Red Shoes
+2959::Misery Index
+2960::The Faceless
+2961::The Chariot
+2962::Asesino
+2963::H2O
+2964::Possessed
+2965::Dissection
+2966::Altaria
+2967::The Acacia Strain
+2968::Terror
+2969::In Fear and Faith
+2970::Necrophagist
+2971::Adept
+2972::Anal Cunt
+2973::I Declare War
+2974::Dr. Acula
+2975::And Hell Followed With
+2976::Attila
+2977::War From a Harlots Mouth
+2978::Molotov Solution
+2979::The Tony Danza Tapdance Extravaganza
+2980::To/Die/For
+2981::Throwdown
+2982::Sadus
+2983::Agnostic Front
+2984::Emigrate
+2985::Dungeon Elite [**]
+2986::iamerror
+2987::Arsonists Get All The Girls
+2988::Greeley Estates
+2989::Coroner
+2990::I:Scintilla
+2991::Conjure One
+2992::De-Phazz
+2993::Corrosion of Conformity
+2994::Virgin Black
+2995::Onslaught
+2996::The Derek Trucks Band
+2997::For the Fallen Dreams
+2998::Sumatra
+2999::Every Time I Die
+3000::Memphis May Fire
+3001::Xentrix
+3002::Times of Grace
+3003::Charlie Musselwhite
+3004::Death Before Dishonor
+3005::Kylesa
+3006::The Locust
+3007::Spectra Paris
+3008::Raunchy
+3009::Element Eighty
+3010::Your Demise
+3011::Spineshank
+3012::Oceana
+3013::Razor
+3014::Beseech
+3015::Imperanon
+3016::An Albatross
+3017::The Human Abstract
+3018::Circle Jerks
+3019::Envy
+3020::United Nations
+3021::Sybreed
+3022::Prodigy
+3023::Sick of It All
+3024::Trap Them
+3025::Walter Trout
+3026::Dead Poetic
+3027::Meg & Dia
+3028::Cephalic Carnage
+3029::Tesseract
+3030::Bonded By Blood
+3031::Casey Jones
+3032::Silent Civilian
+3033::Through the Eyes of the Dead
+3034::HEARTSREVOLUTION
+3035::Amanda Blank
+3036::T.S.O.L.
+3037::AKADO
+3038::Arkaea
+3039::Scanners
+3040::Super Junior
+3041::SHINee
+3042::소녀시대
+3043::BoA
+3044::M. Ward
+3045::Devendra Banhart
+3046::Nick Drake
+3047::Tracy Chapman
+3048::Tilly and the Wall
+3049::The Magnetic Fields
+3050::Jens Lekman
+3051::Neutral Milk Hotel
+3052::The Microphones
+3053::Augustus Pablo
+3054::King Tubby
+3055::Lee "Scratch" Perry
+3056::The Gun Club
+3057::Spiritualized
+3058::The Flying Burrito Brothers
+3059::Loretta Lynn
+3060::Hawkwind
+3061::Leadbelly
+3062::Grandaddy
+3063::Guided by Voices
+3064::Townes Van Zandt
+3065::Red House Painters
+3066::Madredeus
+3067::Mediæval Bæbes
+3068::Her Space Holiday
+3069::Hood
+3070::Ane Brun
+3071::Son House
+3072::Sopor Aeternus & The Ensemble of Shadows
+3073::Parkway Drive
+3074::Avantasia
+3075::You Me At Six
+3076::Mis-Teeq
+3077::Adam Green
+3078::Nina Simone
+3079::Stacie Orrico
+3080::Emma Shapplin
+3081::4minute
+3082::Eazy-E
+3083::Mobb Deep
+3084::Chayanne
+3085::Royce da 5'9"
+3086::Busta Rhymes
+3087::Sara Bareilles
+3088::Lenka
+3089::A Static Lullaby
+3090::Girl Talk
+3091::Spinnerette
+3092::Will Young
+3093::Macy Gray
+3094::This Mortal Coil
+3095::Cold War Kids
+3096::Marit Larsen
+3097::Every Avenue
+3098::Edenbridge
+3099::Lisa Miskovsky
+3100::Krystal Meyers
+3101::Kirlian Camera
+3102::The Organ
+3103::Clannad
+3104::Lisa "Left Eye" Lopes
+3105::Millionaires
+3106::The Casualties
+3107::Evile
+3108::Cadaveria
+3109::Astarte
+3110::Elis
+3111::Jonathan Davis
+3112::Android Lust
+3113::Hillsong United
+3114::The Toy Dolls
+3115::A Place to Bury Strangers
+3116::Crash Test Dummies
+3117::The Wreckers
+3118::Lunatica
+3119::Draconian
+3120::Trail of Tears
+3121::Uh Huh Her
+3122::Planetshakers
+3123::Coal Chamber
+3124::The Submarines
+3125::Mortal Love
+3126::M2M
+3127::Danity Kane
+3128::Doda
+3129::Brodka
+3130::GZA/Genius
+3131::Ghostface Killah
+3132::Natasha Bedingfield
+3133::Aretha Franklin
+3134::James Brown
+3135::Alexandra Burke
+3136::Ewa Farna
+3137::Lady Sovereign
+3138::Dan Balan
+3139::Eve
+3140::Carla Bruni
+3141::Minnie Riperton
+3142::Czesław Śpiewa
+3143::Etta James
+3144::Michael Bublé
+3145::Beady Eye
+3146::Eric B. & Rakim
+3147::Shontelle
+3148::Hey
+3149::Édith Piaf
+3150::Peter Doherty
+3151::Angus & Julia Stone
+3152::Strachy Na Lachy
+3153::Yolanda Be Cool & DCUP
+3154::Paloma Faith
+3155::Stanfour
+3156::Hellogoodbye
+3157::Nosowska
+3158::Ania
+3159::Bow Wow
+3160::Aura Dione
+3161::Cody Simpson
+3162::Natalia Kills
+3163::James Horner
+3164::Lea Michele
+3165::Nneka
+3166::Alexis Jordan
+3167::Tinchy Stryder
+3168::Lauryn Hill
+3169::Mark Ronson & The Business Intl
+3170::Box Car Racer
+3171::Flipsyde
+3172::Soulja Boy
+3173::En Vogue
+3174::Clipse
+3175::Yael Naim
+3176::Flogging Molly
+3177::Smolik
+3178::Ana Johnsson
+3179::Sinéad O'Connor
+3180::Clare Maguire
+3181::Mike Posner
+3182::Justin Nozuka
+3183::T-Pain
+3184::Kaki King
+3185::Vanessa Paradis
+3186::Tom Petty
+3187::Rachael Yamagata
+3188::Måns Zelmerlöw
+3189::Moulin Rouge
+3190::Hope
+3191::Run-D.M.C.
+3192::The Supremes
+3193::Diana Ross and The Supremes
+3194::Hurt
+3195::Pretty Ricky
+3196::Gotan Project
+3197::The Last Goodnight
+3198::Zabili Mi Żółwia
+3199::Cee-Lo
+3200::Cee Lo Green
+3201::Lil Mama
+3202::Anita Baker
+3203::Julian Casablancas
+3204::Poisonblack
+3205::Black Stone Cherry
+3206::Queensberry
+3207::The Blackout
+3208::deadmau5
+3209::Marianas Trench
+3210::Aiden
+3211::Sugarcult
+3212::CKY
+3213::Halestorm
+3214::Emerson, Lake & Palmer
+3215::Kix
+3216::The Saturdays
+3217::Jessie J
+3218::Rebecca Black
+3219::Самое Большое Простое Число
+3220::Red Snapper
+3221::Animal ДжаZ
+3222::Laika
+3223::Killwhitneydead
+3224::Texas in July
+3225::Elio e le Storie Tese
+3226::TV on the Radio
+3227::Blackfield
+3228::Helena Paparizou
+3229::Midge Ure
+3230::Exposé
+3231::X-Dream
+3232::Lights of Euphoria
+3233::Astrud Gilberto
+3234::Athlete
+3235::The Ark
+3236::Alizée
+3237::Huey Lewis & The News
+3238::Morten Harket
+3239::Bad Boys Blue
+3240::Silent Circle
+3241::Fancy
+3242::Astrix
+3243::Chris Spheeris
+3244::عمر دياب
+3245::Cesária Évora
+3246::Supreme Beings of Leisure
+3247::Bread
+3248::Frou Frou
+3249::Ace Ventura
+3250::Guillemots
+3251::Gus Gus
+3252::Boy George
+3253::Jay-Jay Johanson
+3254::Aqualung
+3255::Tahiti 80
+3256::Offer Nissim
+3257::Orient Expressions
+3258::Elegant Machinery
+3259::Mel & Kim
+3260::Bebel Gilberto
+3261::Jody Watley
+3262::Ghosts
+3263::Mumm-Ra
+3264::Bitter:Sweet
+3265::Angtoria
+3266::Terence Trent D'Arby
+3267::Nicola Conte
+3268::Alter Ego
+3269::Samantha Fox
+3270::Anouar Brahem
+3271::Liza Minnelli
+3272::Blue Stone
+3273::Peter Schilling
+3274::Omar Faruk Tekbilek
+3275::Team Sleep
+3276::Up, Bustle and Out
+3277::Parov Stelar
+3278::Asian Dub Foundation
+3279::Chase & Status
+3280::KOTOKO
+3281::安室奈美恵
+3282::モーニング娘。
+3283::大塚愛
+3284::水樹奈々
+3285::林原めぐみ
+3286::北出菜奈
+3287::Zola Jesus
+3288::Blackmore's Night
+3289::Stevie Ray Vaughan
+3290::Rick Wakeman
+3291::Geddy Lee
+3292::Roger Waters
+3293::Van der Graaf Generator
+3294::Focus
+3295::Bachman-Turner Overdrive
+3296::Slade
+3297::Traffic
+3298::Premiata Forneria Marconi
+3299::B.B. King & Eric Clapton
+3300::Neil Young & Crazy Horse
+3301::Inti-Illimani
+3302::Thievery Corporation
+3303::Ornette Coleman
+3304::Michael Gray
+3305::Madvillain
+3306::Sneaker Pimps
+3307::Skye
+3308::Dangerous Muse
+3309::Grizzly Bear
+3310::Martina Topley-Bird
+3311::Smoke City
+3312::Halou
+3313::H.U.V.A. Network
+3314::Aes Dana
+3315::Anja Garbarek
+3316::dZihan & Kamien
+3317::Days of the New
+3318::Shirley Bassey
+3319::Patti LaBelle
+3320::The Field
+3321::Daughter Darling
+3322::Montefiori Cocktail
+3323::Laurent Garnier
+3324::Waldeck
+3325::Rue du Soleil
+3326::Kruder & Dorfmeister
+3327::DeVotchKa
+3328::O Teatro Mágico
+3329::The Tallest Man on Earth
+3330::New Young Pony Club
+3331::Pizzicato Five
+3332::The Dresden Dolls
+3333::Those Dancing Days
+3334::Laura Marling
+3335::Super Furry Animals
+3336::Nightmare
+3337::Rory Gallagher
+3338::Duke Ellington
+3339::St. Vincent
+3340::Brian Wilson
+3341::Owen Pallett
+3342::Jorge Drexler
+3343::The Raveonettes
+3344::Tim Maia
+3345::Janelle Monáe
+3346::Tujiko Noriko
+3347::Vashti Bunyan
+3348::Sons and Daughters
+3349::Nico
+3350::Rasputina
+3351::The Zutons
+3352::Harry and the Potters
+3353::The Ditty Bops
+3354::VHS or Beta
+3355::The Seatbelts
+3356::Louise Attaque
+3357::Au Revoir Simone
+3358::Emmy the Great
+3359::Juli
+3360::Hope Sandoval & The Warm Inventions
+3361::Dum Dum Girls
+3362::Blue Man Group
+3363::Louis XIV
+3364::The Grates
+3365::The Cribs
+3366::The Maccabees
+3367::Envydust
+3368::Head Automatica
+3369::Boy Kill Boy
+3370::The View
+3371::The Futureheads
+3372::Moptop
+3373::Madina Lake
+3374::Juliette and The Licks
+3375::Sirenia
+3376::Kristine Elezaj
+3377::Ticon
+3378::Asura
+3379::Brigitte Bardot
+3380::Hybrid
+3381::Lützenkirchen
+3382::Kiara Rocks
+3383::嵐
+3384::玉置成実
+3385::Grave Digger
+3386::Bonfire
+3387::3 Inches of Blood
+3388::Nazareth
+3389::Sandy Leah
+3390::Mat Kearney
+3391::Nonpoint
+3392::Turbonegro
+3393::Scott Walker
+3394::Gianna Nannini
+3395::Velvet Acid Christ
+3396::David Gray
+3397::Rancid
+3398::The Wallflowers
+3399::Teitur
+3400::Psapp
+3401::Natalie Walker
+3402::Deathstars
+3403::My Life with the Thrill Kill Kult
+3404::Hank Williams Jr.
+3405::Hootie & the Blowfish
+3406::Gov't Mule
+3407::Lene Marlin
+3408::Toadies
+3409::Joy Williams
+3410::Mr. Scruff
+3411::Anna Nalick
+3412::Olivia Ruiz
+3413::Thomas Dybdahl
+3414::Vermillion Lies
+3415::Jerry Lee Lewis
+3416::Charlie Daniels Band
+3417::Blues Traveler
+3418::Jimmy Cliff
+3419::Better Than Ezra
+3420::Vienna Teng
+3421::Bel Canto
+3422::Jonny Lang
+3423::ohGr
+3424::Pigface
+3425::Crosby, Stills & Nash
+3426::Celestial Aeon Project
+3427::Powerman 5000
+3428::Ace Frehley
+3429::Various Artists
+3430::Alcest
+3431::Heaven & Hell
+3432::Tor Lundvall
+3433::All My Faith Lost ...
+3434::Magna Canta
+3435::32Crash
+3436::Jota Quest
+3437::Scars on Broadway
+3438::Upcdowncleftcrightcabc+start
+3439::Banco de Gaia
+3440::I Hear Sirens
+3441::Yonderboi
+3442::Kwoon
+3443::Fuck Buttons
+3444::Destroyalldreamers
+3445::Пелагея
+3446::Rachel's
+3447::Tunturia
+3448::Peter Tosh
+3449::Damian Marley
+3450::Vavamuffin
+3451::Ziggy Marley
+3452::Gentleman
+3453::He Is Legend
+3454::Dance Club Massacre
+3455::Pogo
+3456::Circa Survive
+3457::Kill Paradise
+3458::Tyler, The Creator
+3459::Anna Calvi
+3460::Rev Theory
+3461::The Perishers
+3462::José González
+3463::The Wombats
+3464::Madlib
+3465::Paul Kalkbrenner
+3466::James Blake
+3467::Télépopmusik
+3468::Tesla Boy
+3469::Trentemøller
+3470::Yoav
+3471::Bent
+3472::Sad Lovers and Giants
+3473::The Chameleons
+3474::HEALTH
+3475::Toro y Moi
+3476::SCSI-9
+3477::Zeigeist
+3478::Beach Fossils
+3479::Bad Lieutenant
+3480::Gui Boratto
+3481::Chairlift
+3482::Nathan Fake
+3483::Thieves Like Us
+3484::Kollektiv Turmstrasse
+3485::Extrawelt
+3486::Stephan Bodzin
+3487::Minilogue
+3488::Sascha Funke
+3489::A Camp
+3490::Anthony Rother
+3491::Matthew Dear
+3492::The Mary Onettes
+3493::Erik Satie
+3494::Weather Report
+3495::Nujabes
+3496::Nick Cave & Warren Ellis
+3497::Night Ranger
+3498::Girlicious
+3499::Blitz
+3500::Worm Is Green
+3501::Broken Social Scene
+3502::Great Lake Swimmers
+3503::Grateful Dead
+3504::Anathallo
+3505::Piano Magic
+3506::Do Make Say Think
+3507::Cartola
+3508::The Mountain Goats
+3509::Okkervil River
+3510::Amália Rodrigues
+3511::Celtic Woman
+3512::João Gilberto
+3513::Woods
+3514::Ray LaMontagne
+3515::The Brian Jonestown Massacre
+3516::Streetlight Manifesto
+3517::Ash
+3518::Elis Regina
+3519::Bang Gang
+3520::Baden Powell
+3521::The Radio Dept.
+3522::Stafrænn Hákon
+3523::Owen
+3524::My Morning Jacket
+3525::Amusement Parks on Fire
+3526::Efterklang
+3527::Gray Strawberries
+3528::Omnia
+3529::September Malevolence
+3530::The Meters
+3531::Chapterhouse
+3532::Nara Leão
+3533::Borko
+3534::Electrelane
+3535::Novos Baianos
+3536::Carissa's Wierd
+3537::Under Byen
+3538::Huun-Huur-Tu
+3539::Stan Getz
+3540::Andrei Machado
+3541::Say Hi to Your Mom
+3542::Kitchens of Distinction
+3543::Sun Kil Moon
+3544::Casiotone for the Painfully Alone
+3545::Drive-By Truckers
+3546::Love
+3547::Hermeto Pascoal
+3548::Roger Miller
+3549::Horse Feathers
+3550::Akron/Family
+3551::Tristeza
+3552::A Hawk and a Hacksaw
+3553::Delays
+3554::Rilo Kiley
+3555::Infernal
+3556::dredg
+3557::Trapt
+3558::Billy Currington
+3559::Richard Ashcroft
+3560::From Autumn to Ashes
+3561::Ronan Keating
+3562::Cat Stevens
+3563::Train
+3564::Richie Sambora
+3565::Air Traffic
+3566::Entwine
+3567::Brian McKnight
+3568::Superchic[k]
+3569::Globus
+3570::Semisonic
+3571::Skazi
+3572::Shadow Gallery
+3573::Dark Moor
+3574::Nocturnal Rites
+3575::Lost Horizon
+3576::Evergrey
+3577::TV-2
+3578::Common
+3579::Faith Evans
+3580::Otis Redding
+3581::The Game
+3582::Musiq
+3583::Dizzee Rascal
+3584::Redman
+3585::Utada
+3586::Akufen
+3587::Black Star
+3588::Third Eye Blind
+3589::Jennifer Love Hewitt
+3590::Example
+3591::Coconut Records
+3592::Ginuwine
+3593::Jamie T
+3594::Mario
+3595::112
+3596::Ben Woods
+3597::John Lee Hooker
+3598::Missy Higgins
+3599::Christina Milian
+3600::Lisa Gerrard
+3601::Black Lab
+3602::Pete Yorn
+3603::Ol' Dirty Bastard
+3604::Alkaline Trio
+3605::Pennywise
+3606::Battlelore
+3607::Lucinda Williams
+3608::RZA
+3609::Amber Pacific
+3610::植松伸夫
+3611::Lemon Jelly
+3612::The Blues Brothers
+3613::The Living End
+3614::The J. Geils Band
+3615::Beatsteaks
+3616::Meat Loaf
+3617::Jagged Edge
+3618::Marcy Playground
+3619::Stealers Wheel
+3620::The Click Five
+3621::One Night Only
+3622::Scouting for Girls
+3623::The Crowns
+3624::The Bird and The Bee
+3625::Chiodos
+3626::The Get Up Kids
+3627::The Coasters
+3628::Overkill
+3629::Down
+3630::Nek
+3631::Gipsy Kings
+3632::Ibrahim Ferrer
+3633::Paco de Lucía
+3634::Alejandro Fernández
+3635::Moenia
+3636::Karsh Kale
+3637::Oliver Shanti
+3638::Graham Coxon
+3639::Maino
+3640::Keyshia Cole
+3641::Винтаж
+3642::Booty Luv
+3643::Three 6 Mafia
+3644::Jordan Rudess
+3645::Lunatic Soul
+3646::David Gilmour
+3647::Liquid Tension Experiment
+3648::Al Di Meola
+3649::John Petrucci
+3650::Michael Giacchino
+3651::Django Reinhardt
+3652::Donovan
+3653::Fish
+3654::Fates Warning
+3655::Obscura
+3656::Camel
+3657::Atheist
+3658::Gentle Giant
+3659::David Sylvian
+3660::Nine Horses
+3661::Harry Gregson-Williams
+3662::Beardfish
+3663::Phideaux
+3664::Eloy
+3665::Pestilence
+3666::Kenny G
+3667::Rain Tree Crow
+3668::Goblin
+3669::maudlin of the Well
+3670::Caravan
+3671::Gryphon
+3672::Chroma Key
+3673::Robin Trower
+3674::Chick Corea
+3675::Return to Forever
+3676::Renaissance
+3677::Trey Gunn
+3678::The Rakes
+3679::Dirty Pretty Things
+3680::Milburn
+3681::The Bravery
+3682::Quietdrive
+3683::The Automatic
+3684::The Hoosiers
+3685::Obie Trice
+3686::Bizarre
+3687::Cassiane
+3688::Kim Carnes
+3689::Brenda Lee
+3690::Jack's Mannequin
+3691::Motion City Soundtrack
+3692::The John Butler Trio
+3693::The Cloud Room
+3694::Mark Knopfler
+3695::Haircut 100
+3696::Olivia Newton-John
+3697::Idlewild
+3698::Edwyn Collins
+3699::Orange Juice
+3700::Trio
+3701::Jane Wiedlin
+3702::Passengers
+3703::Andy Taylor
+3704::Nuclear Assault
+3705::Doves
+3706::Hot Hot Heat
+3707::Klaus Badelt
+3708::Kidneythieves
+3709::Grave
+3710::D'espairsRay
+3711::Benjamin Biolay
+3712::Ian Brown
+3713::Eddie Cochran
+3714::Link Wray
+3715::Gary Jules
+3716::Immortal Technique
+3717::Jurassic 5
+3718::Blackalicious
+3719::Bone Thugs-N-Harmony
+3720::Ice Cube
+3721::Cam'ron
+3722::Notorious B.I.G.
+3723::Naughty by Nature
+3724::Proof
+3725::Big L
+3726::Taj Mahal
+3727::Andrew W.K.
+3728::Joe
+3729::Vanilla Ice
+3730::Cap'n Jazz
+3731::Less Than Jake
+3732::Saves the Day
+3733::American Football
+3734::Matt Pond PA
+3735::Karunesh
+3736::Ravi Shankar
+3737::Philip Glass
+3738::Frozen Silence
+3739::Current 93
+3740::Psychic TV
+3741::Ryoji Ikeda
+3742::Zoviet France
+3743::Lustmord
+3744::Alva Noto
+3745::Sergei Rachmaninoff
+3746::Lusine
+3747::Kaya Project
+3748::Tangerine Dream
+3749::Atrium Carceri
+3750::Kammarheit
+3751::Kronos Quartet
+3752::Niyaz
+3753::Steve Roach
+3754::Muslimgauze
+3755::Yat-Kha
+3756::Steve Reich
+3757::Luc Ferrari
+3758::Baskyl
+3759::Modus
+3760::大谷幸
+3761::DJ Food
+3762::Jon Hopkins
+3763::Mythos
+3764::THEreminGIRL
+3765::The Teenagers
+3766::Solar Fields
+3767::The Tear Garden
+3768::Goran Bregović
+3769::Michael Andrews
+3770::Patrick Doyle
+3771::Eric Serra
+3772::Jan Hammer
+3773::Swans
+3774::John Barry
+3775::Dario Marianelli
+3776::Side Liner
+3777::Revolting Cocks
+3778::American Head Charge
+3779::Lords of Acid
+3780::Zeromancer
+3781::Zombina and the Skeletones
+3782::Dog Fashion Disco
+3783::Mortiis
+3784::Sevendust
+3785::Stabbing Westward
+3786::Godhead
+3787::Fun Lovin' Criminals
+3788::Pagoda
+3789::Moonspell
+3790::Gojira
+3791::Balmorhea
+3792::Glen Hansard & Markéta Irglová
+3793::Peace Orchestra
+3794::dEUS
+3795::Diante do Trono
+3796::Carter Burwell
+3797::Dusty Kid
+3798::Modjo
+3799::Nada Surf
+3800::Matia Bazar
+3801::The Glove
+3802::Sex Gang Children
+3803::Altered Images
+3804::The Tornados
+3805::Them Crooked Vultures
+3806::Ólafur Arnalds
+3807::Remy Zero
+3808::Inkubus Sukkubus
+3809::Monty Python
+3810::Steely Dan
+3811::MUM
+3812::Traveling Wilburys
+3813::The Nitty Gritty Dirt Band
+3814::Carly Simon
+3815::This Will Destroy You
+3816::Big Brother & The Holding Company
+3817::Cage the Elephant
+3818::Bill Hicks
+3819::K's Choice
+3820::Little Feat
+3821::Sam & Dave
+3822::Three Dog Night
+3823::Dan Fogelberg
+3824::Poe
+3825::Indigo Girls
+3826::Nikka Costa
+3827::Crosby, Stills, Nash & Young
+3828::And So I Watch You From Afar
+3829::Elysian Fields
+3830::pg.lost
+3831::Maserati
+3832::Dan Auerbach
+3833::Eddie Izzard
+3834::Firefall
+3835::Zwan
+3836::Cine
+3837::Copacabana Club
+3838::E.S. Posthumus
+3839::Of the Wand and the Moon
+3840::Rodrigo y Gabriela
+3841::Bell X1
+3842::Les Savy Fav
+3843::Steve Earle
+3844::Cibo Matto
+3845::Jonny Greenwood
+3846::Susumu Yokota
+3847::Get Cape. Wear Cape. Fly
+3848::Fred Astaire
+3849::Kay Starr
+3850::Jamie Lidell
+3851::Gustavo Santaolalla
+3852::Gnawa Diffusion
+3853::Michelle Branch
+3854::Stephanie McIntosh
+3855::Queen + Paul Rodgers
+3856::Mägo de Oz
+3857::Jill Scott
+3858::The Easybeats
+3859::El Cuarteto de Nos
+3860::Horace Andy
+3861::Os Paralamas do Sucesso
+3862::Plastiscines
+3863::Laurel Aitken
+3864::Chico Science & Nação Zumbi
+3865::The Wailers
+3866::The Warlocks
+3867::Spacemen 3
+3868::Mala Rodríguez
+3869::Aesop Rock
+3870::The Field Mice
+3871::Gustavo Cerati
+3872::Buju Banton
+3873::Glasvegas
+3874::Kinky
+3875::The Rifles
+3876::Gil Scott-Heron
+3877::Elastica
+3878::Paul Weller
+3879::The English Beat
+3880::Estelle
+3881::Toots and the Maytals
+3882::Mansun
+3883::The Feelies
+3884::The Duke Spirit
+3885::Kula Shaker
+3886::Gregory Isaacs
+3887::Burning Spear
+3888::Barrington Levy
+3889::Ike & Tina Turner
+3890::The Tears
+3891::Easy Star All-Stars
+3892::The Thrills
+3893::Bernard Butler
+3894::Free the Robots
+3895::John Mayall & The Bluesbreakers
+3896::Sleeper
+3897::Pharrell
+3898::The Telescopes
+3899::Marianne Faithfull
+3900::The Lucksmiths
+3901::Tindersticks
+3902::Julian Cope
+3903::Pernice Brothers
+3904::You Say Party! We Say Die!
+3905::VETO
+3906::Shout Out Louds
+3907::Curve
+3908::Art Brut
+3909::Love and Rockets
+3910::Fela Kuti
+3911::Windy & Carl
+3912::Grouper
+3913::Dictaphone
+3914::Porn Sword Tobacco
+3915::Ada
+3916::The Herbaliser
+3917::Secos & Molhados
+3918::Neu!
+3919::Pentagram
+3920::Le Orme
+3921::Manfred Mann's Earth Band
+3922::Blue Cheer
+3923::Saint Vitus
+3924::Miike Snow
+3925::Atmosphere
+3926::Roy Ayers
+3927::Sarah Vaughan
+3928::Deerhunter
+3929::Fink
+3930::The Temper Trap
+3931::Maria Gadú
+3932::Seu Jorge
+3933::Zeca Baleiro
+3934::Roberta Sá
+3935::Yuksek
+3936::Djavan
+3937::Wax Poetic
+3938::Elsiane
+3939::Jorge Ben Jor
+3940::Logh
+3941::Donavon Frankenreiter
+3942::The Naked and Famous
+3943::Urge Overkill
+3944::Paulinho Moska
+3945::Mariana Aydar
+3946::3-11 Porter
+3947::Alessandro Safina
+3948::Shaman
+3949::Republica
+3950::Canned Heat
+3951::Natiruts
+3952::Blaze
+3953::Gong
+3954::Derek and the Dominos
+3955::Collective Soul
+3956::Richard Wright
+3957::The Servant
+3958::Ananda Shake
+3959::Library Tapes
+3960::Divine Heresy
+3961::Afghan Whigs
+3962::Bayside
+3963::The Courteeners
+3964::Someone Still Loves You Boris Yeltsin
+3965::The Apples in Stereo
+3966::Final Fantasy
+3967::Delorean
+3968::Wild Nothing
+3969::Set Your Goals
+3970::The Bloody Beetroots
+3971::The Feeling
+3972::Pure Reason Revolution
+3973::Atlas Sound
+3974::Portugal. The Man
+3975::Pnau
+3976::Rotting Christ
+3977::Midnight Juggernauts
+3978::Ampop
+3979::Saltillo
+3980::Bane
+3981::Acceptance
+3982::NOFX
+3983::No Use for a Name
+3984::Tears Run Rings
+3985::Noah and the Whale
+3986::...And You Will Know Us by the Trail of Dead
+3987::Jack Peñate
+3988::Catherine Wheel
+3989::Mute Math
+3990::Goose
+3991::Labradford
+3992::Cloud Cult
+3993::The Secret Handshake
+3994::PlayRadioPlay!
+3995::The Unicorns
+3996::Spleen United
+3997::Polvo
+3998::Scratch Acid
+3999::The Whip
+4000::Zoot Woman
+4001::Go:Audio
+4002::Great Northern
+4003::The Dears
+4004::Hawk Nelson
+4005::Destroy The Runner
+4006::Frankmusik
+4007::Van She
+4008::No Age
+4009::Voxtrot
+4010::Lights Out Asia
+4011::Sleater-Kinney
+4012::We Have Band
+4013::The Monochrome Set
+4014::Mission of Burma
+4015::Bishop Allen
+4016::The Dodos
+4017::Swervedriver
+4018::Edward Sharpe & the Magnetic Zeros
+4019::Steve Aoki
+4020::Thurston Moore
+4021::Plastilina Mosh
+4022::Sleeping at Last
+4023::French Teen Idol
+4024::Hate Eternal
+4025::Sugar Ray
+4026::Film School
+4027::Secret Shine
+4028::Surfer Blood
+4029::The Republic Tigers
+4030::We Smoke Fags
+4031::The Big Pink
+4032::Under the Influence of Giants
+4033::Soundpool
+4034::Paper Route
+4035::Sick Puppies
+4036::BarlowGirl
+4037::Ryan Cabrera
+4038::Akcent
+4039::Vanilla Sky
+4040::Carolina Liar
+4041::Casting Crowns
+4042::Vertical Horizon
+4043::Falling Up
+4044::Orson
+4045::T.M.Revolution
+4046::Pain Confessor
+4047::The Blood Brothers
+4048::Diablo Swing Orchestra
+4049::Satyricon
+4050::Karnivool
+4051::Winds of Plague
+4052::Deathspell Omega
+4053::Anata
+4054::Blut aus Nord
+4055::Closure In Moscow
+4056::近藤浩治
+4057::Koop
+4058::PMMP
+4059::Boys Noize
+4060::Orphaned Land
+4061::Andromeda
+4062::Hanni Kohl
+4063::Steve Vai
+4064::Haggard
+4065::Mors Principium Est
+4066::Café Tacuba
+4067::Jarabe de Palo
+4068::Ark
+4069::Marty Friedman
+4070::Maná
+4071::Ojos de Brujo
+4072::Cacophony
+4073::I'm from Barcelona
+4074::Diablo
+4075::Paul Anka
+4076::Damn Yankees
+4077::Diabolical Masquerade
+4078::Jumbo
+4079::Banco del Mutuo Soccorso
+4080::Sacred Reich
+4081::Septic Flesh
+4082::Legion of the Damned
+4083::Rosetta Stone
+4084::Jane Birkin & Serge Gainsbourg
+4085::Bill Haley
+4086::Bella Morte
+4087::Museum
+4088::The Airborne Toxic Event
+4089::Legowelt
+4090::Robert Plant
+4091::Elmore James
+4092::Sonny Boy Williamson
+4093::Freddie King
+4094::Albert King
+4095::Citizen Cope
+4096::Dr. John
+4097::Otis Rush
+4098::Grant Green
+4099::Junior Wells
+4100::Lightnin' Hopkins
+4101::Koko Taylor
+4102::Cursive
+4103::Ted Leo and The Pharmacists
+4104::Laura Veirs
+4105::La Oreja de Van Gogh
+4106::Fernanda Takai
+4107::As Blood Runs Black
+4108::akissforjersey
+4109::Unearth
+4110::Bury Your Dead
+4111::Sonny Rollins
+4112::Cauterize
+4113::Rufio
+4114::My American Heart
+4115::The Hush Sound
+4116::Andy McKee
+4117::Breathe Electric
+4118::The Audition
+4119::Moneen
+4120::The Bled
+4121::Broadway
+4122::LoveHateHero
+4123::División Minúscula
+4124::Daphne Loves Derby
+4125::KSU
+4126::Four Year Strong
+4127::For Today
+4128::Maroon
+4129::The Agony Scene
+4130::Earth Crisis
+4131::Emarosa
+4132::Morningwood
+4133::Hird
+4134::The Dave Brubeck Quartet
+4135::Gustav Mahler
+4136::Franz Joseph Haydn
+4137::Farid Farjad
+4138::梶浦由記
+4139::Marco Borsato
+4140::Joe Walsh
+4141::Sailor Moon
+4142::Montrose
+4143::Nephew
+4144::Volbeat
+4145::Colin Hay
+4146::John Cale
+4147::Beth Gibbons & Rustin Man
+4148::Bay City Rollers
+4149::Free
+4150::Booker T. & The MG's
+4151::Eddie Money
+4152::Bob Seger
+4153::Gene Vincent
+4154::Peter & Gordon
+4155::John Mellencamp
+4156::Bill Haley and the Comets
+4157::Wanda Jackson
+4158::The Swinging Blue Jeans
+4159::The Cowsills
+4160::Foghat
+4161::Fats Domino
+4162::Harry Belafonte
+4163::Ricky Nelson
+4164::Lesley Gore
+4165::The Jeff Healey Band
+4166::Tommy James & The Shondells
+4167::Little River Band
+4168::The Marshall Tucker Band
+4169::The Lively Ones
+4170::Dion & The Belmonts
+4171::The Music Machine
+4172::Kenny Rogers
+4173::Waylon Jennings
+4174::The Beautiful South
+4175::The Left Banke
+4176::Carl Perkins
+4177::Tennessee Ernie Ford
+4178::Bo Diddley
+4179::Blackfoot
+4180::Heathen
+4181::Forbidden
+4182::Exumer
+4183::Our Lady Peace
+4184::Dishwalla
+4185::Gama Bomb
+4186::Dark Angel
+4187::Joseph Arthur
+4188::Saving Abel
+4189::All About Eve
+4190::The Breeders
+4191::Letzte Instanz
+4192::British Sea Power
+4193::Dead Fish
+4194::Fear Before the March of Flames
+4195::Téléphone
+4196::White Rose Movement
+4197::Milla Jovovich
+4198::Renaud
+4199::Georges Brassens
+4200::Alain Bashung
+4201::Prince & The Revolution
+4202::Prince and the New Power Generation
+4203::Mariza
+4204::Gal Costa
+4205::Ry Cooder
+4206::Francis Cabrel
+4207::2002
+4208::Jane Birkin
+4209::Phil Ochs
+4210::Mitchel Musso
+4211::Esmée Denters
+4212::The Ready Set
+4213::New Boyz
+4214::Stereolab
+4215::Throwing Muses
+4216::Cranes
+4217::Deadlock
+4218::Innerpartysystem
+4219::川井憲次
+4220::Empyrium
+4221::Javier Navarrete
+4222::Rick James
+4223::Bobby Brown
+4224::The Polyphonic Spree
+4225::Chew Lips
+4226::Soulwax
+4227::Scarlett Johansson
+4228::DatA
+4229::2 Many DJ's
+4230::Danger
+4231::Filthy Dukes
+4232::Danny
+4233::Zac Efron
+4234::Polly Scattergood
+4235::Marié Digby
+4236::Ben Lee
+4237::Bethany Joy Lenz
+4238::Tyler Hilton
+4239::Grace Potter and the Nocturnals
+4240::Heather Nova
+4241::Die Antwoord
+4242::Muchy
+4243::Reverend and The Makers
+4244::Wild Beasts
+4245::Dying Fetus
+4246::Trouble
+4247::Summoning
+4248::Autopsy
+4249::Vanilla Ninja
+4250::Moi dix Mois
+4251::Charlie Clouser
+4252::Azam Ali
+4253::My Brightest Diamond
+4254::Shearwater
+4255::Slint
+4256::Vas
+4257::High on Fire
+4258::Olivier Messiaen
+4259::Richard Thompson
+4260::Richard Wagner
+4261::Count Basie
+4262::Benny Goodman
+4263::Shocking Blue
+4264::Jermaine Jackson
+4265::Frida
+4266::Neneh Cherry
+4267::Wendy Carlos
+4268::Lalo Schifrin
+4269::Doris Day
+4270::Billy Preston
+4271::The Waterboys
+4272::Ian McCulloch
+4273::Julee Cruise
+4274::Beth Orton
+4275::Shulman
+4276::Ott
+4277::坂本龍一
+4278::LFO
+4279::Fila Brazillia
+4280::Entheogenic
+4281::Alpha
+4282::Jon Kennedy
+4283::Murcof
+4284::Jega
+4285::Jan Jelinek
+4286::Medwyn Goodall
+4287::Keiko Matsui
+4288::Jim Brickman
+4289::Urban Myth Club
+4290::Moonbootica
+4291::Sven Väth
+4292::Dykehouse
+4293::Dubtribe Sound System
+4294::Deuter
+4295::VFSix
+4296::Raven-Symoné
+4297::Adiemus
+4298::Therapy?
+4299::Can
+4300::Terry Reid
+4301::Secret Service
+4302::Ours
+4303::Krokus
+4304::Cowboy Junkies
+4305::Fountains of Wayne
+4306::Nick Cave
+4307::Del Shannon
+4308::Randy Crawford
+4309::Michael Angelo Batio
+4310::Tokyo Blade
+4311::Bernard Herrmann
+4312::Renan Luce
+4313::The Shangri-Las
+4314::Juliana Hatfield
+4315::John Scofield
+4316::Brad Mehldau
+4317::Nina Hagen
+4318::Lizzy Borden
+4319::Hellyeah
+4320::Blake Lewis
+4321::Jeremy Camp
+4322::Neon Trees
+4323::Flight of the Conchords
+4324::Bushido
+4325::Anders Manga
+4326::Crematory
+4327::Death Angel
+4328::Virgin Steele
+4329::Avalanch
+4330::Farben Lehre
+4331::The Pillows
+4332::Tinariwen
+4333::David Holmes
+4334::Halford
+4335::Billy Corgan
+4336::Stephen Lynch
+4337::Mercedes Sosa
+4338::Peter Broderick
+4339::Cirque du Soleil
+4340::Luis Miguel
+4341::Elbow
+4342::Everclear
+4343::Do As Infinity
+4344::Griffin House
+4345::Qntal
+4346::Bacilos
+4347::Lu
+4348::We Are The Ocean
+4349::This Town Needs Guns
+4350::Immanu El
+4351::Agnes
+4352::The Starting Line
+4353::Salt The Wound
+4354::Nightmare of You
+4355::Say Anything
+4356::Burning Skies
+4357::Still Remains
+4358::Matchbook Romance
+4359::Just Surrender
+4360::The Red Chord
+4361::The Music
+4362::Reggie and the Full Effect
+4363::fun.
+4364::The Damned Things
+4365::As Cities Burn
+4366::Young Galaxy
+4367::autoKratz
+4368::The Echelon Effect
+4369::Decoder
+4370::Roberta Flack
+4371::Pedro the Lion
+4372::Rogue Wave
+4373::Young Knives
+4374::There For Tomorrow
+4375::Seabear
+4376::The Moldy Peaches
+4377::Liquido
+4378::Colour Haze
+4379::Tiger Army
+4380::Nerina Pallot
+4381::Capsule
+4382::鈴木あみ
+4383::Sienna Skies
+4384::3
+4385::Third Day
+4386::El Canto del Loco
+4387::Exilia
+4388::Kiln
+4389::Midlake
+4390::Roots Manuva
+4391::The Bouncing Souls
+4392::Anouk
+4393::Deaf Center
+4394::Aidan Baker
+4395::Jóhann Jóhannsson
+4396::Battles
+4397::Cut Chemist
+4398::Gregor Samsa
+4399::Jaga Jazzist
+4400::The Kilimanjaro Darkjazz Ensemble
+4401::Soldout
+4402::Alex Smoke
+4403::Two Lone Swordsmen
+4404::Gabriel Ananda
+4405::Mira Calix
+4406::Sylvain Chauveau
+4407::Dirty Three
+4408::El-P
+4409::Descendents
+4410::Kingdom
+4411::The Vandals
+4412::Dance of Days
+4413::Milow
+4414::Lloyd
+4415::Marques Houston
+4416::Electric Wizard
+4417::Propagandhi
+4418::Lifelover
+4419::Edge of Sanity
+4420::dälek
+4421::Restart
+4422::Medicine
+4423::Fobia
+4424::Cool Kids of Death
+4425::Strike
+4426::Marjorie Estiano
+4427::Ida Maria
+4428::The Holloways
+4429::oOoOO
+4430::Modern Witch
+4431::Bolt Action Five
+4432::EPMD
+4433::Discharge
+4434::D.R.I.
+4435::7Seconds
+4436::Skip James
+4437::Fear
+4438::Adolescents
+4439::Green River
+4440::Cro-Mags
+4441::Wax Tailor
+4442::The Cooper Temple Clause
+4443::Stina Nordenstam
+4444::Lycia
+4445::Audrey
+4446::Harland
+4447::Mýa
+4448::Secede
+4449::X-Marks the Pedwalk
+4450::Solitary Experiments
+4451::Seefeel
+4452::T. Raumschmiere
+4453::Klinik
+4454::Quantic
+4455::Northern Lite
+4456::Audion
+4457::Underground Resistance
+4458::Jackson and His Computer Band
+4459::GreenGender
+4460::Alex Under
+4461::Panik
+4462::Baroness
+4463::Bert Jansch
+4464::Ashley Roberts
+4465::Cordel do Fogo Encantado
+4466::Teoman
+4467::The Dubliners
+4468::Darvin
+4469::The Walkmen
+4470::The Black Angels
+4471::The Tea Party
+4472::¡Forward, Russia!
+4473::Carl Orff
+4474::Georges Bizet
+4475::Yasmin Levy
+4476::Peter Brötzmann
+4477::Mulatu Astatke
+4478::Ali Farka Touré & Toumani Diabaté
+4479::Erkan Oğur
+4480::Irfan
+4481::Joe Henderson
+4482::Coleman Hawkins
+4483::Wayne Shorter
+4484::Zoë Keating
+4485::Serkan Süleymaniye
+4486::Mercan Dede
+4487::Mark Owen
+4488::Jesus Jones
+4489::White Town
+4490::Françoise Hardy
+4491::Tim Urban
+4492::Gladys Knight & The Pips
+4493::Ben E. King
+4494::Eve 6
+4495::Edwin McCain
+4496::Montell Jordan
+4497::mewithoutYou
+4498::Copeland
+4499::45 Grave
+4500::4hero
+4501::808 State
+4502::Feeder
+4503::James Dean Bradfield
+4504::Louis Prima
+4505::Gorefest
+4506::Big Mama Thornton
+4507::The Hooters
+4508::Dar Williams
+4509::Mates of State
+4510::Ben Frost
+4511::Oficina G3
+4512::NOMAK
+4513::Swollen Members
+4514::Jimmy Reed
+4515::Billy Squier
+4516::Julie London
+4517::The Gaslight Anthem
+4518::Nicole Atkins
+4519::Sugarplum Fairy
+4520::Little Man Tate
+4521::New Radicals
+4522::Chantal Kreviazuk
+4523::Jackson Browne
+4524::BBMak
+4525::Lisa Loeb
+4526::Wim Mertens
+4527::Morton Feldman
+4528::Pink Turns Blue
+4529::Steve Bug
+4530::Deee-Lite
+4531::Scout Niblett
+4532::I Monster
+4533::Boozoo Bajou
+4534::Maps And Diagrams
+4535::Paris Combo
+4536::Hildur Guðnadóttir
+4537::Joe McElderry
+4538::Britt Nicole
+4539::Hillsong
+4540::Chris & Cosey
+4541::Telepathe
+4542::Dima Bilan
+4543::Luis Fonsi
+4544::Dionne Bromfield
+4545::Alex Ubago
+4546::Franz Schubert
+4547::Michael Nyman
+4548::Nâdiya
+4549::Fernanda Brum
+4550::Twin Shadow
+4551::Machine Drum
+4552::Mest
+4553::The Hold Steady
+4554::Cannonball Adderley
+4555::Platinum Blonde
+4556::Ned's Atomic Dustbin
+4557::Moist
+4558::久石譲
+4559::Alpinestars
+4560::Jonathan Coulton
+4561::Carter the Unstoppable Sex Machine
+4562::Martha and the Muffins
+4563::Kenickie
+4564::Magnet
+4565::Giorgio Moroder
+4566::Catatonia
+4567::Pete Townshend
+4568::Charlotte Martin
+4569::Brooke Fraser
+4570::György Ligeti
+4571::Zbigniew Preisner
+4572::Belleruche
+4573::Molotov
+4574::Jawbreaker
+4575::Gomez
+4576::Youssou N'Dour
+4577::Nileppez
+4578::Keith Moon
+4579::Lulu
+4580::Joan as Police Woman
+4581::Dezerter
+4582::O. Children
+4583::Gevende
+4584::Bülent Ortaçgil
+4585::Feridun Düzağaç
+4586::Be Your Own Pet
+4587::A.C. Newman
+4588::Zooey Deschanel
+4589::Shantel
+4590::Global Deejays
+4591::Wonder Girls
+4592::Brazilian Girls
+4593::Drexciya
+4594::Coeur de Pirate
+4595::SoKo
+4596::Nine Black Alps
+4597::The Sunshine Underground
+4598::Gridlock
+4599::Sparta
+4600::Elefant
+4601::Fantasia
+4602::Fikret Kızılok
+4603::Astrobrite
+4604::Darren Criss
+4605::Abney Park

data/ref/lastfm/train_data.df

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7db71a60d3da4c449091fbc254efc9602f81eec6eb9a60534811ccd61974dbc1
+size 2462846

data/ref/movielens/Test_data.df

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d38308a0078d4518dc9eeb619c8d1afb53d830712cc7d36ac51cc47eaafc97d6
+size 43718

data/ref/movielens/Val_data.df

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a950900d0055649e2983ef403d189d5fdc5bd4f84d229a377806d371b32e460
+size 43200

data/ref/movielens/train_data.df

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bac4c88b11e6f76d914c2b22aadbb97315a78432bb79f424f7015775053b9546
+size 9633075

data/ref/steam/Test_data.df

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2766b9faf90e3531a35207c01fbef2f655af97c21a6ba7d18581bbef2bc6ad59
+size 59142

data/ref/steam/id2name.txt

@@ -0,0 +1,3581 @@
+0::Ironbound
+1::Real Pool 3D - Poolians
+2::Half-Life
+3::The Ship: Murder Party
+4::DEFCON
+5::The Ship: Single Player
+6::Garry's Mod
+7::Gumboy - Crazy Adventures™
+8::Vegas: Make It Big™
+9::Civilization IV®: Warlords
+10::X-COM: Terror From the Deep
+11::Tomb Raider: Anniversary
+12::Final DOOM
+13::Master Levels for Doom II
+14::DOOM II
+15::Wolfenstein 3D
+16::QUAKE Mission Pack 1: Scourge of Armagon
+17::QUAKE Mission Pack 2: Dissolution of Eternity
+18::Genesis Rising
+19::Warhammer® 40,000: Dawn of War® - Dark Crusade
+20::Command & Conquer: Red Alert 3
+21::Prison Tycoon 3™: Lockdown
+22::18 Wheels of Steel: American Long Haul
+23::Tycoon City: New York
+24::Death to Spies
+25::Unreal 2: The Awakening
+26::Silent Hunter® III
+27::Luxor 3
+28::Sprint Cars Road to Knoxville
+29::Ride! Carnival Tycoon
+30::Eternity's Child
+31::X-COM: Enforcer
+32::X-COM: Apocalypse
+33::S.T.A.L.K.E.R.: Clear Sky
+34::The Witcher: Enhanced Edition Director's Cut
+35::Multiwinia
+36::Sid Meier's Civilization IV: Colonization
+37::Brothers in Arms: Hell's Highway™
+38::Xpand Rally Xtreme
+39::Heroes of Might & Magic V: Hammers of Fate
+40::Brothers in Arms: Earned in Blood™
+41::The Settlers®: Rise Of An Empire Gold Edition
+42::Bully: Scholarship Edition
+43::Saints Row 2
+44::Rise of the Argonauts
+45::Knights of Honor
+46::Unreal Tournament 3 Black
+47::The Graveyard
+48::Burnout Paradise: The Ultimate Box
+49::The Maw
+50::Puzzle Kingdoms
+51::X-Blades
+52::Zeno Clash
+53::King's Bounty: The Legend
+54::Death Track®: Resurrection
+55::Freedom Force
+56::Darkest Hour: Europe '44-'45
+57::Prototype™
+58::The Elder Scrolls III: Morrowind® Game of the Year Edition
+59::Armored Fist 3
+60::BRAINPIPE: A Plunge to Unhumanity
+61::Hunting Unlimited 2010
+62::Tales of Monkey Island Complete Pack
+63::Armed and Dangerous®
+64::The Secret of Monkey Island: Special Edition
+65::TimeShift™
+66::Bionic Commando
+67::Osmos
+68::Fallout 2: A Post Nuclear Role Playing Game
+69::Crash Time 2
+70::World of Zoo
+71::Windosill
+72::Machinarium
+73::Need for Speed: Shift
+74::Command & Conquer: Red Alert 3 - Uprising
+75::Painkiller: Resurrection
+76::SpellForce 2 - Anniversary Edition
+77::Rush for Berlin Gold
+78::Hammerfight
+79::SpellForce - Platinum Edition
+80::Secret Files 2: Puritas Cordis
+81::Dragon Age: Origins
+82::Aztaka
+83::LEGO® Star Wars™ - The Complete Saga
+84::Command & Conquer 3: Kane's Wrath
+85::Wings of Prey
+86::Fallout 3: Game of the Year Edition
+87::Roogoo
+88::Pirates, Vikings, and Knights II
+89::Super Laser  Racer
+90::Mount & Blade: Warband
+91::M.U.D. TV
+92::The Misadventures of P.B. Winterbottom
+93::Iron Grip: Warlord
+94::Flight of the Icarus
+95::Ziro
+96::SEGA Mega Drive and Genesis Classics
+97::Alpha Protocol™
+98::Kane & Lynch 2: Dog Days
+99::Puzzle Quest 2
+100::Burn Zombie Burn!
+101::Victoria I Complete
+102::Arma 2: British Armed Forces
+103::Chime
+104::Swarm Arena
+105::Serious Sam Classic: The Second Encounter
+106::X: Tension
+107::Governor of Poker 2 - Premium Edition
+108::Grotesque Tactics: Evil Heroes
+109::Doc Clock: The Toasted Sandwich of Time
+110::Puzzle Bots
+111::Nimbus
+112::Flight Control HD
+113::Pound of Ground
+114::Pat & Mat
+115::Alternativa
+116::Call of Duty®: Black Ops
+117::Arcadia
+118::Oddworld: Stranger's Wrath HD
+119::Grand Theft Auto III
+120::A.R.E.S.: Extinction Agenda
+121::Dead Space™ 2
+122::The Sims™ 3
+123::YOU DON’T KNOW JACK®
+124::DUNGEONS - Steam Special Edition
+125::BIT.TRIP RUNNER
+126::Dinner Date
+127::Majesty 2 Collection
+128::A.V.A. Alliance of Valiant Arms™
+129::Spiral Knights
+130::Solar 2
+131::F.E.A.R. 3
+132::Dungeons and Dragons: Daggerdale
+133::Nancy Drew®: Secrets Can Kill REMASTERED
+134::Nancy Drew®: The Captive Curse
+135::EDGE
+136::Puzzle Pirates
+137::Warhammer 40,000: Space Marine
+138::Men of War: Vietnam
+139::Wasteland Angel
+140::Nancy Drew®: Shadow at the Water's Edge
+141::Bunch of Heroes
+142::The Binding of Isaac
+143::Off-Road Drive
+144::Dead Rising 2: Off the Record
+145::Memoir '44 Online
+146::Serious Sam: The Random Encounter
+147::Puzzler World 2
+148::Airline Tycoon 2
+149::The Haunted: Hells Reach
+150::DC Universe™ Online
+151::Saints Row: The Third Initiation Station
+152::Fractal: Make Blooms Not War
+153::Galactic Civilizations® II: Ultimate Edition
+154::Earth Defense Force: Insect Armageddon
+155::X3: Albion Prelude
+156::Trine 2: Complete Story
+157::Oil Rush
+158::King Arthur II: The Role-Playing Wargame
+159::Fortune Summoners
+160::Star Trek Online
+161::Jagged Alliance - Back in Action
+162::Titan Attacks!
+163::Alan Wake
+164::Realm of the Mad God
+165::Painkiller: Recurring Evil
+166::Stacking
+167::Wizorb
+168::Myst V
+169::Yesterday
+170::Midnight Mysteries 4: Haunted Houdini
+171::Ridge Racer™ Unbounded
+172::Imagine Earth
+173::Scary Girl
+174::Binary Domain
+175::Analogue: A Hate Story
+176::Toy Soldiers
+177::Resident Evil: Operation Raccoon City
+178::Ticket to Ride
+179::The Lord of the Rings Online™
+180::Indie Game: The Movie
+181::Krater
+182::The Dark Eye: Chains of Satinav
+183::Tom Clancy's Ghost Recon: Future Soldier™
+184::Quantum Conundrum
+185::Tribes: Ascend
+186::Source Filmmaker
+187::eXceed 2nd - Vampire REX
+188::eXceed 3rd - Jade Penetrate Black Package
+189::Death Rally
+190::Hero Academy
+191::Iron Brigade
+192::Galaxy on Fire 2™ Full HD
+193::Sleeping Dogs
+194::Din's Curse
+195::Batman: Arkham City - Game of the Year Edition
+196::To the Moon
+197::Intrusion 2
+198::Carrier Command: Gaea Mission
+199::3D-Coat V4.8
+200::Cortex Command
+201::Rocketbirds: Hardboiled Chicken
+202::Doom 3: BFG Edition
+203::A Game of Dwarves
+204::Hotline Miami
+205::Guns of Icarus Online
+206::POSTAL 2
+207::F1 RACE STARS™
+208::Sine Mora
+209::Stealth Bastard Deluxe
+210::The Book of Unwritten Tales: The Critter Chronicles
+211::Mabinogi
+212::Triple Town
+213::Legacy of Kain: Defiance
+214::Iron Sky: Invasion
+215::Angelica Weaver: Catch Me When You Can
+216::Fieldrunners 2
+217::Ace Combat Assault Horizon - Enhanced Edition
+218::Retrovirus
+219::Cities XL Platinum
+220::Super House of Dead Ninjas
+221::Highborn
+222::Tomb Raider - The Final Hours Digital Book
+223::Monster Loves You!
+224::Alien Spidy
+225::POSTAL
+226::Cut the Rope
+227::Anodyne
+228::Another World – 20th Anniversary Edition
+229::Sang-Froid - Tales of Werewolves
+230::Cubemen 2
+231::Surgeon Simulator
+232::Poker Night 2
+233::StarDrive
+234::Sanctum 2
+235::Reus
+236::Resident Evil Revelations / Biohazard Revelations
+237::Remember Me
+238::Penny Arcade's On the Rain-Slick Precipice of Darkness 4
+239::Mortal Kombat Komplete Edition
+240::Soldier Front 2
+241::TrackMania² Valley
+242::FINAL FANTASY VII
+243::BIT.TRIP FATE
+244::Ittle Dew
+245::CastleStorm
+246::Skulls of the Shogun
+247::Rise of the Triad
+248::Cloudberry Kingdom™
+249::Inquisitor
+250::Worms Clan Wars
+251::Electronic Super Joy
+252::PixelJunk™ Monsters Ultimate
+253::Lilly Looking Through
+254::America's Army: Proving Grounds
+255::Game Dev Tycoon
+256::Delver
+257::MirrorMoon EP
+258::Castle of Illusion
+259::Ironclad Tactics
+260::Foul Play
+261::Scribblenauts Unmasked: A DC Comics Adventure
+262::Farming Simulator 2013 Titanium Edition
+263::SNOW
+264::Soundodger+
+265::Iesabel
+266::Knights and Merchants
+267::Eleusis
+268::Bridge Constructor
+269::Bad Hotel
+270::140
+271::Anachronox
+272::Knytt Underground
+273::Deus Ex: Human Revolution - Director's Cut
+274::Contagion
+275::Blood of the Werewolf
+276::Master Reboot
+277::Alone in the Dark
+278::Deadly Premonition: The Director's Cut
+279::State of Decay
+280::Long Live The Queen
+281::Rain Blood Chronicles: Mirage
+282::Vox
+283::Kingdoms Rise
+284::Ring Runner: Flight of the Sages
+285::9.03m
+286::The Mysterious Cities of Gold
+287::The Last Express Gold Edition
+288::Finding Teddy
+289::Dark Fall 2: Lights Out
+290::Dark Fall: The Journal
+291::LogoMaker 4
+292::Broken Sword 5 - the Serpent's Curse
+293::Darkout
+294::Dominions 4: Thrones of Ascension
+295::Wooden Sen'SeY
+296::Shufflepuck Cantina Deluxe
+297::King’s Bounty: Legions
+298::FATE
+299::RaySupreme 3D
+300::Cold War
+301::CONSORTIUM
+302::Nidhogg
+303::Galcon Legends
+304::Humans Must Answer
+305::Estranged: Act I
+306::Paper Sorcerer
+307::Hero Siege
+308::The Castle Doctrine
+309::Saturday Morning RPG
+310::Serena
+311::Octodad: Dadliest Catch
+312::Aqua Kitty - Milk Mine Defender
+313::Sentinel 3: Homeworld
+314::Aces Wild: Manic Brawling Action!
+315::GTGD S1: More Than a Gamer
+316::BlazBlue: Calamity Trigger
+317::Disciples III: Reincarnation
+318::The Plan
+319::LocoCycle
+320::Girls Like Robots
+321::METAL SLUG 3
+322::Card City Nights
+323::Ikaruga
+324::Hexcells
+325::Epic Battle Fantasy 4
+326::Assassin's Creed Freedom Cry
+327::Castlevania: Lords of Shadow 2
+328::Towtruck Simulator 2015
+329::Dominique Pamplemousse
+330::1954 Alcatraz
+331::Constant C
+332::Deus Ex: The Fall
+333::LUFTRAUSERS
+334::Danmaku Unlimited 2
+335::Heroine's Quest: The Herald of Ragnarok
+336::Quest of Dungeons
+337::Black Mirror II
+338::Abyss: The Wraiths of Eden
+339::The Tomorrow War
+340::El Matador
+341::Marauder
+342::Harvester
+343::Evolution RTS
+344::House of 1,000 Doors: Family Secrets Collector's Edition
+345::Volt
+346::Secrets of Rætikon
+347::Putt-Putt® Joins the Parade
+348::Heldric - The legend of the shoemaker
+349::Abalone
+350::UFO: Aftermath
+351::RC Cars
+352::Ascension to the Throne
+353::FarSky
+354::Depths of Fear :: Knossos
+355::Nostradamus: The Last Prophecy
+356::Vangers
+357::Street Racing Syndicate
+358::Clockwork Tales: Of Glass and Ink
+359::Rogue Shooter: The FPS Roguelike
+360::Brigade E5: New Jagged Union
+361::7,62 High Calibre
+362::Black Rainbow
+363::BloodRayne Betrayal
+364::Sweezy Gunner
+365::Indie Graphics Bundle - Royalty Free Sprites
+366::The Whispered World Special Edition
+367::Descent: FreeSpace – The Great War
+368::Coldfire Keep
+369::Fantasy Grounds
+370::Chronology
+371::The Last Tinker™: City of Colors
+372::Wildlife Park 3
+373::Rhiannon: Curse of the Four Branches
+374::Fearless Fantasy
+375::Orc Attack: Flatulent Rebellion
+376::Spy Fox 3 "Operation Ozone"
+377::Pajama Sam 3: You Are What You Eat From Your Head To Your Feet
+378::Rage Runner
+379::Pretentious Game
+380::Legionwood 2: Rise of the Eternal's Realm - Director's Cut
+381::DRAKERZ-Confrontation
+382::Paradigm Shift
+383::Retro Game Crunch
+384::I Am Vegend - Zombiegeddon
+385::A.I.M.2 Clan Wars
+386::Numba Deluxe
+387::CAPSULE
+388::Defense Zone 2
+389::GearCity
+390::The Fall
+391::The Campaign Series: Fall Weiss
+392::Dysan the Shapeshifter
+393::Enigmatis 2: The Mists of Ravenwood
+394::Richard & Alice
+395::Lifeless Planet Premier Edition
+396::Magicite
+397::Castle: Never Judge a Book by its Cover
+398::Echo of the Wilds
+399::Reversion - The Escape (1st Chapter)
+400::Enemy Front
+401::Faery - Legends of Avalon
+402::A Wizard's Lizard
+403::Bot Colony
+404::Stick RPG 2: Director's Cut
+405::Heroes Rise: The Hero Project
+406::Beyond Space Remastered Edition
+407::MotoGP™14
+408::Spoiler Alert
+409::Monsters Ate My Birthday Cake
+410::Claire
+411::Deponia: The Complete Journey
+412::House of 1000 Doors: The Palm of Zoroaster Collector's Edition
+413::Namariel Legends: Iron Lord Premium Edition
+414::Evil Pumpkin: The Lost Halloween
+415::Rooms: The Main Building
+416::Necronomicon: The Dawning of Darkness
+417::Pivvot
+418::Steel & Steam: Episode 1
+419::Scooby Doo! & Looney Tunes Cartoon Universe: Adventure
+420::Wildlife Park 2
+421::Wildlife Park 2 - Fantasy
+422::Wildlife Park 2 - Dino World
+423::Wildlife Park 2 - Horses
+424::Pixel Hunter
+425::Detective Case and Clown Bot in: Murder in the Hotel Lisbon
+426::Pressured
+427::Toxic Bunny HD
+428::Ghostship Aftermath
+429::Lantern Forge
+430::Sokobond
+431::Data Hacker: Initiation
+432::My Ex-Boyfriend the Space Tyrant
+433::Dungeon Defenders Eternity
+434::Third Eye Crime
+435::Machines At War 3
+436::Vertical Drop Heroes HD
+437::Crazy Plant Shop
+438::The Tower
+439::Road Not Taken
+440::War on Folvos
+441::Wyv and Keep: The Temple of the Lost Idol
+442::Another Perspective
+443::Mega Coin Squad
+444::Heavy Fire: Afghanistan
+445::Pixeluvo
+446::Platypus II
+447::Advanced Tactics Gold
+448::Platypus
+449::Nux
+450::Heroes & Legends: Conquerors of Kolhar
+451::Shadowgate (2014)
+452::Curse: The Eye of Isis
+453::Deep Under the Sky
+454::Amerzone: The Explorer’s Legacy
+455::The Journey Down: Chapter Two
+456::How to Survive
+457::One Day For Ched
+458::The Flying Dutchman
+459::Fable Anniversary
+460::Jacob Jones and the Bigfoot Mystery : Episode 1
+461::The Collider
+462::Color Symphony
+463::My Lands: Black Gem Hunting
+464::WAKFU
+465::Pro Rugby Manager 2015
+466::Enola
+467::Wasteland 2: Director's Cut
+468::Cho Dengeki Stryker All Ages Version
+469::Magicmaker
+470::Middle-earth™: Shadow of Mordor™
+471::Front Page Sports Football
+472::Left in the Dark: No One on Board
+473::METAL SLUG X
+474::Moto Racer Collection
+475::Disney Princess: My Fairytale Adventure
+476::Metal Dead
+477::Styx: Master of Shadows
+478::Crow
+479::Costume Quest 2
+480::Tron 2.0
+481::Sigils of Elohim
+482::The Evil Within
+483::Enigmatis: The Ghosts of Maple Creek
+484::The Treasures of Montezuma 3
+485::Time Rifters
+486::Screencheat
+487::Vampires: Guide Them to Safety!
+488::Sign Motion
+489::Sproggiwood
+490::Enforcer: Police Crime Action
+491::REVOLVER360 RE:ACTOR
+492::The Stalin Subway: Red Veil
+493::Daedalus - No Escape
+494::The Moon Sliver
+495::Miscreated
+496::The Interactive Adventures of Dog Mendonça & Pizzaboy®
+497::Alea Jacta Est
+498::The Binding of Isaac: Rebirth
+499::Broken Sword 4 - the Angel of Death
+500::Command HQ
+501::Eradicator
+502::Winged Sakura: Mindy's Arc
+503::Pitiri 1977
+504::Gunspell - Steam Edition
+505::Gold Rush! Anniversary
+506::Deadlings: Rotten Edition
+507::Toybox Turbos
+508::Amphora
+509::Randal's Monday
+510::Ski-World Simulator
+511::The Blue Flamingo
+512::Adventure Time: The Secret Of The Nameless Kingdom
+513::WARMACHINE: Tactics
+514::Letter Quest: Grimm's Journey
+515::Death Skid Marks
+516::AppGameKit: Easy Game Development
+517::Luna: Shattered Hearts: Episode 1
+518::Brink of Consciousness: The Lonely Hearts Murders
+519::Craft The World
+520::If My Heart Had Wings
+521::Shroud of the Avatar: Forsaken Virtues
+522::Haunted House: Cryptic Graves
+523::Tesla Breaks the World!
+524::Joe Dever's Lone Wolf HD Remastered
+525::Mining Industry Simulator
+526::Galcon 2: Galactic Conquest
+527::iO
+528::Bet On Soldier
+529::Elegy for a Dead World
+530::JUJU
+531::FINAL FANTASY® XIII-2
+532::SunAge: Battle for Elysium
+533::Town of Salem
+534::Dead Effect
+535::BLOCKADE 3D
+536::Shadows of War
+537::Mechs & Mercs: Black Talons
+538::Squirreltopia
+539::Disorder
+540::TRISTOY
+541::Crystal Catacombs
+542::Witch's Pranks: Frog's Fortune Collector's Edition
+543::HuniePop
+544::Cat Goes Fishing
+545::Resident Evil / biohazard HD REMASTER
+546::PARTICLE MACE
+547::Among Ripples
+548::Stranded Deep
+549::Guild Commander
+550::Venetica - Gold Edition
+551::Tulpa
+552::The Lady
+553::Overture
+554::Terra Incognita ~ Chapter One: The Descendant
+555::Astray
+556::Forsaken Isle
+557::Barter Empire
+558::Pixel Heroes: Byte & Magic
+559::Wickland
+560::Plazma Being
+561::Samudai
+562::Dark Forester
+563::Spirit Run - Fire vs. Ice
+564::Damned Nation Reborn
+565::Barbarian Brawl
+566::On A Roll 3D
+567::Hotline Miami 2: Wrong Number Digital Comic
+568::Odysseus: Long Way Home
+569::Night Shift
+570::Over 9000 Zombies!
+571::Oddworld: New 'n' Tasty
+572::DRAGON BALL XENOVERSE
+573::DYNASTY WARRIORS® 8 Empires
+574::White Night
+575::Tallowmere
+576::Damnation City of Death
+577::Mighty Dungeons
+578::Marble Age
+579::ARMED SEVEN
+580::Shelter 2
+581::Drizzlepath
+582::Automation - The Car Company Tycoon Game
+583::Sid Meier's Starships
+584::Spirit of War
+585::Bermuda
+586::Oceanhorn: Monster of Uncharted Seas
+587::Woolfe - The Red Hood Diaries
+588::Trapped Dead: Lockdown
+589::Incognito
+590::Factions: Origins of Malu
+591::12 Labours of Hercules
+592::Walkover
+593::Exowar
+594::Dungeon Highway
+595::Hospital Manager
+596::Red Lake
+597::Spirits of Xanadu
+598::Claws & Feathers
+599::Grim Legends 2: Song of the Dark Swan
+600::An Octave Higher
+601::Pixel Puzzles 2: Anime
+602::RIDE
+603::Jones On Fire
+604::Deadly Sin
+605::Half-Life 2: Update
+606::A Pixel Story
+607::Attack of the Labyrinth +
+608::AdVenture Capitalist
+609::Bloodsports.TV
+610::Hare In The Hat
+611::The Defenders: The Second Wave
+612::gravilon
+613::Grass Simulator
+614::Astronaut Simulator
+615::Adventures of Bertram Fiddle: Episode 1: A Dreadly Business
+616::Copa Petrobras de Marcas
+617::Defend Your Life: TD
+618::Dead Synchronicity: Tomorrow Comes Today
+619::Hero Generations
+620::Love And Order
+621::The Undying Plague
+622::Westerado: Double Barreled
+623::Telepath Tactics
+624::House of Caravan
+625::Highlands
+626::Airport Simulator 2015
+627::Parcel
+628::Environmental Station Alpha
+629::Blue Rose
+630::Crest - an indirect god sim
+631::Mind Snares: Alice's Journey
+632::Crypt of the NecroDancer
+633::Crypt of the NecroDancer Extended Soundtrack
+634::Hypt
+635::The Lost Battalion: All Out Warfare
+636::Breakout Invaders
+637::RPG Maker 2003
+638::Ys VI: The Ark of Napishtim
+639::The Music Machine
+640::Action Henk
+641::Treeker: The Lost Glasses
+642::Galactic Civilizations III
+643::Trainz: A New Era
+644::Luna's Wandering Stars
+645::Age of Castles: Warlords
+646::Dustbowl
+647::NEON STRUCT
+648::Spaceman Sparkles 2
+649::Porcunipine
+650::Teddy Terror
+651::OBEY
+652::True Bliss
+653::The Incredible Adventures of Van Helsing III
+654::Wyrmsun
+655::Teddy Floppy Ear - The Race
+656::A Bastard's Tale
+657::AKIBA'S TRIP: Undead ＆ Undressed
+658::Homesick
+659::Sleep Attack
+660::Blender Game Asset Creation
+661::BLADESTORM: Nightmare
+662::Alter World
+663::Waste Walkers
+664::The Silent Age
+665::Tiamat X
+666::stratO
+667::MASSIVE CHALICE
+668::Sirius Online
+669::Soccer Rage
+670::ZombieRun
+671::You Must Build A Boat
+672::Duck Game
+673::Chicken Invaders 3
+674::Alone in the Dark: Illumination™
+675::Super Star Path
+676::Koala Kids
+677::Nomad
+678::CDF Ghostship
+679::Age of Fear: The Undead King
+680::Lost Lands: Dark Overlord
+681::Action Alien
+682::Spy Bugs
+683::Scarab Tales
+684::Prismatica
+685::Farm Frenzy: Hurricane Season
+686::The Amber Throne
+687::oO
+688::Lilly and Sasha: Guardian Angels
+689::FaceRig
+690::Once Bitten, Twice Dead!
+691::Hello Kitty and Sanrio Friends Racing
+692::City of Fools
+693::Trap Them
+694::Girlfriend Rescue
+695::Chime Sharp
+696::RC Mini Racers
+697::Squarelands
+698::CroNix
+699::Super Dungeon Run
+700::Cosmic Rocket Defender
+701::Lethal RPG: War
+702::Aberoth
+703::Super Hipster Lumberjack
+704::"Glow Ball" - The billiard puzzle game
+705::Dungeon League
+706::Microcosmum: survival of cells
+707::Project Druid - 2D Labyrinth Explorer-
+708::Siege Wars
+709::Hydraulic Empire
+710::MANOS
+711::Gunjitsu
+712::Dream Chamber
+713::SAGA
+714::Empyrion - Galactic Survival
+715::Celestian Tales: Old North
+716::Royal Bounty HD
+717::Wanderlust Adventures
+718::Blaster Shooter GunGuy!
+719::Gravity Error
+720::Skyrim Script Extender (SKSE)
+721::Hacknet
+722::Angry Arrows
+723::Silver Creek Falls: Chapter 1
+724::C-Wars
+725::Three Digits
+726::Curses 'N Chaos
+727::Planetary Annihilation: TITANS
+728::Big Thinkers Kindergarten
+729::Reign of Bullets
+730::The Settlers Online
+731::Race Track Builder
+732::Pure Hold'em
+733::Afterlife Empire
+734::Inevitability
+735::Teeworlds
+736::Planet of the Eyes
+737::ANKI
+738::Sound Shift
+739::Kult: Heretic Kingdoms
+740::Giana Sisters: Dream Runners
+741::RFLEX
+742::Fran Bow
+743::Onikira - Demon Killer
+744::The Black Watchmen
+745::Dead In Bermuda
+746::Playing History 2 - Slave Trade
+747::Calvino Noir
+748::One Piece Pirate Warriors 3
+749::Exile's End
+750::Mad Max: Fury Road
+751::Painters Guild
+752::NOBUNAGA'S AMBITION: Sphere of Influence
+753::iZBOT
+754::Gridberd
+755::Master Spy
+756::MadOut Ice Storm
+757::EasyAntiCheat eSports
+758::Ravenmark: Scourge of Estellion
+759::Queen's Quest: Tower of Darkness
+760::Superstatic
+761::Aerannis
+762::Arcane Sorcery
+763::The Archetype
+764::Three Heroes
+765::Battle of the Bulge
+766::Train Simulator
+767::Star Command Galaxies
+768::Splendor
+769::TY the Tasmanian Tiger 4
+770::Metal Reaper Online
+771::CodeSpells
+772::Airport Madness: Time Machine
+773::Forsaken Fortress Strategy
+774::Templar Battleforce
+775::Doodle God
+776::Lost Lands: A Hidden Object Adventure
+777::Toto Temple Deluxe
+778::Armikrog
+779::A.I. Invasion
+780::Game Corp DX
+781::Super Sky Arena
+782::Hidden Object Bundle 4 in 1
+783::Slipstream 5000
+784::Normality
+785::Zenohell
+786::Downwell
+787::Mushroom 11
+788::War of Beach
+789::Warhammer 40,000: Deathwatch - Enhanced Edition
+790::Zombie Grinder
+791::Lonath Online
+792::Dungeon Manager ZV
+793::Overlord: Fellowship of Evil
+794::Kingdom: Classic
+795::Noct
+796::Armor Clash
+797::Divine Slice of Life
+798::Outpost 13
+799::A Wolf in Autumn
+800::Divinity: Original Sin - Enhanced Edition
+801::Statues
+802::Infinite Space III: Sea of Stars
+803::The Last Crown: Midnight Horror
+804::Indie Game Battle
+805::Voidspire Tactics
+806::The Last NightMary - A Lenda do Cabeça de Cuia
+807::Anno 2205™
+808::Space - The Return Of The Pixxelfrazzer
+809::Your Quest
+810::Swordbreaker The Game
+811::The Incredible Adventures of Van Helsing: Final Cut
+812::Krosmaster Arena
+813::Garden Rescue: Christmas Edition
+814::Dispatcher
+815::Dota 2 Player Profiles
+816::Showing Tonight: Mindhunters Incident
+817::Uriel's Chasm 2: את
+818::True Lover's Knot
+819::Bitardia
+820::Osteya
+821::Gender Bender DNA Twister Extreme
+822::Jurassic Island: The Dinosaur Zoo
+823::One More Dungeon
+824::RC Simulation 2.0
+825::Strania - The Stella Machina -
+826::FIM Speedway Grand Prix 15
+827::Lowglow
+828::Reveal The Deep
+829::Wild Season
+830::GUILTY GEAR Xrd -SIGN-
+831::MechWarrior Online™
+832::OH! RPG!
+833::Gabe Newell Simulator 2.0
+834::Hatoful Boyfriend: Holiday Star
+835::Turnover
+836::Gigachess
+837::Taimumari
+838::Super Slam Dunk Touchdown
+839::Waves 2
+840::River City Super Sports Challenge ~All Stars Special~
+841::The Withering
+842::The fall of gods
+843::Swapperoo
+844::Our Nation's Miner
+845::What's under your blanket !?
+846::Squirbs
+847::Chiptune Champion
+848::That Dragon, Cancer
+849::Eight Mini Racers
+850::LoveBeat
+851::Pro Basketball Manager 2016
+852::Zombie Wars: Invasion
+853::BATTLE PIXELS
+854::Kivi, Toilet and Shotgun
+855::Dragon's Dogma: Dark Arisen
+856::Astro Lords: Oort Cloud
+857::No Turning Back: The Pixel Art Action-Adventure Roguelike
+858::Scrap Mechanic
+859::World's Dawn
+860::Knight Online
+861::Incredible Dracula: Chasing Love Collector's Edition
+862::Apocalypse Hotel - The Post-Apocalyptic Hotel Simulator!
+863::LEGO® MARVEL's Avengers
+864::Midnight
+865::Time of Dragons
+866::Spellweaver
+867::Dead6hot
+868::Pocket Rumble
+869::Tales of Symphonia
+870::Horror in the Asylum
+871::Tap Tap Legions - Epic battles within 5 seconds!
+872::Soccer Manager
+873::SnakEscape
+874::Doors
+875::Defragmented
+876::Break Chance Memento
+877::Jumpix Jump
+878::Super Helmets on Fire DX Ultra Edition Plus Alpha
+879::My Name is Mayo
+880::Marmoset Hexels 3
+881::Army of Tentacles: (Not) A Cthulhu Dating Sim
+882::We Know the Devil
+883::Layers of Fear
+884::Street Fighter V
+885::Gurgamoth
+886::Find Out
+887::Story Of the Survivor
+888::Lamia's Game Room
+889::String Theory
+890::CABAL Online
+891::No Pineapple Left Behind
+892::Super Night Riders
+893::Astro Duel
+894::NeonXSZ
+895::OutDrive
+896::ALONE IN SPACE
+897::Just Death
+898::Shadwen
+899::The Kindred
+900::Atlantic Fleet
+901::Cat on a Diet
+902::Lucy -The Eternity She Wished For-
+903::Balrum
+904::Banzai Escape
+905::Age of Gladiators
+906::Trigger Runners
+907::New York Taxi Simulator
+908::CUPID - A free to play Visual Novel
+909::Mazement
+910::Wild Animal Racing
+911::Tom Clancy’s The Division™
+912::Mind Zero
+913::From Earth
+914::Alekhine's Gun
+915::Unknown Battle
+916::Hidden Object 6-in-1 bundle
+917::101 Ways to Die
+918::Metal Assault
+919::Cubicolor
+920::Forgotten Myths CCG
+921::Borstal
+922::Vortex: The Gateway
+923::Atari Vault
+924::Paws: A Shelter 2 Game
+925::Villagers
+926::SAMOLIOTIK
+927::At the Mountains of Madness
+928::Super Robot Jump Jump
+929::Christmas Adventure: Candy Storm
+930::Super Arcade Football
+931::Anna's Quest
+932::NO THING
+933::Jet Set Knights
+934::Automata Empire
+935::S2ENGINE HD
+936::Warden: Melody of the Undergrowth
+937::Judgment: Apocalypse Survival Simulation
+938::Twilight Struggle
+939::Welkin Road
+940::Massive
+941::METAL SLUG 2
+942::Pang Adventures
+943::Imperia Online
+944::Gahkthun of the Golden Lightning Steam Edition
+945::Spaceport Hope
+946::Muddy Heights® 2
+947::ARCADE GAME SERIES: DIG DUG
+948::ARCADE GAME SERIES: Ms. PAC-MAN
+949::Klabi
+950::Save the Dodos
+951::Drusilla Dreams
+952::Colours of Magic: Aqua Teeter
+953::Ghoul Kid
+954::Cornerstone: The Song of Tyrim
+955::Endorlight
+956::The Panic Room
+957::Abandoned Knight
+958::Back in 1995
+959::N.E.R.O.: Nothing Ever Remains Obscure
+960::Cubium Dreams
+961::Mr Nibbles Forever
+962::Puzzle Galaxies
+963::Zombillie
+964::The Pit And The Pendulum
+965::Aselia the Eternal -The Spirit of Eternity Sword-
+966::Fragments of Him
+967::Share
+968::Shadow Complex Remastered
+969::Parkitect
+970::Neon Drive
+971::Princess Isabella: The Rise of an Heir
+972::Goliath
+973::The House in Fata Morgana
+974::Autumn
+975::SMASHING THE BATTLE
+976::Evo Explores
+977::SHOCK TROOPERS
+978::Romopolis
+979::htoL#NiQ: The Firefly Diary / htoL#NiQ-ホタルノニッキ-
+980::Bitardia Cards: Memes of 2ch
+981::GROOVY
+982::Survive in Space
+983::METAGAL
+984::Soulcaster: Part I & II
+985::Grandpa's Table
+986::The Mahjong Huntress
+987::Warriors' Wrath
+988::Hyper Bounce Blast
+989::KNIGHTS
+990::The Way of Life Free Edition
+991::The NADI Project
+992::Magma Tsunami
+993::Summer Sale
+994::Investigator
+995::Anima Gate of Memories
+996::Home Behind
+997::A Healer Only Lives Twice
+998::Rescue Lucy
+999::Impossible Quest
+1000::Sudoku Quest
+1001::Ghost 1.0
+1002::Death Goat
+1003::GemBreak
+1004::OESE
+1005::CrazyCars3D
+1006::Hover 2030
+1007::Vilmonic
+1008::Twisted Worlds
+1009::Mahjong Destiny
+1010::MAZE LORD
+1011::XSplit
+1012::Farnham Fables
+1013::PAC-MAN 256
+1014::ChuSingura46+1 S
+1015::Affairs of the Court: Choice of Romance
+1016::Chicken Assassin - Master of Humiliation
+1017::Earth Space Colonies
+1018::Vinyl
+1019::Tomoyo After ~It's a Wonderful Life~ English Edition
+1020::RWBY: Grimm Eclipse
+1021::The Lion's Song: Episode 1 - Silence
+1022::Goblins and Grottos
+1023::Radical Spectrum: Volume 1
+1024::Fossil Echo
+1025::Replica
+1026::Tomato Jones
+1027::UNDER NIGHT IN-BIRTH Exe:Late
+1028::Muv-Luv
+1029::Alteric
+1030::RimWorld
+1031::Empty Horizons
+1032::Mahjong Deluxe 2: Astral Planes
+1033::BrainBread 2
+1034::Red Rope: Don't Fall Behind
+1035::Fabulous Food Truck
+1036::DreamBreak
+1037::Intruder Alert: Ixian Operations
+1038::Starbound
+1039::Inverted
+1040::Lovely Planet Arcade
+1041::Legends of the Universe - StarCore
+1042::Marvel: Ultimate Alliance 2
+1043::Strange Night
+1044::Ratz Instagib
+1045::The Return Home
+1046::Totally Unbalanced
+1047::Intrude
+1048::SurvHive
+1049::Snow Horse
+1050::Pixel Cup Soccer 17
+1051::Mibibli's Quest
+1052::The Cat! Porfirio's Adventure
+1053::Little King's Story
+1054::Aircraft War X
+1055::Conclusion
+1056::BlackSmith HIT
+1057::Street Legal Racing: Redline v2.3.1
+1058::Music Maker 2017 Premium Steam Edition
+1059::Bear With Me
+1060::Existentia
+1061::Heart&Slash
+1062::Blade Ballet
+1063::Winter Novel
+1064::FORTIFY
+1065::Ray Gigant
+1066::One Night Two Crazies
+1067::Pixel Puzzles 2: RADical ROACH
+1068::Cubway
+1069::Typoman
+1070::Grow Up
+1071::Monsters and Monocles
+1072::ACE Academy
+1073::Deus Ex: Mankind Divided
+1074::AdVenture Communist
+1075::Automobilista
+1076::SMILE GAME BUILDER
+1077::Panzer Warfare
+1078::STANDBY
+1079::Optika
+1080::Tribal Pass
+1081::Fright Light
+1082::SHINRAI - Broken Beyond Despair
+1083::Caesar™ IV
+1084::Quest for Glory 1-5
+1085::Caesar™ 3
+1086::Four Sided Fantasy
+1087::Hidden Object - 12 in 1 bundle
+1088::Agent Walker: Secret Journey
+1089::Gochi-Show! -How To Learn Japanese Cooking Game-
+1090::Football Mogul 15
+1091::Police Tactics: Imperio
+1092::Halcyon 6: Starbase Commander
+1093::The Tale of Doris and the Dragon - Episode 1
+1094::Flat Heroes
+1095::Black Sand Drift
+1096::Stronghold Legends: Steam Edition
+1097::Trials of Azra
+1098::Rustangelo
+1099::Paladins®
+1100::Woodle Tree 2: Worlds
+1101::Minimized
+1102::Bad ass babes
+1103::Wheels of Aurelia
+1104::Zenith
+1105::NBA 2K17
+1106::Heart's Medicine - Time to Heal
+1107::Pixel-Warfare: Pro
+1108::Fractured Space
+1109::Sorcery! Part 4
+1110::Dog Sled Saga
+1111::Pavilion
+1112::EARTHLOCK: Festival of Magic
+1113::Sig.NULL
+1114::Clustertruck
+1115::Anykey Simulator
+1116::Bloody Walls
+1117::Hardware Engineering
+1118::Particle Fleet: Emergence
+1119::NEON Ultra
+1120::Recursed
+1121::MegaTagmension Blanc + Neptune VS Zombies (Neptunia)
+1122::Zup!
+1123::Platro
+1124::Industry Manager: Future Technologies
+1125::Scrap Garden - The Day Before
+1126::Major\Minor - Complete Edition
+1127::Hidden Dimensions 3
+1128::Ballistic Protection
+1129::BitMaster
+1130::Rubek
+1131::Feel The Snow
+1132::Stay Close
+1133::High Octane Drift
+1134::Heroes & Generals
+1135::The Whisperer in Darkness
+1136::True Fear: Forsaken Souls
+1137::The Infinite Black
+1138::Russian SuperHero Dead Ivan
+1139::Bad Caterpillar
+1140::Party Saboteurs
+1141::PAYDAY 2: John Wick Weapon Pack
+1142::Sid Meier’s Civilization® VI
+1143::Drift GEAR Racing Free
+1144::Hired Ops
+1145::Sky Break
+1146::Kim
+1147::BossConstructor
+1148::Ginger: Beyond the Crystal
+1149::Halloween Forever
+1150::Slayaway Camp
+1151::Grab the Bottle
+1152::That's Mahjong!
+1153::Equin: The Lantern
+1154::PHAT PHROG
+1155::The Elder Scrolls V: Skyrim Special Edition
+1156::NSFW ~ Not a Simulator For Working
+1157::Miniature - The Story Puzzle
+1158::Balloon Blowout
+1159::GO AWAY, THERE'S KUMIS OVER THERE!
+1160::Zombie Exodus: Safe Haven
+1161::Memory's Dogma CODE:01
+1162::Vanguards
+1163::Anime Studio Simulator
+1164::Trimmer Tycoon
+1165::Bye-Bye, Wacky Planet
+1166::Soda Girls
+1167::Planet Explorers
+1168::Goblin and Coins
+1169::World of Tanks Blitz
+1170::Sethian
+1171::Silence
+1172::SPINGUN
+1173::Chimpact 1 - Chuck's Adventure
+1174::Don't open the doors!
+1175::The Sandbox Evolution - Craft a 2D Pixel Universe!
+1176::Creeper World 2: Anniversary Edition
+1177::ICEY
+1178::Zone4
+1179::Club Naughty
+1180::Shuffle!
+1181::Rugby Union Team Manager 2017
+1182::InfinitasDM
+1183::Secret Santa
+1184::Counter Agents
+1185::Dungeon Souls
+1186::Beastiarium
+1187::Drop Alive
+1188::GAROU: MARK OF THE WOLVES
+1189::ZombieCarz
+1190::Root Of Evil: The Tailor
+1191::Dream Quest
+1192::The Dreamlord
+1193::BIOS
+1194::Glittermitten Grove
+1195::8-Bit Invaders!
+1196::Blue Tear
+1197::Moekuri: Adorable + Tactical SRPG
+1198::Art of War: Red Tides
+1199::Stern Pinball Arcade
+1200::Polygon Attack
+1201::Duke of Alpha Centauri
+1202::Trick and Treat - Visual Novel
+1203::Lily's Day Off
+1204::Last Answer
+1205::Power Hover
+1206::Bad Dream: Coma
+1207::Laraan
+1208::Illyriad - Grand Strategy MMO
+1209::I Am The Hero
+1210::Galaxy Admirals
+1211::STARDROP
+1212::Commands & Colors: The Great War
+1213::Imperium Galactica II
+1214::Crab Dub
+1215::Sticker Craft
+1216::A Crashlands Story: Dev Diary
+1217::SpiritSphere
+1218::Tiles & Tales
+1219::PERFECT ANGLE: The puzzle game based on optical illusions
+1220::Felix Jumpman
+1221::Guild Quest
+1222::Warcube
+1223::Nyheim
+1224::SQUAKE
+1225::Fergus The Fly
+1226::A House of Many Doors
+1227::MOBIUS FINAL FANTASY™
+1228::Off-Road Paradise: Trial 4x4
+1229::Rock God Tycoon
+1230::BOOR
+1231::Battle Islands: Commanders
+1232::Fairy Fencer F Advent Dark Force | フェアリーフェンサー エフ ADVENT DARK FORCE | 妖精劍士 F ADVENT DARK FORCE
+1233::Copoka
+1234::Mall Empire
+1235::Golf It!
+1236::The Curse Of Yendor
+1237::Zup! 4
+1238::Spoids
+1239::Cavern Escape
+1240::bob's game
+1241::Night in the Woods
+1242::Unexplored
+1243::Anarchy Online
+1244::Open Sorcery
+1245::Shift Happens
+1246::Northgard
+1247::The Exiled
+1248::Seasteader
+1249::Blackwake
+1250::The Inner Darkness
+1251::Furious Angels
+1252::For The King
+1253::Turn Around
+1254::VThree
+1255::Ino
+1256::Zoop! - Hunter's Grimm
+1257::Hurricane
+1258::ULTIMATE MARVEL VS. CAPCOM 3
+1259::SimAirport
+1260::Star Merchant
+1261::Disc Jam
+1262::Lightspeed Frontier
+1263::Atelier Firis: The Alchemist and the Mysterious Journey / フィリスのアトリエ ～不思議な旅の錬金術士～
+1264::LEGO® Worlds
+1265::Reflex Arena
+1266::Maximum Override
+1267::Good Archer
+1268::Blink
+1269::Lifeline
+1270::WARTILE
+1271::Crisis in the Kremlin
+1272::Toukiden 2
+1273::Splody
+1274::Flagster
+1275::Monster Slayers
+1276::Infinite Tanks
+1277::Microgons
+1278::Ballistic Overkill
+1279::NIGHTSTAR: Rogue Wings
+1280::Free! - Iwatobi Swim Club
+1281::Mighty Party
+1282::State of Anarchy: Master of Mayhem
+1283::Idle Evolution
+1284::AFL Evolution
+1285::Cannons-Defenders: Steam Edition
+1286::GIBZ
+1287::Kith - Tales from the Fractured Plateaus
+1288::Slime-san
+1289::Fausts Alptraum
+1290::Happy Campers
+1291::The Signal From Tölva
+1292::Bayonetta
+1293::Aaero
+1294::A Rose in the Twilight / ロゼと黄昏の古城
+1295::Cosmic Star Heroine
+1296::Skyforge
+1297::The Wild Eternal
+1298::Full Throttle Remastered
+1299::Odyssey - The Next Generation Science Game
+1300::Pixel Sand
+1301::Brawlout
+1302::Academagia: The Making of Mages
+1303::Storm Riders
+1304::Hearthlands
+1305::Agatha Knife
+1306::Attack Heroes
+1307::TumbleSeed
+1308::Dawn of Andromeda
+1309::Welcome to Moreytown
+1310::Fur Fun
+1311::Chronicle of Innsmouth
+1312::Birthdays the Beginning / バースデイズ・ザ・ビギニング
+1313::Dead Cells
+1314::Market Tycoon
+1315::STEAM HAMMER
+1316::Empathy: Path of Whispers
+1317::Bounty Train
+1318::Rabbit Story
+1319::Arcfall
+1320::Bokida - Heartfelt Reunion
+1321::Mages of Mystralia
+1322::Portal Knights
+1323::The Infectious Madness of Doctor Dekker
+1324::Skylar & Plux: Adventure On Clover Island
+1325::A Hole New World
+1326::The Superfluous
+1327::StarCrawlers
+1328::Starpoint Gemini Warlords
+1329::50 years
+1330::Zombidle : REMONSTERED
+1331::Rising Storm 2: Vietnam
+1332::Formula Fusion
+1333::Shotgun Legend
+1334::TEKKEN 7
+1335::Blitzkrieg 3
+1336::Monolith
+1337::Bunker 58
+1338::Lines
+1339::Flood of Light
+1340::Rocket Wars
+1341::Pro Cycling Manager 2017
+1342::Orake 2D MMORPG
+1343::Zafehouse Diaries 2
+1344::Nex Machina
+1345::Libra of the Vampire Princess
+1346::Saga of Tanya the Evil
+1347::Ticket to Earth
+1348::Micro Machines World Series
+1349::Zooicide
+1350::Three Twenty One
+1351::Monsters' Den: Godfall
+1352::Still Not Dead
+1353::qop
+1354::Antisphere
+1355::Crash Force®
+1356::Block Survival: Legend of the Lost Islands
+1357::Hard Reset
+1358::Hand Simulator
+1359::Tangledeep
+1360::Just Cause™ 3: Multiplayer Mod
+1361::PWND
+1362::Book Of Potentia 2
+1363::Shadow Bug
+1364::Fable Fortune
+1365::Airmen
+1366::Onirim - Solitaire Card Game
+1367::Vostok Inc.
+1368::Stream Avatars
+1369::The Low Road
+1370::SUDOKU
+1371::Crossout
+1372::Sky Knights
+1373::Call of Duty®: Modern Warfare® Remastered
+1374::SolarGun
+1375::Legends of Ellaria
+1376::Capitalism 2
+1377::Alchemyland
+1378::Nash Racing
+1379::The Legend of Heroes: Trails of Cold Steel
+1380::Saurian
+1381::Juanito Arcade Mayhem
+1382::THE KING OF FIGHTERS: DESTINY
+1383::Cash Crop
+1384::Achievement Hunter: Spinner Edition
+1385::Draw Puzzle
+1386::MINDNIGHT
+1387::The Shrouded Isle
+1388::CUBOTS The Origins
+1389::Cat Quest
+1390::Mega Man Legacy Collection 2 / ロックマン クラシックス コレクション 2
+1391::Sine Mora EX
+1392::Lost Technology
+1393::Gangs of Space
+1394::Looterkings
+1395::Fragmented
+1396::Drive Isle
+1397::Lumber King
+1398::Discord Bot Maker
+1399::White Day: A Labyrinth Named School
+1400::Yet Another Zombie Defense HD
+1401::BROKE PROTOCOL: Online City RPG
+1402::Island Dash
+1403::Pixel Traffic: Circle Rush
+1404::Epic Cards Battle 2-Dragons Rising(TCG)
+1405::Greedy Guns
+1406::Ways of History
+1407::Doodle God Blitz
+1408::Find this!
+1409::Reaching for Petals
+1410::Kitten Rampage
+1411::SLI-FI: 2D Planet Platformer
+1412::Brawl of Ages
+1413::The Ultimatest Battle
+1414::Trackless
+1415::Hand of the Gods
+1416::Epic Tavern
+1417::PRO EVOLUTION SOCCER 2018
+1418::Oriental Empires
+1419::Rover Builder
+1420::Jettomero: Hero of the Universe
+1421::ARENA an Age of Barbarians story
+1422::STELLATUM
+1423::World of Castles
+1424::ATOMEGA
+1425::The Caribbean Sail
+1426::Super Fancy Pants Adventure
+1427::Arrow Heads
+1428::Auto Age: Standoff
+1429::Heat Signature
+1430::Figment
+1431::DwarfCorp
+1432::Road Dogs
+1433::About Elise
+1434::The Painscreek Killings
+1435::Killer Instinct
+1436::Over The Moonlight
+1437::RollerCoaster Tycoon® Classic
+1438::Vaporum
+1439::Airport CEO
+1440::Stick Fight: The Game
+1441::JYDGE
+1442::SORE
+1443::Mushroom Wars 2
+1444::Cubic
+1445::Wunderdoktor
+1446::Gold Rush: The Game
+1447::Yorkshire Gubbins
+1448::Party Hard Tycoon
+1449::Bomber Crew
+1450::Tank Force
+1451::Debris
+1452::Battlevoid: Sector Siege
+1453::Scream Collector
+1454::Forever Home
+1455::JumpSky
+1456::Path Out
+1457::FURIDASHI: Drift Cyber Sport
+1458::Warspear Online
+1459::Mini Guns
+1460::Star Story: The Horizon Escape
+1461::Team Four Star RPG
+1462::A Wonder
+1463::LEGO® Marvel Super Heroes 2
+1464::Deep Sky Derelicts
+1465::Door Kickers: Action Squad
+1466::CarX Drift Racing Online
+1467::Remaya Idle
+1468::The Mammoth: A Cave Painting
+1469::Red Crucible®: Reloaded
+1470::AirMech Wastelands
+1471::Dominions 5 - Warriors of the Faith
+1472::Ylands
+1473::汉匈决战/Gloria Sinica: Han Xiongnu Wars
+1474::SpellForce 3
+1475::Gunlock
+1476::Umineko: Golden Fantasia
+1477::I wanna be The Cat
+1478::Gorogoa
+1479::War Planet Online: Global Conquest
+1480::Modern Combat Versus
+1481::Golden Hornet
+1482::PLAYERUNKN1WN: Friendly Fire
+1483::Wooden Battles
+1484::Flora
+1485::Fruity Smoothie
+1486::X-Morph: Defense
+1487::Long Live Santa!
+1488::Decent Icons
+1489::Dinosaurs A Prehistoric Adventure
+1490::Gauntlet of IRE
+1491::Ashes Cricket
+1492::Finding Paradise
+1493::Farmer's Dynasty
+1494::Rumu
+1495::Zup! 7
+1496::OKAMI HD / 大神 絶景版
+1497::March of Empires
+1498::Space Wars: Interstellar Empires
+1499::Super Turbo Demon Busters!
+1500::Shadowhand
+1501::SUPERHOT: MIND CONTROL DELETE
+1502::Getting Over It with Bennett Foddy
+1503::Cookies vs. Claus
+1504::Nocturnal Hunt
+1505::Bare Metal
+1506::Jay Fighter: Remastered
+1507::Cobalt WASD
+1508::La Tale - Evolved
+1509::Carcassonne - Tiles & Tactics
+1510::BattleRush
+1511::My Free Zoo
+1512::Wolfenstein II: The New Colossus
+1513::Base Defense
+1514::Battle Chef Brigade
+1515::Yu-Gi-Oh! Duel Links
+1516::東方天空璋 ～ Hidden Star in Four Seasons.
+1517::Star Traders: Frontiers
+1518::Tactical Monsters Rumble Arena
+1519::Football Manager Touch 2018
+1520::Gladiator School
+1521::Oik 3
+1522::Superflight
+1523::Hand of Fate 2
+1524::True or False Universe
+1525::Need For Drink
+1526::One Strike
+1527::I Am Overburdened
+1528::Hollowed
+1529::The Norwood Suite
+1530::ШП
+1531::Zwei: The Ilvard Insurrection
+1532::Lines by Nestor Yavorskyy
+1533::Codename CURE
+1534::LSD
+1535::Dresden Files Cooperative Card Game
+1536::Coffee Run
+1537::Lost Dimension
+1538::Flightless
+1539::Black Clover
+1540::SIMULACRA
+1541::AER Memories of Old
+1542::ClickRaid
+1543::Capsa
+1544::Real Farm
+1545::Molten Armor
+1546::The Inner World - The Last Wind Monk
+1547::Beyond the Void
+1548::Warbanners
+1549::Freaky Awesome
+1550::Rex: Another Island
+1551::WWE 2K18
+1552::Brawlhalla
+1553::South Park™: The Fractured But Whole™
+1554::Mercury Fallen
+1555::Robot King Part I: Rebooted and Ready
+1556::Dungeons 3
+1557::Diamo XL
+1558::Dragon Glory
+1559::Middle-earth™: Shadow of War™
+1560::Incline
+1561::Rainbow Snake
+1562::Press X to Not Die
+1563::Battle Chasers: Nightwar
+1564::Divided We Fall
+1565::Castle Clicker : Building Tycoon
+1566::Throne of Lies® The Online Game of Deceit
+1567::Bermuda - Lost Survival
+1568::Kritika Online
+1569::Total War: WARHAMMER II
+1570::The Guild 3
+1571::Hob
+1572::Project Nimbus
+1573::Pinball FX3
+1574::Niche - a genetics survival game
+1575::Holdfast: Nations At War
+1576::Ancient Frontier
+1577::HumanKind: The Awakening
+1578::Age of Gladiators II
+1579::openCanvas 7
+1580::Tricolour Lovestory
+1581::Blast Out
+1582::Another Lost Phone: Laura's Story
+1583::Super Hydorah
+1584::Simulator hipstera 2k17
+1585::Lone Warrior
+1586::HIVESWAP: Act 1
+1587::Tooth and Tail
+1588::FPV Drone Simulator
+1589::Fjong
+1590::Academia : School Simulator
+1591::Insidia
+1592::Stash
+1593::Find You
+1594::Darkestville Castle
+1595::Card City Nights 2
+1596::Madu Maths
+1597::Fear For Freedom
+1598::On a Roll
+1599::1bitHeart
+1600::Far Space Halloween edition
+1601::Loading
+1602::Robocraft
+1603::Alphabear: Hardcover Edition
+1604::The Escapists 2
+1605::Achievement Hunter: Darkness
+1606::Blue Horizon
+1607::CastleMiner Warfare
+1608::Moero Chronicle | 限界凸記 モエロクロニクル | 極限凸記　萌萌編年史
+1609::Startup Company
+1610::Team Racing League
+1611::Halcyon 6: Starbase Commander (LIGHTSPEED EDITION)
+1612::Super POTUS Trump
+1613::Achievement Hunter: Overdose
+1614::Code 7
+1615::Spears 'n' Spades
+1616::LawBreakers
+1617::Hellblade: Senua's Sacrifice
+1618::Strategy & Tactics: Dark Ages
+1619::Quest Hunter
+1620::The Hunting God
+1621::Machine World 2
+1622::Black Squad
+1623::Fighting Fantasy Legends
+1624::Tricone Lab
+1625::Fate/EXTELLA
+1626::Iron Tides
+1627::Cursed Treasure 2
+1628::Planet Ancyra Chronicles
+1629::Solar Settlers
+1630::Dreamfall Chapters
+1631::Pastry Lovers
+1632::Strike Vector EX
+1633::Aporia: Beyond The Valley
+1634::Kingdoms and Castles
+1635::Slash It Ultimate
+1636::Dead Purge: Outbreak
+1637::Star Trek Timelines
+1638::Wild Guns Reloaded
+1639::Minecraft: Story Mode - Season Two
+1640::Omegaland
+1641::Unturned
+1642::Visual Novel Engine
+1643::√Letter - Root Letter -
+1644::Bomb Defense
+1645::Cricket Captain 2017
+1646::Baobabs Mausoleum Ep.1: Ovnifagos Don´t Eat Flamingos
+1647::SAELIG
+1648::PLANETS OF WAR
+1649::Kimulator 2: The Bottle Flip Master
+1650::TAP TOUCH RUN
+1651::龙魂时刻
+1652::Your Smile Beyond Twilight:黄昏下的月台上
+1653::b
+1654::welcome to heaven
+1655::Miss Kobayashi's Dragon Maid
+1656::Gabriel Dropout
+1657::Kemono Friends
+1658::3..2..1..Grenades!
+1659::Astro Boy: Edge of Time
+1660::SQUIDS FROM SPACE
+1661::Art Of Gravity
+1662::TinkerQuarry
+1663::Formicide
+1664::DiRT 4
+1665::Pixel Worlds
+1666::Crystal City
+1667::Strategic Command WWII: War in Europe
+1668::Passpartout: The Starving Artist
+1669::Conarium
+1670::Battlesloths 2025: The Great Pizza Wars
+1671::UltraGoodness
+1672::After The End: The Harvest
+1673::Might & Mayhem
+1674::Lydia
+1675::Tokyo 42
+1676::Company of Heroes: Eastern Front
+1677::Shotgun Farmers
+1678::The Land of Pain
+1679::XField Paintball 3
+1680::Don't cut your hand
+1681::First Strike: Final Hour
+1682::Balthazar's Dream
+1683::VEGAS Movie Studio 14 Steam Edition
+1684::Flying Tigers: Shadows Over China
+1685::Green Cat
+1686::Life is Feudal: Forest Village
+1687::Detective Butler: Maiden Voyage Murder
+1688::MidBoss
+1689::Quarantine
+1690::Oafmatch
+1691::You Are God
+1692::ROKH
+1693::Old Man's Journey
+1694::Regalia: Of Men and Monarchs
+1695::Andarilho
+1696::Bombinator
+1697::Blockle
+1698::Run Away
+1699::SPACEPLAN
+1700::ATOMINE
+1701::Last Dream: World Unknown
+1702::Blossoms Bloom Brightest
+1703::Cublast HD
+1704::NEXT JUMP: Shmup Tactics
+1705::Arcane Mapper
+1706::Jidousha Shakai
+1707::What Remains of Edith Finch
+1708::Games of Glory
+1709::Ruin of the Reckless
+1710::Cuit
+1711::Deformers
+1712::Everything
+1713::Orcs Must Die! Unchained
+1714::Animal Rivals
+1715::ONRAID
+1716::Life Beetle
+1717::Kingdom of Loot
+1718::My Name is You
+1719::iStorm
+1720::ZRoll
+1721::Tuebor: I Will Defend
+1722::Bulletstorm: Full Clip Edition
+1723::The Quest for Achievements
+1724::Warstone TD
+1725::The Franz Kafka Videogame
+1726::Golf for Workgroups
+1727::The Great Whale Road
+1728::Berserk
+1729::Mob Psycho 100
+1730::Ghost In The Shell: Stand Alone Complex
+1731::One Eyed Kutkh
+1732::The Eagle's Heir
+1733::HOUND
+1734::Fallout Shelter
+1735::Fences
+1736::Life Forge ORPG
+1737::News Tycoon
+1738::Battle Brothers
+1739::Weapon Shop Fantasy
+1740::Bush Hockey League
+1741::Dungeon Creepster
+1742::Molemen Must Die!
+1743::Tower Defense - Fantasy Legends Tower Game
+1744::Factory Engineer
+1745::The Crow's Eye
+1746::Running Sausage
+1747::The Tenth Line
+1748::Macbat 64
+1749::Antagonist
+1750::Kona
+1751::Viktor, a Steampunk Adventure
+1752::NieR:Automata™
+1753::MyWorld - Action RPG Maker
+1754::Cosmic Express
+1755::Virtual Rides 3 - Funfair Simulator
+1756::The Keep
+1757::64.0
+1758::Clone Drone in the Danger Zone
+1759::Introvert Quest
+1760::Future Unfolding
+1761::Real Heroes: Firefighter
+1762::Tacopocalypse
+1763::Monster Monpiece
+1764::aMAZE
+1765::Nephise
+1766::Arma: Cold War Assault Mac/Linux
+1767::Chosen 2
+1768::A Girls Fabric Face
+1769::Another Adventure
+1770::Blossom Tales: The Sleeping King
+1771::Universe in Fire
+1772::MAKE IT as an Artist
+1773::Tower!3D Pro
+1774::Faeria
+1775::Tanki Online
+1776::Turret Terminator
+1777::Dad Quest
+1778::Vive le Roi
+1779::Shadows of Adam
+1780::Magazime Editor
+1781::CropDuster Supreme
+1782::Herald: An Interactive Period Drama - Book I & II
+1783::8-Bit Armies: Arena (Free)
+1784::The Wardrobe
+1785::LOGistICAL
+1786::9Dragons
+1787::Dropzone
+1788::Command Ops 2
+1789::Oik
+1790::Upside Down
+1791::Dear Esther: Landmark Edition
+1792::Semispheres
+1793::MX Nitro
+1794::8-Bit Armies: Arena
+1795::Heavy Gear Assault
+1796::WayOut 2: Hex
+1797::Realpolitiks
+1798::Cabals: Card Blitz
+1799::The Wild Eight
+1800::Blood Harvest
+1801::OR
+1802::Max's Big Bust - A Captain Nekorai Tale
+1803::WWE 2K17
+1804::Induction
+1805::Gunmetal Arcadia
+1806::Xenon Valkyrie
+1807::Numberline
+1808::ANIMALITY
+1809::Alwa's Awakening
+1810::Midas Gold Plus
+1811::BIGFOOT
+1812::Itineris
+1813::Reptilians Must Die!
+1814::HitBox
+1815::Disgaea 2 PC / 魔界戦記ディスガイア2 PC
+1816::Double Dragon IV
+1817::Sakura Agent
+1818::Miner Ultra Adventures
+1819::A Normal Lost Phone
+1820::Witanlore: Dreamtime
+1821::Heroes Evolved
+1822::Avorion
+1823::Hellenica
+1824::Once Upon an All Hallow's Eve
+1825::Frequent Flyer
+1826::Urban Empire
+1827::Mortifero Motus
+1828::M.E.R.C.
+1829::Monumental Failure
+1830::Pinkman
+1831::Atomic 79
+1832::Zup! 3
+1833::Tales of Berseria™
+1834::Don't Chat With Strangers
+1835::Unalive
+1836::UNO
+1837::Word Killer: Zorgilonian Chronicles
+1838::Let's Draw
+1839::Score a goal (Physical football)
+1840::Blind Love
+1841::Behind the Memory
+1842::Random Journey
+1843::Where are my Internets?
+1844::Rumble Fighter: Unleashed
+1845::Volleyball Unbound - Pro Beach Volleyball
+1846::Xmas Shooting - Scramble!!
+1847::Castle Battles
+1848::VERSUS: The Elite Trials
+1849::Super Blue Boy Planet
+1850::Bamboo EP
+1851::Endless Fables: The Minotaur's Curse
+1852::Defense Zone 3 Ultra HD
+1853::The Little Acre
+1854::Palinurus
+1855::Disney Infinity 3.0: Gold Edition
+1856::Prominence Poker
+1857::LogicBots
+1858::Triennale Game Collection
+1859::Campfire: One of Us Is the Killer
+1860::Indie Game Sim
+1861::Galactic Fighter
+1862::Hunger Dungeon
+1863::Hade
+1864::Coffee Shop Tycoon
+1865::Void Pyramid
+1866::Minion Masters
+1867::Ahnayro: The Dream World
+1868::Terminal Hacker
+1869::Construct: Escape the System
+1870::Charlie's Adventure
+1871::Atom Fishing II
+1872::Shattered Throne
+1873::Quern - Undying Thoughts
+1874::Mad Hunter
+1875::Turret Architect
+1876::Peak Angle: Drift Online
+1877::Eon Altar
+1878::Front Office Football Eight
+1879::Star Trek: Starfleet Command Gold Edition
+1880::Germ Wars
+1881::Disastr_Blastr
+1882::Alicemare
+1883::DesertLand 2115
+1884::Glass Masquerade
+1885::拯救大魔王2 Rescue the Great Demon 2
+1886::Book Series - Alice in Wonderland
+1887::Kopanito All-Stars Soccer
+1888::Dreamcage Escape
+1889::Screeps
+1890::Drunk On Nectar
+1891::ENIGMA:
+1892::Ittle Dew 2
+1893::CLASH
+1894::Nodiatis
+1895::Multimirror
+1896::Spellstone
+1897::Resin
+1898::Badiya
+1899::One Tower
+1900::CRACKHEAD
+1901::Kokurase - Episode 1
+1902::Fairyland: Fairy Power
+1903::Motorsport Manager
+1904::Notruf 112 | Emergency Call 112
+1905::VOI
+1906::One Night Stand
+1907::Driver Booster 4 for Steam
+1908::WayOut
+1909::Dead Age
+1910::Steam Dev Days
+1911::Call of Duty®: Infinite Warfare
+1912::Melon Simulator™
+1913::Zero G Arena
+1914::Pro Skater 2D
+1915::Moto Racer  4
+1916::Slap The Fly
+1917::CameraBag Photo
+1918::Creepy Castle
+1919::Sombrero: Spaghetti Western Mayhem
+1920::Blight of the Immortals
+1921::VERSUS SQUAD
+1922::Yomawari: Night Alone / 夜廻
+1923::The Puppet Master
+1924::Dessert Storm
+1925::JumpBall
+1926::Murder Mystery Adventure
+1927::Cursed Castilla (Maldita Castilla EX)
+1928::Astral Heroes
+1929::Rusty Lake: Roots
+1930::AdventureQuest 3D
+1931::Lance A Lot®: Classic Edition
+1932::Soulless: Ray Of Hope
+1933::9th Dawn II
+1934::Crush Crush
+1935::NUKED KNIGHT
+1936::Physic Monster
+1937::Remaining in a dream
+1938::CrossWorlds: Escape
+1939::Razortron 2000
+1940::Near Midnight
+1941::Eventide 2: The Sorcerers Mirror
+1942::Duke Nukem 3D: 20th Anniversary World Tour
+1943::WRC 6 FIA World Rally Championship
+1944::Breathing Fear
+1945::Mafia III
+1946::Doorways: Old Prototype
+1947::Aragami
+1948::Unclaimed World
+1949::The Space Garden
+1950::The Sun Will Rise
+1951::Zombo Buster Rising
+1952::Helmet Heroes
+1953::Rogue Operatives Hide and Seek
+1954::Masquerada: Songs and Shadows
+1955::Rescue Bear Operation
+1956::FourChords Guitar Karaoke
+1957::Slayer Shock
+1958::Merger 3D
+1959::Princess Maker 2 Refine
+1960::God of Word
+1961::Trick & Treat
+1962::Make America Great Again: The Trump Presidency
+1963::Stellar Tactics
+1964::Dream Car Builder
+1965::Talewind
+1966::DROD: Gunthro and the Epic Blunder
+1967::Tentacult!
+1968::The Uncertain: Episode 1 - The Last Quiet Day
+1969::Toadled
+1970::Gun Bombers
+1971::Pankapu
+1972::Hospitalize
+1973::Think To Die
+1974::Fall of Civilization
+1975::Squeezone
+1976::Eclipse: New Dawn for the Galaxy
+1977::Diaries of a Spaceport Janitor
+1978::Make America Great Again
+1979::Solitaire Royale
+1980::Soccer Manager 2017
+1981::Beautiful Japanese Scenery - Animated Jigsaws
+1982::Dots eXtreme
+1983::Carrie's Order Up!
+1984::PAC-MAN™ CHAMPIONSHIP EDITION 2
+1985::Mad Games Tycoon
+1986::Monsti
+1987::Otherland MMO
+1988::Envy the Dead
+1989::What The Box?
+1990::Chess Knight 2
+1991::Zombitatos the end of the Pc master race
+1992::Pixel Puzzles Ultimate
+1993::Pixelscape: Oceans
+1994::STEINS;GATE
+1995::Star Crusade CCG
+1996::Nanoborg
+1997::Dungeon Rushers: Crawler RPG
+1998::Revolution 60
+1999::The Tale of a Common Man
+2000::Tadpole Treble
+2001::Windscape
+2002::Cashtronauts
+2003::The Last Hope
+2004::Destiny of Ancient Kingdoms™
+2005::Reset 1-1
+2006::Thorne - Son of Slaves (Ep.2)
+2007::The Hive
+2008::Master of Orion
+2009::Virginia
+2010::Dangerous Relationship
+2011::Momento Temporis: Light from the Deep
+2012::Picross Touch
+2013::Ultimate Arena
+2014::Hardware Engineers
+2015::J.U.R : Japan Underground Racing
+2016::Orbital X
+2017::Combat Core
+2018::Clean'Em Up
+2019::Girl Amazon Survival
+2020::No Man's Sky
+2021::Siralim 2
+2022::8-Bit Hordes
+2023::Genius Greedy Mouse
+2024::My Butler
+2025::Meridian: Squad 22
+2026::Kimulator : Fight for your destiny
+2027::Through Abandoned 2. The Forest
+2028::Handsome Mr. Frog
+2029::Tricky Towers
+2030::ABZU
+2031::Rising Islands
+2032::Near Death
+2033::MANDAGON
+2034::Hacker Evolution IMMERSION
+2035::D.N.Age
+2036::Blade Arcus from Shining: Battle Arena
+2037::Barnyard Mahjong 3
+2038::The Amazing Shinsengumi: Heroes in Love
+2039::Riptide GP: Renegade
+2040::Kingdom Rush Frontiers
+2041::Ghost of a Tale
+2042::Quadrilateral Cowboy
+2043::SolForge
+2044::SUPERFIGHT
+2045::Zavix Tower
+2046::Polarity
+2047::Emerland Solitaire: Endless Journey
+2048::Mahluk:Dark demon
+2049::Mainlining
+2050::Domain Defense
+2051::Pirates of the Polygon Sea
+2052::NECROPOLIS: BRUTAL EDITION
+2053::Rot Gut
+2054::Killbot
+2055::Bot Vice
+2056::ESEA
+2057::Song of the Deep
+2058::Moon Colonization Project
+2059::Haven Moon
+2060::Jonah's Path
+2061::Star Rangers™ XE
+2062::Dragonpath
+2063::FreeHolder
+2064::Megadimension Neptunia VII
+2065::Killing Time at Lightspeed: Enhanced Edition
+2066::Hero Zero
+2067::Rugby Challenge 3
+2068::Crush Your Enemies
+2069::LUMBERMANCER
+2070::Osozaki 遅咲き Late Blooming - First
+2071::Potato Thriller
+2072::Umbrella Corps™/Biohazard Umbrella Corps™
+2073::Asemblance
+2074::Ortus Regni
+2075::Evil Maze
+2076::Urban Pirate
+2077::Space Run Galaxy
+2078::AWA
+2079::Envoy 2
+2080::Awareness Rooms
+2081::Transmissions: Element 120
+2082::Polandball: Can into Space!
+2083::Bastard Bonds
+2084::Fantasy Kingdom Simulator
+2085::Legends of Callasia
+2086::The Hat Man: Shadow Ward
+2087::One Small Fire At A Time
+2088::Vroomist
+2089::Choice of Alexandria
+2090::WildStar
+2091::Tick Tock Bang Bang
+2092::Gloria Victis
+2093::Emporea: Realms of War and Magic
+2094::Legacy of the Elder Star
+2095::VirtualHere For Steam Link
+2096::WASTED
+2097::The Orb Chambers™
+2098::Sol Trader
+2099::Evolution
+2100::Pepe Porcupine
+2101::Amulet of Dreams
+2102::Dark Days
+2103::Wanda - A Beautiful Apocalypse
+2104::Hope Lake
+2105::Ember Kaboom
+2106::Age of Barbarian Extended Cut
+2107::BloodGate
+2108::Onechanbara Z2: Chaos
+2109::Kabitis
+2110::Brigador: Up-Armored Edition
+2111::Grim Legends 3: The Dark City
+2112::Artificial Defense
+2113::Descent: Road to Legend
+2114::SENRAN KAGURA SHINOVI VERSUS
+2115::Beater Spirit
+2116::Runeous: Part One
+2117::The Concourse
+2118::BlackShot: Mercenary Warfare FPS
+2119::Dead Island Definitive Edition
+2120::Tower!3D
+2121::Overhell
+2122::Zenodyne R
+2123::ABC Coloring Town
+2124::Neon Hardcorps
+2125::Garlock Online
+2126::Iron Impact
+2127::Aurora Nights
+2128::Seduce Me 2: The Demon War
+2129::FleetCOMM
+2130::Total War: WARHAMMER
+2131::Shoppe Keep
+2132::BATTLE PIXEL'S SURVIVAL GROUND
+2133::Please Hold
+2134::Melody's Escape
+2135::ORCS
+2136::The Dweller
+2137::Linkrealms
+2138::NotCoD™
+2139::Koihime Enbu
+2140::The Mims Beginning
+2141::Remnants of a Beautiful Day
+2142::The Abbey of Crime Extensum
+2143::Space Codex
+2144::EPΘCH
+2145::Rogue System
+2146::Demetrios - The BIG Cynical Adventure
+2147::RefRain - prism memories -
+2148::DOOM
+2149::Murasaki
+2150::Trawl
+2151::Who's Your Daddy
+2152::She Wants Me Dead
+2153::Mini Golf Mundo
+2154::TASTEE: Lethal Tactics
+2155::My Secret Pets!
+2156::Super Blue Fighter
+2157::Ryzom
+2158::Theory Test UK 2016/17 - Driving Test Success
+2159::Istrolid
+2160::Stellaris
+2161::Star Saviors
+2162::Worm.is: The Game
+2163::Liveza: Death of the Earth
+2164::Cyber City 2157: The Visual Novel
+2165::I and Me
+2166::Gun Rocket
+2167::Aurora Dusk: Steam Age
+2168::Initia: Elemental Arena
+2169::Worlds Adrift Island Creator
+2170::Puzzle Box
+2171::Oil Enterprise
+2172::Bloons TD Battles
+2173::Megamagic: Wizards of the Neon Age
+2174::Don't Starve Together
+2175::Rogue Stormers
+2176::Endciv
+2177::OPUS: The Day We Found Earth
+2178::Western Press
+2179::My Night Job
+2180::Plastic Playground
+2181::Typefighters (Steam Edition)
+2182::Tales Across Time
+2183::Danganronpa 2: Goodbye Despair
+2184::HEX: Shards of Fate
+2185::Voxel Warfare Online
+2186::IS Defense
+2187::The Banner Saga 2
+2188::Night Blights
+2189::Goetia
+2190::Pub Encounter
+2191::Club Life
+2192::Last Hope - Tower Defense
+2193::Grand Designer
+2194::DARK SOULS™ III
+2195::StarsOne
+2196::Tenrow
+2197::The Hero Project: Redemption Season
+2198::Game Tycoon 2
+2199::Virtual Rogue
+2200::Tower Unite
+2201::Fortify
+2202::MXGP2 - The Official Motocross Videogame
+2203::Mushroom Wars
+2204::A Blind Legend
+2205::Axes and Acres
+2206::Campus Notes - forget me not.
+2207::Odd||Even
+2208::Little Walker
+2209::Outrage
+2210::Project Starship
+2211::Sorcery! Part 3
+2212::Domino Sky
+2213::Metaverse
+2214::Until I Have You
+2215::Womb Room
+2216::Daughter of Shadows: An SCP Breach Event
+2217::Epsilon corp.
+2218::CHKN
+2219::Run Rabbit Run
+2220::Blue Sheep
+2221::Trackday Manager
+2222::Tokyo Babel
+2223::Catch a Falling Star
+2224::[the Sequence]
+2225::Bunker Punks
+2226::Epistory - Typing Chronicles
+2227::Trial by Viking
+2228::XenoShyft
+2229::The Last Door: Season 2 - Collector's Edition
+2230::R.B.I. Baseball 16
+2231::Angry Video Game Nerd II: ASSimilation
+2232::MiniGolf Mania
+2233::Forestry 2017 - The Simulation
+2234::I Am Caligula
+2235::Holodrive
+2236::Slain: Back from Hell
+2237::A Lenda do Herói - O Herói desta Canção
+2238::Airstrike HD
+2239::NOBUNAGA'S AMBITION: Sphere of Influence - Ascension / 信長の野望･創造 戦国立志伝
+2240::LostWinds
+2241::Wolcen: Lords of Mayhem
+2242::Disposable Heroes
+2243::Heroes of The West
+2244::Day of the Tentacle Remastered
+2245::The Next Door
+2246::Spaceman Sparkles 3
+2247::Congo
+2248::Helen's Mysterious Castle
+2249::War Birds: WW2 Air strike 1942
+2250::Square's Route
+2251::Hyper Box
+2252::Victory and Glory: Napoleon
+2253::Hustle Cat
+2254::World's Fastest Pizza
+2255::Mars 2030
+2256::Out There Somewhere
+2257::SPATIAL SOUND CARD
+2258::There's Poop In My Soup
+2259::Beat Da Beat
+2260::The Secret Order 3: Ancient Times
+2261::Moon Hunters
+2262::Gremlins, Inc.
+2263::Rabiez: Epidemic
+2264::Mind Games
+2265::Paper Train Traffic
+2266::Space Pilgrim Episode IV: Sol
+2267::Mystica: The Ninth Society
+2268::GTGD S3 How To Make A Game
+2269::LiEat
+2270::Perfect Universe - Play with Gravity
+2271::UnReal World
+2272::Stardew Valley
+2273::Trulon: The Shadow Engine
+2274::STAR WARS™ Galactic Battlegrounds Saga
+2275::Deponia Doomsday
+2276::STAR WARS™ Rebellion
+2277::Space Impossible
+2278::XBlaze Code: Embryo
+2279::Collider
+2280::Where's My Mommy?
+2281::Bus Simulator 16
+2282::Karaski: What Goes Up...
+2283::Acorn Assault: Rodent Revolution
+2284::Rock Paper Scissors Champion
+2285::Party Jousting
+2286::Master of Orion 2
+2287::Heaven's Hope - Special Edition
+2288::And So It Was
+2289::Gnomoria
+2290::Governor of Poker 3
+2291::Steamroll
+2292::Broken Dreams
+2293::The Quest
+2294::Magdalena
+2295::ChromaGun
+2296::Kindred Spirits on the Roof
+2297::SHOWTIME 2073
+2298::NARUTO SHIPPUDEN: Ultimate Ninja STORM 4
+2299::Mystic Destinies: Serendipity of Aeons
+2300::Wayward Terran Frontier: Zero Falls
+2301::Lucent Heart
+2302::XCOM® 2
+2303::Moonlight
+2304::The Political Machine 2016
+2305::Curvatron
+2306::Rusty Lake Hotel
+2307::Sunny Hillride
+2308::CALENDULA
+2309::Bombshell
+2310::DarkMaus
+2311::Aozora Meikyuu
+2312::Mystic Saga
+2313::Narcissu 10th Anniversary Anthology Project
+2314::Super Mustache
+2315::PulseCharge
+2316::AI: Rampage
+2317::FullBlast
+2318::Eventide: Slavic Fable
+2319::Rabi-Ribi
+2320::Swiftly
+2321::Crashlands
+2322::Lif
+2323::Let's Sing 2016
+2324::Move or Die
+2325::Naval Action
+2326::Close Order
+2327::Subterrain
+2328::Pythagoria
+2329::Warlords Battlecry III
+2330::Between Me and The Night
+2331::O3DX
+2332::Rivalry
+2333::Slybots: Frantic Zone
+2334::Cyber Team Manager
+2335::The Bug Butcher
+2336::Resident Evil 0 / biohazard 0 HD REMASTER
+2337::Darkest Dungeon®
+2338::Protoshift
+2339::Echoes of Aetheria
+2340::Highrise Heroes: Word Challenge
+2341::Evertown
+2342::Fairy Tale Mysteries: The Puppet Thief
+2343::Blood Code
+2344::sZone-Online
+2345::Linea, the Game
+2346::A Wild Catgirl Appears!
+2347::The Hurricane of the Varstray -Collateral hazard-
+2348::Turok
+2349::Lightbender
+2350::Moonshot
+2351::Puzzle Strike
+2352::Dance of Death
+2353::Gnomes Garden
+2354::Swords and Sorcery - Underworld - Definitive Edition
+2355::Monsterland
+2356::Star Nomad 2
+2357::Squad
+2358::Project Pulsation
+2359::Metal War Online: Retribution
+2360::Stories of Bethem: Full Moon
+2361::PewDiePie: Legend of the Brofist
+2362::LIGHTNING RETURNS™: FINAL FANTASY® XIII
+2363::Vehicle Simulator
+2364::Romance of the Three Kingdoms Maker / 三国志ツクール
+2365::Comic Book Hero: The Greatest Cape
+2366::The Mean Greens - Plastic Warfare
+2367::Manyland
+2368::Nuclear Throne
+2369::SquareCells
+2370::Our Love Will Grow
+2371::Dr. Langeskov, The Tiger, and The Terribly Cursed Emerald: A Whirlwind Heist
+2372::Vampire Legends: The True Story of Kisilova
+2373::Militia
+2374::Legacy of Dorn: Herald of Oblivion
+2375::Ruzar - The Life Stone
+2376::Corgi Warlock
+2377::The Last Dream: Developer's Edition
+2378::Hit Tank PRO
+2379::Mayjasmine Episode01 - What is God?
+2380::Handball 16
+2381::Yet Another World
+2382::Might & Magic Heroes Online
+2383::Dead Acres
+2384::Mighty Switch Force! Academy
+2385::Idle Civilization
+2386::Zero Punctuation: Hatfall - Hatters Gonna Hat Edition
+2387::Switch Galaxy Ultra
+2388::Vendetta - Curse of Raven's Cry
+2389::Terraformer Expedition to Mars
+2390::Nusakana
+2391::The Consuming Shadow
+2392::Hard West
+2393::Rescue Team 5
+2394::Dungeon Kingdom: Sign of the Moon
+2395::Cross Set
+2396::GOCCO OF WAR
+2397::Back to Dinosaur Island
+2398::Survivor Squad: Gauntlets
+2399::Shadow Ninja: Apocalypse
+2400::Knight Squad
+2401::Trouble In The Manor
+2402::Affected Zone Tactics
+2403::World War II GI
+2404::Last Heroes
+2405::Steel Ocean
+2406::STEEL STRIDER
+2407::rFactor 2
+2408::Legend (1994)
+2409::Piercing Blow
+2410::Impossible Creatures Steam Edition
+2411::Fallout 4
+2412::Steam Controller
+2413::Defend The Highlands
+2414::The Odyssey: Winds of Athena
+2415::Without Within 2
+2416::Diabolical
+2417::HotLead
+2418::Ironclads 2: American Civil War
+2419::Conflicks - Revolutionary Space Battles
+2420::The Secret Order 2: Masked Intent
+2421::ePic Character Generator
+2422::Darksiders II Deathinitive Edition
+2423::Wave of Darkness
+2424::Zero Reflex : Black Eye Edition
+2425::Battle Battalions
+2426::Dragon Fin Soup
+2427::PONCHO
+2428::Mirrored - Chapter 1
+2429::Fallout: A Post Nuclear Role Playing Game
+2430::Sonic Lost World
+2431::Bit Shifter
+2432::The Extinction
+2433::The Seven Years War (1756-1763)
+2434::Pray For Diamonds
+2435::Devil's Bluff
+2436::Morphine
+2437::Deathless: The City's Thirst
+2438::ALPAGES : THE FIVE BOOKS
+2439::Warhammer: End Times - Vermintide
+2440::RPG Maker MV
+2441::200% Mixed Juice!
+2442::Flight of the Paladin
+2443::Heavy Fire: Shattered Spear
+2444::Mu Complex
+2445::Chaos Reborn
+2446::System Shock: Enhanced Edition
+2447::Wurm Unlimited
+2448::PixelJunk™ Shooter Ultimate
+2449::Eternal Step
+2450::Rogue State
+2451::The Jackbox Party Pack 2
+2452::Minecraft: Story Mode - A Telltale Games Series
+2453::Deus Ex: Revision
+2454::Bedlam
+2455::Beyond Sol
+2456::Soccer Manager 2016
+2457::March of Industry: Very Capitalist Factory Simulator Entertainments
+2458::There Was A Caveman
+2459::2064: Read Only Memories
+2460::SKYHILL
+2461::Korwin The Game
+2462::Meridian: Age of Invention
+2463::Arcana Heart 3 LOVE MAX!!!!!
+2464::Stigmat
+2465::Ninja Pizza Girl
+2466::Lumber Island - That Special Place
+2467::Puzzle Ball
+2468::ARM PLANETARY PROSPECTORS Asteroid Resource Mining
+2469::Black & White Bushido
+2470::Hylics
+2471::Ratings War
+2472::Silver Creek Falls: Chapter 2
+2473::The Escapists: The Walking Dead
+2474::NBA 2K16
+2475::Blowy Fish
+2476::Shoppy Mart: Steam Edition
+2477::Shapes of Gray
+2478::Kanji Training Game
+2479::Golden Rush
+2480::A Kiss For The Petals - Remembering How We Met
+2481::Stairs
+2482::Tic-Toc-Tower
+2483::Orion: A Sci-Fi Visual Novel
+2484::Keen Dreams
+2485::Tango Fiesta
+2486::Explosionade
+2487::Invasion
+2488::TankZone Battle
+2489::Destiny Warriors RPG
+2490::The Juicer
+2491::Dungeon Nightmares II : The Memory
+2492::Future Farmer
+2493::Fate Tectonics
+2494::Alien Robot Monsters
+2495::Warhammer 40,000: Regicide
+2496::Shower With Your Dad Simulator 2015: Do You Still Shower With Your Dad
+2497::Act of Aggression - Reboot Edition
+2498::Streamline
+2499::Kitchen Simulator 2015
+2500::Raiden IV: OverKill
+2501::Cute Things Dying Violently
+2502::Snik
+2503::A Wise Use of Time
+2504::Rampage Knights
+2505::Zoo Empire
+2506::Wars and Warriors: Joan of Arc
+2507::Almightree: The Last Dreamer
+2508::Rememoried
+2509::Tinboy
+2510::One More Line
+2511::Port of Call
+2512::Proto Raider
+2513::Redemption: Eternal Quest
+2514::Clown House (Palyaço Evi)
+2515::I Shall Remain
+2516::Grandia® II Anniversary Edition
+2517::Flywrench
+2518::Pretty Girls Mahjong Solitaire
+2519::Owys
+2520::Evoland 2
+2521::Party Hard
+2522::Hegemony III: Clash of the Ancients
+2523::Crookz - The Big Heist
+2524::WARMODE
+2525::Age of Survival
+2526::Pump-Action Captain
+2527::Curse of the Crescent Isle DX
+2528::In Between
+2529::Forget Me Not: My Organic Garden
+2530::ORBITOR
+2531::Super Mega Baseball: Extra Innings
+2532::VERSUS: The Lost Ones
+2533::Zanzarah: The Hidden Portal
+2534::ABD: A Beautiful Day
+2535::Volume
+2536::RPG MO
+2537::Fingered
+2538::Illuminascii
+2539::Nightside
+2540::Fitz the Fox
+2541::Spider: Rite of the Shrouded Moon
+2542::Shu's Garden
+2543::Shadow of Kingdoms
+2544::12 Labours of Hercules III: Girl Power
+2545::Putrefaction
+2546::Hidden: On the trail of the Ancients
+2547::Dream
+2548::Job the Leprechaun
+2549::Go Home - Rage incoming
+2550::The Deletion
+2551::Stick 'Em Up 2: Paper Adventures
+2552::Legend of Kay Anniversary
+2553::Hogs of War
+2554::Chronicon
+2555::Spooky's Jump Scare Mansion
+2556::Another Star
+2557::Inferno 2
+2558::The Viceroy
+2559::Airport Madness 4
+2560::Borderless Gaming
+2561::Way of the Samurai 4
+2562::FEIST
+2563::Zenzizenzic
+2564::Terra Nova: Strike Force Centauri
+2565::Time Clickers
+2566::RETSNOM
+2567::Energy Balance
+2568::Stranded In Time
+2569::Niko: Through The Dream
+2570::Executive Assault
+2571::RPG Maker 2000
+2572::001 Game Creator
+2573::Fighties
+2574::Lost Lands: The Four Horsemen
+2575::Steve Chong Finds Out That Suicide is a Bad Idea
+2576::MechaNika
+2577::SEEP Universe
+2578::TeraBlaster
+2579::How to Take Off Your Mask
+2580::Scraps: Modular Vehicle Combat
+2581::Cricket Captain 2015
+2582::Super Cyborg
+2583::Clandestinity of Elsie
+2584::Gary Grigsby's War in the East
+2585::Cosmophony
+2586::Trove
+2587::The Red Solstice
+2588::Weapons Genius
+2589::Crash Dive
+2590::Austin High
+2591::Gaming In Color
+2592::Corrosion: Cold Winter Waiting [Enhanced Edition]
+2593::Quiplash
+2594::RONIN
+2595::Vector Thrust
+2596::Unpossible
+2597::Vapour
+2598::Devilry
+2599::Yargis - Space Melee
+2600::Sumo Revise
+2601::Kings of Kung Fu
+2602::Super 3-D Noah's Ark
+2603::Centauri Sector
+2604::Medieval: Total War™ - Collection
+2605::SHOGUN: Total War™ - Collection
+2606::Interstellar Rift
+2607::Devil May Cry® 4 Special Edition
+2608::Masterspace
+2609::Shot In The Dark
+2610::Monster Jam Battlegrounds
+2611::Sherlock Holmes Consulting Detective: The Case of the Mummy's Curse
+2612::TransPlan
+2613::Out of Reach
+2614::Tap Heroes
+2615::Wimp: Who Stole My Pants?
+2616::12 Labours of Hercules II: The Cretan Bull
+2617::XIIZEAL
+2618::16bit Trader
+2619::Butsbal
+2620::SimpleRockets
+2621::New kind of adventure
+2622::Cave Coaster
+2623::Organ Biker
+2624::Flix The Flea
+2625::Avenging Angel
+2626::3D MiniGolf
+2627::Horizon Shift
+2628::Star Horizon
+2629::Subject 13
+2630::Locoland
+2631::Chip's Challenge 1
+2632::Chip's Challenge 2
+2633::Portal of Evil: Stolen Runes Collector's Edition
+2634::Spaceforce Rogue Universe HD
+2635::GUILTY GEAR XX ACCENT CORE PLUS R
+2636::Stones of Sorrow
+2637::Magnetic: Cage Closed
+2638::Quadrant
+2639::Zombie Zoeds
+2640::Dev Guy
+2641::Crossfire: Dungeons
+2642::Sunset
+2643::Bunker - The Underground Game
+2644::GameGuru
+2645::Making History: The Calm and the Storm Gold Edition
+2646::S.K.I.L.L. - Special Force 2 (Shooter)
+2647::D Series OFF ROAD Driving Simulation
+2648::Breezeblox
+2649::CrossCode
+2650::Broadsword : Age of Chivalry
+2651::Grey Cubes
+2652::Higurashi When They Cry Hou - Ch.1 Onikakushi
+2653::Batla
+2654::Close Combat - Panthers in the Fog
+2655::Quell Memento
+2656::Invisible, Inc.
+2657::LOST ORBIT
+2658::Despair
+2659::Airport Firefighters - The Simulation
+2660::Why Am I Dead At Sea
+2661::FINAL FANTASY IV: THE AFTER YEARS
+2662::Toren
+2663::Shrooms
+2664::Selfie : Sisters of the Amniotic Lens
+2665::Yomi
+2666::Star Trek™ : 25th Anniversary
+2667::Snakebird
+2668::Controller Companion
+2669::Alexia Crow and the Cave of Heroes
+2670::Egg Returns Home
+2671::GemCraft - Chasing Shadows
+2672::Exanima
+2673::The Weaponographist
+2674::Lux Delux
+2675::Order of Battle: World War II
+2676::Solarix
+2677::Abyss Raiders: Uncharted
+2678::Soul Locus
+2679::R.B.I. Baseball 15
+2680::Jump/Boxer
+2681::Earthtongue
+2682::The Little Crane That Could
+2683::Kerbal Space Program
+2684::Son of Nor
+2685::Angry Video Game Nerd: The Movie
+2686::STAR WARS™ - X-Wing Special Edition
+2687::STAR WARS™ X-Wing vs TIE Fighter - Balance of Power Campaigns™
+2688::Uncanny Valley
+2689::Kaiju-A-GoGo
+2690::Assassin’s Creed® Chronicles: China
+2691::GameLoading: Rise of the Indies
+2692::MX vs. ATV Unleashed
+2693::Asguaard
+2694::Grand Theft Auto V
+2695::Mortal Kombat X
+2696::BoxesWithGuns
+2697::Escape Machines
+2698::The Emptiness Deluxe Edition
+2699::SWR JST DX Selective Memory Erase Effect
+2700::Finders
+2701::I am Bread
+2702::Square Heroes
+2703::Primal Carnage: Extinction
+2704::Solar System Conflict
+2705::VoidExpanse
+2706::Chronicles of Teddy
+2707::Titan Souls
+2708::Out There: Ω Edition
+2709::Motivational Growth
+2710::Pillars of Eternity
+2711::Cultures - Northland
+2712::Please, Don’t Touch Anything
+2713::Cyberpunk 3776
+2714::It came from space, and ate our brains
+2715::Othello
+2716::World of Mixed Martial Arts 3
+2717::VRC PRO
+2718::An Alternative Reality: The Football Manager Documentary
+2719::Immune - True Survival
+2720::Top Trumps Turbo
+2721::Vox Populi Vox Dei 2
+2722::Tennis Elbow 2013
+2723::Chicken Invaders 5
+2724::Ghost Encounters: Deadwood - Collector's Edition
+2725::Hektor
+2726::Dodge
+2727::Soccertron
+2728::Dreaming Sarah
+2729::Hotline Miami 2: Wrong Number
+2730::Ori and the Blind Forest
+2731::ShellShock Live
+2732::Dreamscapes: Nightmare's Heir - Premium Edition
+2733::After Reset RPG
+2734::ASA: A Space Adventure - Remastered Edition
+2735::Ubinota
+2736::Pregnancy
+2737::rFactor
+2738::Savage Lands
+2739::Synonymy
+2740::Dustoff Heli Rescue
+2741::Stay Alight
+2742::Runestone Keeper
+2743::Pneuma: Breath of Life
+2744::The Dark Stone from Mebara
+2745::Bard to the Future
+2746::Tales of the Orient: The Rising Sun
+2747::Belladonna
+2748::Cubicle Quest
+2749::A Druid's Duel
+2750::Republique
+2751::Disney•Pixar Finding Nemo
+2752::Mystery of Neuschwanstein
+2753::HassleHeart
+2754::BlastZone 2
+2755::Camera Obscura
+2756::You Are Not A Banana: Better Edition
+2757::The Book of Unwritten Tales 2
+2758::Raptor: Call of The Shadows - 2015 Edition
+2759::The Escapists
+2760::Reload
+2761::Plush
+2762::Wooden Floor
+2763::Total War: ATTILA
+2764::Stock Car Extreme
+2765::TerraTech
+2766::Goats on a Bridge
+2767::Terra Lander
+2768::Mystery Masters: Psycho Train Deluxe Edition
+2769::Just Get Through
+2770::SickBrick
+2771::Steam Heroes
+2772::eden*
+2773::Exodus Wars: Fractured Empire
+2774::Deathtrap
+2775::Front Office Football Seven
+2776::Seven Kingdoms 2 HD
+2777::Natural Soccer
+2778::Infect and Destroy
+2779::Supreme League of Patriots
+2780::Life is Strange - Episode 1
+2781::Drive to Hell
+2782::Stardust Vanguards
+2783::Streets of Chaos
+2784::Heroes® of Might & Magic® III - HD Edition
+2785::Gravity Ghost
+2786::The Uninvited: MacVenture Series
+2787::The Old Tree
+2788::Shadowgate: MacVenture Series
+2789::868-HACK
+2790::8BitMMO
+2791::Pahelika: Secret Legends
+2792::Phantom Breaker: Battle Grounds
+2793::Dark Gates
+2794::Lost in a Forest
+2795::Kingdoms CCG
+2796::Decisive Campaigns: The Blitzkrieg from Warsaw to Paris
+2797::Mimpi
+2798::Combat Monsters
+2799::Citizens of Earth
+2800::Disillusions Manga Horror
+2801::MotorSport Revolution
+2802::Double Dragon Trilogy
+2803::HyperRogue
+2804::Cahors Sunset
+2805::Unhack
+2806::BattleSpace
+2807::Sky Gamblers: Storm Raiders
+2808::Cubot
+2809::dUpLicity ~Beyond the Lies~
+2810::Towers of Altrac - Epic Defense Battles
+2811::Ninja Guy
+2812::Solar War
+2813::One Late Night: Deadline
+2814::Dwarf Tower
+2815::Nameless ~The one thing you must recall~
+2816::Xsyon - Prelude
+2817::Rooster Teeth vs. Zombiens
+2818::Sportsfriends
+2819::Chronicles of a Dark Lord: Episode 1 Tides of Fate Complete
+2820::The Repopulation
+2821::Outcast 1.1
+2822::Microsoft Flight Simulator X: Steam Edition
+2823::Rime Berta
+2824::Marvin's Mittens
+2825::THE KING OF FIGHTERS '98 ULTIMATE MATCH FINAL EDITION
+2826::Sky Mercenaries
+2827::Last Inua
+2828::MODO indie
+2829::It's A Wipe!
+2830::BlazBlue: Continuum Shift Extend
+2831::About Love, Hate and the other ones
+2832::Words for Evil
+2833::ALLTYNEX Second
+2834::Cargo 3
+2835::Carmageddon TDR 2000
+2836::Club Manager 2015
+2837::Bombing Bastards
+2838::fault - milestone one
+2839::Magnifico
+2840::1Quest
+2841::Potatoman Seeks the Troof
+2842::The Old City: Leviathan
+2843::Rising World
+2844::Tales from the Borderlands
+2845::Feel-A-Maze
+2846::Ilamentia
+2847::Akaneiro: Demon Hunters
+2848::The Jackbox Party Pack
+2849::Eternal Winter
+2850::Rollers of the Realm
+2851::Cinemaware Anthology: 1986-1991
+2852::The Sun and Moon
+2853::Bloons TD 5
+2854::Basketball Pro Management 2015
+2855::Cherry Tree High I! My! Girls!
+2856::Robotex
+2857::Spriter Pro
+2858::Assassin's Creed® Unity
+2859::World of Subways 1 – The Path
+2860::Koya Rift
+2861::The Detail
+2862::Lords Of The Fallen™
+2863::Majestic Nights
+2864::Buzz Aldrin's Space Program Manager
+2865::NS2: Combat
+2866::Depth
+2867::Skara - The Blade Remains
+2868::Hardland
+2869::Squishy the Suicidal Pig
+2870::Pajama Sam: Games to Play on Any Day
+2871::Fatty Bear's Birthday Surprise
+2872::Slave Zero
+2873::Ziggurat
+2874::Data Hacker: Corruption
+2875::Door Kickers
+2876::DarkEnd
+2877::Supreme Ruler Ultimate
+2878::May’s Mysteries: The Secret of Dragonville
+2879::Borderlands: The Pre-Sequel
+2880::Last Knight: Rogue Rider Edition
+2881::Disney Epic Mickey 2:  The Power of Two
+2882::Disney•Pixar Cars 2: The Video Game
+2883::Disney•Pixar Brave: The Video Game
+2884::A Golden Wake
+2885::Sid Meier's Colonization (Classic)
+2886::Schein
+2887::Boot Hill Heroes
+2888::ShaderTool
+2889::Split/Second
+2890::Disney TRON: Evolution
+2891::Gauntlet™ Slayer Edition
+2892::Schrödinger’s Cat And The Raiders Of The Lost Quark
+2893::TransOcean: The Shipping Company
+2894::911: First Responders®
+2895::Roadside Assistance Simulator
+2896::Kraven Manor
+2897::Dandelion - Wishes brought to you -
+2898::Command: Modern Air / Naval Operations WOTY
+2899::Outland
+2900::Shadow Puppeteer
+2901::The Stalin Subway
+2902::openCanvas 6
+2903::Boson X
+2904::Season Match
+2905::Bridge Constructor Medieval
+2906::Voyage: Journey to the Moon
+2907::Runers
+2908::RECYCLE
+2909::Train Fever
+2910::Anarchy Arcade
+2911::Dead Rising 3 Apocalypse Edition
+2912::Ford Street Racing
+2913::Ford Racing 3
+2914::Ford Racing Off Road
+2915::Hack 'n' Slash
+2916::The Guild Gold Edition
+2917::Transport Giant
+2918::Toast Time
+2919::FATE: The Traitor Soul
+2920::The Great War 1918
+2921::Bravada
+2922::Sentris
+2923::Mountain
+2924::Haunted
+2925::ACE - Arena: Cyber Evolution
+2926::Dungeons: The Eye of Draconus
+2927::Risen 3 - Titan Lords
+2928::Chess 2: The Sequel
+2929::Depth Hunter 2: Deep Dive
+2930::Cubic Castles
+2931::Ultra Street Fighter® IV
+2932::Heroes of a Broken Land
+2933::From the Depths
+2934::Rex Rocket
+2935::Space Hack
+2936::Back to Bed
+2937::Pure Pool
+2938::Blackbay Asylum
+2939::Guns and Robots
+2940::Industry Empire
+2941::Colin McRae Rally
+2942::Spud's Quest
+2943::Unrest
+2944::Gods Will Be Watching
+2945::Frayed Knights: The Skull of S'makh-Daon
+2946::Terrorhedron Tower Defense
+2947::UnEpic
+2948::Fall of the New Age Premium Edition
+2949::Murder Miners
+2950::Total Pro Golf 3
+2951::Small Town Terrors Pilgrim's Hook Collector's Edition
+2952::Blood: One Unit Whole Blood
+2953::Blood II: The Chosen + Expansion
+2954::Quest for Infamy
+2955::Stained
+2956::Cyto
+2957::Isomer
+2958::Robot Rescue Revolution
+2959::Alpha Zylon
+2960::Wayward Manor
+2961::Aperture Tag: The Paint Gun Testing Initiative
+2962::Ground Pounders
+2963::Shantae: Risky's Revenge - Director's Cut
+2964::Wild Warfare
+2965::Whispering Willows
+2966::AutoTileGen
+2967::Ship Simulator: Maritime Search and Rescue
+2968::The House
+2969::Railroad Pioneer
+2970::Bridge Constructor Playground
+2971::Soldiers: Heroes of World War II
+2972::Shattered Planet
+2973::School Bus Fun
+2974::Aeon Command
+2975::Asteria
+2976::Aura Kingdom
+2977::Battleplan: American Civil War
+2978::Z
+2979::Sniper Elite 3
+2980::Heroes Rise: The Prodigy
+2981::Pulstar
+2982::Crimsonland
+2983::Heileen 2: The Hands Of Fate
+2984::Ichi
+2985::Tex Murphy: Overseer
+2986::SPINTIRES™
+2987::Space Run
+2988::The Incredible Adventures of Van Helsing II
+2989::Dungeon of Elements
+2990::Probability 0
+2991::The Entente Gold
+2992::Floating Point
+2993::The Lost Crown
+2994::Haunt the House: Terrortown
+2995::Racer 8
+2996::Putt-Putt®: Pep's Birthday Surprise
+2997::Pajama Sam's Sock Works
+2998::Pajama Sam's Lost & Found
+2999::Ionball 2: Ionstorm
+3000::Freespace 2
+3001::Crimzon Clover WORLD IGNITION
+3002::Dead Bits
+3003::Pandora: First Contact
+3004::Manhunter
+3005::Astebreed: Definitive Edition
+3006::Pixel Boy and the Ever Expanding Dungeon
+3007::Salammbô: Battle for Carthage
+3008::Lost Marbles
+3009::Battlepaths
+3010::Legends of Persia
+3011::1001 Spikes
+3012::Defiance
+3013::Dark Raid
+3014::Noir Syndrome
+3015::Metal Planet
+3016::Splatter - Zombie Apocalypse
+3017::Meltdown
+3018::Pajama Sam 4: Life Is Rough When You Lose Your Stuff!
+3019::Putt-Putt® Enters the Race
+3020::Game Character Hub
+3021::ibb & obb
+3022::FINAL FANTASY III
+3023::Antisquad
+3024::Super Game Jam
+3025::Panzer Tactics HD
+3026::Ballad of Solar
+3027::Monochroma
+3028::G-Ball
+3029::Arcadecraft
+3030::Wolfenstein: The New Order
+3031::Q.U.B.E: Director's Cut
+3032::World of Guns: Gun Disassembly
+3033::Firefighters 2014
+3034::Always Sometimes Monsters
+3035::Panzer Elite Action Gold Edition
+3036::Jumpdrive
+3037::DYNASTY WARRIORS 8: Xtreme Legends Complete Edition / 真・三國無双７ with 猛将伝
+3038::Ascendant
+3039::Empress Of The Deep
+3040::Memento Mori 2
+3041::Iron Storm
+3042::Empress Of The Deep 2: Song Of The Blue Whale
+3043::RUNNING WITH RIFLES
+3044::1849
+3045::Terminal Velocity
+3046::Secret Agent
+3047::Action! - Gameplay Recording and Streaming
+3048::Tesla Effect: A Tex Murphy Adventure
+3049::The Impossible Game
+3050::Malevolence: The Sword of Ahkranox
+3051::8BitBoy™
+3052::GunZ 2: The Second Duel
+3053::Haegemonia: Legions of Iron
+3054::Putt-Putt® and Pep's Dog on a Stick
+3055::Spy Fox 2 "Some Assembly Required"
+3056::Pajama Sam 2: Thunder and Lightning Aren't So Frightening
+3057::Colonies Online
+3058::Farming World
+3059::The Dungeoning
+3060::Eurofighter Typhoon
+3061::Death Rally (Classic)
+3062::Millennium - A New Hope
+3063::Battlepillars Gold Edition
+3064::Slip
+3065::Escape The Museum
+3066::Evopollution
+3067::Always Remember Me
+3068::Millie
+3069::The Last Federation
+3070::Thinking with Time Machine
+3071::Sentinel
+3072::FRACT OSC
+3073::Starlight Inception™™
+3074::Life Goes On: Done to Death
+3075::Wargame: Red Dragon
+3076::NEStalgia
+3077::Dark Lore Mysteries: The Hunt For Truth
+3078::Avoid - Sensory Overload
+3079::Strike Suit Zero: Director's Cut
+3080::LEGO® The Hobbit™
+3081::Aggression: Europe Under Fire
+3082::Haunted Past: Realm of Ghosts
+3083::Mirror Mysteries 2
+3084::Warlock 2: The Exiled
+3085::Princess Isabella - Return of the Curse
+3086::ReignMaker
+3087::White Haven Mysteries
+3088::Spy Fox in "Dry Cereal"
+3089::Praetorians
+3090::Adventure Time: Finn and Jake's Epic Quest
+3091::Collapse
+3092::Castlevania: Lords of Shadow – Mirror of Fate HD
+3093::Creeper World 3: Arc Eternal
+3094::TUG
+3095::Rogue's Tale
+3096::Real Horror Stories Ultimate Edition
+3097::Batman™: Arkham Origins Blackgate - Deluxe Edition
+3098::Gray Matter
+3099::3 Stars of Destiny
+3100::Hive
+3101::Strategic War in Europe
+3102::YAIBA: NINJA GAIDEN Z
+3103::Ubersoldier II
+3104::Explodemon
+3105::Call of Duty®: Ghosts
+3106::Desert Gunner
+3107::Desert Thunder
+3108::US and THEM
+3109::Revolution Ace
+3110::10 Second Ninja
+3111::Incoming Forces
+3112::City of Steam: Arkadia
+3113::Return to Mysterious Island 2
+3114::Infested Planet
+3115::Corporate Lifestyle Simulator
+3116::Dragons and Titans
+3117::Soulbringer
+3118::The Book of Legends
+3119::Post Master
+3120::Riptide GP2
+3121::Race To Mars
+3122::Mitsurugi Kamui Hikae
+3123::Procyon
+3124::Where Angels Cry
+3125::Iron Soul
+3126::Chuck's Challenge 3D
+3127::One Finger Death Punch
+3128::Descent 2
+3129::Humanity Asset
+3130::Holy Avatar vs. Maidens of the Dead
+3131::Ninja Cats vs Samurai Dogs
+3132::Rambo The Video Game: Baker Team
+3133::Go! Go! Nippon! ~My First Trip to Japan~
+3134::Major Mayhem
+3135::Journal
+3136::WazHack
+3137::Drox Operative
+3138::Masters of the World - Geopolitical Simulator 3
+3139::Super Killer Hornet: Resurrection
+3140::Dead Man's Draw
+3141::The LEGO® Movie - Videogame
+3142::Aveyond 3-1: Lord of Twilight
+3143::Graviteam Tactics: Operation Star
+3144:://N.P.P.D. RUSH//- The milk of Ultraviolet
+3145::Let's Sing
+3146::MorphVOX Pro - Voice Changer
+3147::Loadout
+3148::Glacier 3: The Meltdown
+3149::Might & Magic X - Legacy
+3150::KAMI
+3151::Original War
+3152::Strike Vector
+3153::Broken Age
+3154::KickBeat Steam Edition
+3155::Hitman: Contracts
+3156::Blackguards
+3157::Steam Bandits: Outpost
+3158::Loren The Amazon Princess
+3159::METAL GEAR RISING: REVENGEANCE
+3160::Realms of Arkania 1 - Blade of Destiny Classic
+3161::3089 -- Futuristic Action RPG
+3162::Kingdom Rush
+3163::Unearthed: Trail of Ibn Battuta - Episode 1 - Gold Edition
+3164::Grimm
+3165::Dementium II HD
+3166::The Walking Dead: Season 2
+3167::Zigfrak
+3168::RPG Maker XP
+3169::Stick it to The Man!
+3170::Blockland
+3171::Just Cause 2: Multiplayer Mod
+3172::Samurai Gunn
+3173::Small World 2
+3174::OMSI 2: Steam Edition
+3175::MURI
+3176::Nimble Quest
+3177::The Novelist
+3178::Speedball 2 HD
+3179::FINAL FANTASY VIII
+3180::Draw a Stickman: EPIC
+3181::Postmortem: One Must Die (Extended Cut)
+3182::Vector
+3183::Half-Life: Before
+3184::Tank Operations: European Campaign
+3185::Gas Guzzlers Extreme
+3186::Blood Knights
+3187::Earth 2140
+3188::Injustice: Gods Among Us Ultimate Edition
+3189::Intake
+3190::Type:Rider
+3191::Anomaly Warzone Earth Mobile Campaign
+3192::Valdis Story: Abyssal City
+3193::Magicka: Wizards of the Square Tablet
+3194::Sniper Elite: Nazi Zombie Army 2
+3195::Heli Heroes
+3196::Journey of a Roach
+3197::The 7th Guest
+3198::Stronghold HD
+3199::FORCED: Slightly Better Edition
+3200::Dragon's Lair 2: Time Warp
+3201::Demonicon
+3202::NARUTO SHIPPUDEN: Ultimate Ninja STORM 3 Full Burst HD
+3203::WRC 4 FIA World Rally Championship
+3204::Democracy 3
+3205::Talisman: Prologue
+3206::Urban Chaos
+3207::Megabyte Punch
+3208::Day One : Garry's Incident
+3209::Dysfunctional Systems: Learning to Manage Chaos
+3210::Giana Sisters: Twisted Dreams - Rise of the Owlverlord
+3211::Gorky 17
+3212::KnightShift
+3213::Legends of Aethereus
+3214::Daikatana
+3215::Salvation Prophecy
+3216::Air Conflicts: Vietnam
+3217::Silent Storm Gold Edition
+3218::PlayClaw 5 - Game Recording and Streaming
+3219::Artemis Spaceship Bridge Simulator
+3220::Memoria
+3221::Brothers - A Tale of Two Sons
+3222::Hate Plus
+3223::Tom Clancy’s Splinter Cell Blacklist
+3224::Skullgirls
+3225::Spelunky
+3226::Guncraft
+3227::Leisure Suit Larry in the Land of the Lounge Lizards: Reloaded
+3228::Storm
+3229::The Apogee Throwback Pack
+3230::Interstellar Marines
+3231::Risk
+3232::The Night of the Rabbit
+3233::Cubetractor
+3234::The Swapper
+3235::The Incredible Adventures of Van Helsing
+3236::Dust: An Elysian Tail
+3237::Super Puzzle Platformer Deluxe
+3238::Gunpoint
+3239::Super Sanctum TD
+3240::Anomaly 2
+3241::Jack Lumber
+3242::Far Cry 3 - Blood Dragon
+3243::Dyad
+3244::Darkfall Unholy Wars
+3245::ORION: Prelude
+3246::Sacred Citadel
+3247::Age of Empires II HD
+3248::Dungeon Hearts
+3249::DisplayFusion
+3250::Cities in Motion 2
+3251::DLC Quest
+3252::Shattered Haven
+3253::Angry Birds Space
+3254::Go Home Dinosaurs!
+3255::The Showdown Effect
+3256::Arma II: DayZ Mod
+3257::The Banner Saga: Factions
+3258::Brutal Legend
+3259::Impire
+3260::Construct 2
+3261::Proteus
+3262::Sonic & All-Stars Racing Transformed
+3263::Omerta - City of Gangsters
+3264::10,000,000
+3265::BIT.TRIP VOID
+3266::Ace of Spades: Battle Builder
+3267::GTR 2 FIA GT Racing Game
+3268::Party of Sin
+3269::Far Cry 3
+3270::LEGO The Lord of the Rings
+3271::Tomb Raider V: Chronicles
+3272::Little Inferno
+3273::Sonic Adventure 2
+3274::Scribblenauts Unlimited
+3275::Cherry Tree High Comedy Club
+3276::Natural Selection 2
+3277::Pid
+3278::iBomber Attack
+3279::Conquest of Elysium 3
+3280::Deadlight
+3281::Viking: Battle for Asgard
+3282::Zombie Driver HD
+3283::Aerofly FS 1 Flight Simulator
+3284::Mark of the Ninja
+3285::Of Orcs And Men
+3286::R.A.W. Realms of Ancient War
+3287::Blood Bowl: Chaos Edition
+3288::McPixel
+3289::Planets Under Attack
+3290::Call of Duty: Black Ops - Mac Edition
+3291::Torchlight II
+3292::Air Conflicts: Pacific Carriers
+3293::Continent of the Ninth Seal
+3294::Tryst
+3295::The Basement Collection
+3296::Spirits
+3297::I Am Alive
+3298::DARK SOULS™: Prepare To Die™ Edition
+3299::Great Big War Game
+3300::Super Crate Box
+3301::Galactic Civilizations® I: Ultimate Edition
+3302::Cladun X2 / クラシックダンジョンX2
+3303::Check vs Mate
+3304::Thirty Flights of Loving
+3305::Symphony
+3306::Deponia
+3307::Unmechanical
+3308::eXceed - Gun Bullet Children
+3309::Inversion™
+3310::Wanderlust: Rebirth
+3311::Orcs Must Die! 2
+3312::Prototype 2
+3313::Dungeonbowl - Knockout Edition
+3314::Endless Space® - Collection
+3315::Adventures of Shuggy
+3316::Fray: Reloaded Edition
+3317::Game of Thrones
+3318::Iron Front: Digital War Edition
+3319::Conflict Desert Storm™
+3320::Magical Diary: Horse Hall
+3321::Thief™ Gold
+3322::Botanicula
+3323::Avernum 6
+3324::Avernum 4
+3325::Blades of Time
+3326::Sherlock Holmes and The Hound of The Baskervilles
+3327::Lone Survivor: The Director's Cut
+3328::The Walking Dead
+3329::A Valley Without Wind
+3330::Warlock - Master of the Arcane
+3331::Avernum: Escape From the Pit
+3332::Superbrothers: Sword & Sworcery EP
+3333::The Witcher 2: Assassins of Kings Enhanced Edition
+3334::Deep Black: Reloaded
+3335::Super MNC
+3336::All Zombies Must Die!: Scorepocalypse
+3337::Stronghold Kingdoms
+3338::Blackwell Deception
+3339::Blackwell Convergence
+3340::Kingdoms of Amalur: Reckoning™
+3341::Sonic CD
+3342::Unstoppable Gorg
+3343::Crusader Kings Complete
+3344::Insane 2
+3345::SOL: Exodus
+3346::Age of Empires® III: Complete Collection
+3347::LEGO Harry Potter: Years 5-7
+3348::Choplifter HD
+3349::Sideway™ New York
+3350::Renegade Ops
+3351::Worms Pinball
+3352::Worms Blast
+3353::Steel Storm: Burning Retribution
+3354::Might & Magic: Clash of Heroes
+3355::Dungeons - The Dark Lord
+3356::Hitogata Happa
+3357::The Clockwork Man: The Hidden World
+3358::A Game of Thrones - Genesis
+3359::F1 2011
+3360::Bastion
+3361::From Dust
+3362::Avadon: The Black Fortress
+3363::Legend of Fae
+3364::8-Bit Commando
+3365::Chantelise - A Tale of Two Sisters
+3366::E.Y.E: Divine Cybermancy
+3367::Kohan: Ahriman's Gift
+3368::Cossacks: European Wars
+3369::American Conquest: Fight Back
+3370::American Conquest
+3371::Cossacks II: Napoleonic Wars
+3372::Cossacks II: Battle for Europe
+3373::Achron
+3374::Gatling Gears
+3375::Demolition Company Gold Edition
+3376::Trauma
+3377::ARMA: Gold Edition
+3378::Trapped Dead
+3379::Operation Flashpoint: Red River
+3380::Vertex Dispenser
+3381::Elizabeth Find M.D. - Diagnosis Mystery - Season 2
+3382::Zombie Pirates
+3383::The Next BIG Thing
+3384::Cargo! The Quest for Gravity
+3385::Hydrophobia: Prophecy
+3386::Portal 2 - The Final Hours
+3387::Syberia
+3388::NightSky
+3389::SEGA Bass Fishing
+3390::Atom Zombie Smasher
+3391::Post Apocalyptic Mayhem
+3392::Revenge of the Titans
+3393::LEGO® Star Wars™ III - The Clone Wars™
+3394::Shift 2 Unleashed
+3395::Dungeon Siege
+3396::Agricultural Simulator 2011: Extended Edition
+3397::Men of War: Assault Squad
+3398::Painkiller Redemption
+3399::Magicka
+3400::Blue Toad Murder Files™: The Mysteries of Little Riddle
+3401::EVE Online
+3402::Dive to the Titanic
+3403::The UnderGarden
+3404::Alien Breed 3: Descent
+3405::MX vs. ATV Reflex
+3406::Blood Bowl® Legendary Edition
+3407::Winter Voices
+3408::NyxQuest: Kindred Spirits
+3409::Lost Horizon
+3410::Aura: Fate of the Ages
+3411::Lugaru HD
+3412::Diamond Dan
+3413::Power of Defense
+3414::The Guild II
+3415::Theatre of War
+3416::Theatre of War 2: Kursk 1943
+3417::Day of Defeat: Source
+3418::Blacklight: Tango Down
+3419::Future Wars
+3420::Chaser
+3421::Singularity™
+3422::F.E.A.R.
+3423::Gridrunner Revolution
+3424::Dawn of Discovery™: Venice
+3425::Arsenal of Democracy: A Hearts of Iron Game
+3426::Silent Hunter 5®: Battle of the Atlantic
+3427::Shatter
+3428::Dragon Age™: Origins Awakening
+3429::Command & Conquer 4: Tiberian Twilight
+3430::Dark Fall: Lost Souls
+3431::Hazen: The Dark Whispers
+3432::Ironclads: American Civil War
+3433::Booster Trooper
+3434::Nancy Drew®: Warnings at Waverly Academy
+3435::Aliens versus Predator Classic 2000
+3436::Aliens vs. Predator™
+3437::Gyromancer
+3438::Rogue Warrior
+3439::King's Bounty: Armored Princess
+3440::Broken Sword 3 - the Sleeping Dragon
+3441::Broken Sword 2 - the Smoking Mirror: Remastered
+3442::Altitude
+3443::9th Company: Roots Of Terror
+3444::Samorost 2
+3445::Nancy Drew®: The Haunted Carousel
+3446::Sacraboar
+3447::Nancy Drew®: Danger on Deception Island
+3448::Borderlands
+3449::Big Brain Wolf
+3450::Lucidity™
+3451::Red Faction Guerrilla Steam Edition
+3452::Zuma's Revenge!
+3453::STAR WARS™ - Dark Forces
+3454::STAR WARS™ Jedi Knight - Mysteries of the Sith™
+3455::Twin Sector
+3456::MDK
+3457::AaAaAA!!! - A Reckless Disregard for Gravity
+3458::MDK 2
+3459::Nation Red
+3460::Red Faction
+3461::Red Faction II
+3462::Batman: Arkham Asylum Game of the Year Edition
+3463::Cooking Dash®
+3464::Chocolatier®: Decadence by Design™
+3465::East India Company
+3466::Sniper Elite
+3467::Bad Rats: the Rats' Revenge
+3468::The Dig®
+3469::Supreme Ruler 2020 Gold
+3470::Delta Force: Task Force Dagger
+3471::Tachyon: The Fringe
+3472::Delta Force
+3473::Delta Force — Black Hawk Down: Team Sabre
+3474::Arma 2
+3475::Trine Enchanted Edition
+3476::Overlord II
+3477::Lunnye Devitsy
+3478::F-16 Multirole Fighter
+3479::Crayon Physics Deluxe
+3480::Alien Shooter: Revisited
+3481::Zombie Shooter
+3482::Restaurant Empire II
+3483::Mightier
+3484::Dark Sector
+3485::The Path
+3486::COIL
+3487::Caster
+3488::Eternal Silence
+3489::Eschalon: Book I
+3490::Spectromancer
+3491::Hospital Tycoon
+3492::Tom Clancy's Splinter Cell Double Agent®
+3493::Driver® Parallel Lines
+3494::Warhammer 40,000: Dawn of War II
+3495::Puzzle Quest: Galactrix
+3496::SlamIt Pinball Big Score
+3497::Tom Clancy's EndWar™
+3498::Grand Ages: Rome
+3499::Monster Trucks Nitro
+3500::Europa Universalis III Complete
+3501::Exodus from the Earth
+3502::Starscape
+3503::Precipice of Darkness, Episode Two
+3504::7 Wonders: Treasures of Seven
+3505::Left 4 Dead
+3506::Prince of Persia: Warrior Within™
+3507::Prince of Persia®: The Sands of Time
+3508::Prince of Persia®
+3509::Need for Speed Undercover
+3510::Mass Effect
+3511::SPORE™
+3512::Imperium Romanum Gold Edition
+3513::INSURGENCY: Modern Infantry Combat
+3514::Age of Chivalry
+3515::D.I.P.R.I.P. Warm Up
+3516::Zombie Panic! Source
+3517::Insecticide Part 1
+3518::Sacred Gold
+3519::Tom Clancy's Ghost Recon® Desert Siege™
+3520::Tom Clancy's Ghost Recon®
+3521::Silverfall: Earth Awakening
+3522::Assassin's Creed™: Director's Cut Edition
+3523::Trackmania United Forever Star Edition
+3524::Virtual Villagers: A New Home
+3525::Cold Fear™
+3526::Tom Clancy's Rainbow Six Lockdown™
+3527::Precipice of Darkness, Episode One
+3528::Warhammer® 40,000: Dawn of War® - Soulstorm
+3529::Safecracker: The Ultimate Puzzle Adventure
+3530::Full Spectrum Warrior: Ten Hammers
+3531::ThreadSpace: Hyperbol
+3532::Full Spectrum Warrior
+3533::DOOM 3
+3534::Quake III Arena
+3535::QUAKE II Mission Pack: Ground Zero
+3536::HeXen: Deathkings of the Dark Citadel
+3537::QUAKE III: Team Arena
+3538::Painkiller Overdose
+3539::Call of Juarez™
+3540::Call of Duty® 4: Modern Warfare®
+3541::Amazing Adventures The Lost Tomb™
+3542::Heretic: Shadow of the Serpent Riders
+3543::Sam & Max 105: Reality 2.0
+3544::Geometry Wars: Retro Evolved
+3545::Just Cause
+3546::Hitman: Codename 47
+3547::Commandos 2: Men of Courage
+3548::Thief: Deadly Shadows
+3549::Deus Ex: Game of the Year Edition
+3550::Jade Empire™: Special Edition
+3551::Peggle Deluxe
+3552::Silverfall
+3553::Runaway, The Dream of The Turtle
+3554::GUN™
+3555::Call of Duty: United Offensive
+3556::Sid Meier's Civilization® III Complete
+3557::Shattered Union
+3558::Medieval II: Total War™
+3559::Heroes of Annihilated Empires
+3560::Global Adventures
+3561::Six Days of Snow
+3562::Bombernauts
+3563::Bloodworks
+3564::Sid Meier's Pirates!
+3565::Half-Life 2: Lost Coast
+3566::Red Orchestra: Ostfront 41-45
+3567::Space Empires IV Deluxe
+3568::Earth 2160
+3569::Half-Life Deathmatch: Source
+3570::SiN Episodes: Emergence
+3571::Typer Shark! Deluxe
+3572::Bejeweled Deluxe
+3573::Dynomite Deluxe
+3574::AstroPop Deluxe
+3575::Zuma Deluxe
+3576::Insaniquarium Deluxe
+3577::PLAYERUNKN4WN: Zombie
+3578::Gothic 1
+3579::Geneforge 1
+3580::Counter-Strike: Condition Zero

data/ref/steam/test.ipynb

@@ -0,0 +1,190 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "                                                      seq  next  len_seq  \\\n",
+      "0       [3442, 3448, 3451, 97, 180, 78, 3300, 152, 340...  3462       10   \n",
+      "1       [3448, 3451, 97, 180, 78, 3300, 152, 3401, 185...  3354       10   \n",
+      "2       [3451, 97, 180, 78, 3300, 152, 3401, 185, 3462...   274       10   \n",
+      "3       [97, 180, 78, 3300, 152, 3401, 185, 3462, 3354...  3276       10   \n",
+      "4       [180, 78, 3300, 152, 3401, 185, 3462, 3354, 27...  3225       10   \n",
+      "...                                                   ...   ...      ...   \n",
+      "151051  [129, 2411, 1033, 204, 498, 3242, 3505, 3224, ...  1734       10   \n",
+      "151052  [2411, 1033, 204, 498, 3242, 3505, 3224, 2524,...  1945       10   \n",
+      "151053  [1033, 204, 498, 3242, 3505, 3224, 2524, 1099,...  1134       10   \n",
+      "151054  [3431, 2808, 3214, 3284, 3001, 800, 2636, 3581...  2636        7   \n",
+      "151055  [14, 58, 3510, 15, 68, 86, 3238, 2411, 1134, 2...  1073       10   \n",
+      "\n",
+      "                                                   review  \\\n",
+      "0       [Maybe a niche thing, but it has solid gamepla...   \n",
+      "1       [Borderlands at its core is spectacular: the g...   \n",
+      "2       [You get to be the hero of the people WHILE bl...   \n",
+      "3       [The game is far from perfect, I could point o...   \n",
+      "4       [Very instructive', \"There's a ton of stuff wr...   \n",
+      "...                                                   ...   \n",
+      "151051  [This game is the best when I was bored playin...   \n",
+      "151052  [I don't even know where to begin on how good ...   \n",
+      "151053  [BrainBread 2 has really fun single player bec...   \n",
+      "151054  [Starting the game, you will go into an area w...   \n",
+      "151055  [Think the original Doom but with more awesome...   \n",
+      "\n",
+      "                                              next_review  \n",
+      "0       I played to the end, I feel obligated to recom...  \n",
+      "1       I was pretty shocked by how much I liked this ...  \n",
+      "2       words cant explain how good this game is GET I...  \n",
+      "3       best game i ever bought a rts with actual play...  \n",
+      "4       Extremely punishing game, but every death is p...  \n",
+      "...                                                   ...  \n",
+      "151051            I had this game since it first came out  \n",
+      "151052    give me a reason to actually hate white people.  \n",
+      "151053  Heroes and Generals is a really fun game but r...  \n",
+      "151054  The game is fun it's a cool concept, and I lik...  \n",
+      "151055  It's Wasted Potential but still a pretty good ...  \n",
+      "\n",
+      "[151056 rows x 5 columns]\n"
+     ]
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "\n",
+    "# 读取pickle文件\n",
+    "df = pd.read_pickle(\"./train_data.df\")\n",
+    "\n",
+    "# 打印数据框\n",
+    "print(df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "                                                    seq  next  len_seq  \\\n",
+      "0     [1221, 899, 490, 1985, 255, 3058, 3119, 2930, ...  1417       10   \n",
+      "1     [3448, 855, 2710, 6, 2694, 3581, 3581, 3581, 3...  2694        5   \n",
+      "2     [3255, 3399, 3330, 3500, 2729, 1335, 68, 1866,...   642       10   \n",
+      "3     [273, 3462, 2656, 2033, 3333, 3284, 2110, 470,...   470        8   \n",
+      "4     [3284, 1134, 2992, 727, 3442, 831, 6, 76, 2899...  2899        9   \n",
+      "...                                                 ...   ...      ...   \n",
+      "1189  [6, 2986, 2683, 495, 3580, 3171, 234, 164, 2, ...  2853       10   \n",
+      "1190  [2262, 2694, 1358, 883, 2174, 6, 2683, 1604, 4...  1478       10   \n",
+      "1191  [150, 1099, 3488, 3580, 2725, 3171, 535, 1622,...   206       10   \n",
+      "1192  [206, 6, 52, 3272, 204, 926, 498, 2712, 2496, ...  3202       10   \n",
+      "1193  [3251, 3300, 1822, 150, 2160, 2046, 6, 1038, 2...  1602       10   \n",
+      "\n",
+      "                                                 review  \\\n",
+      "0     [Really nice game, Would recommend!, meh, This...   \n",
+      "1     [finished it, its good but after i finished it...   \n",
+      "2     [Paradox Interactive\\tdouble the price for alm...   \n",
+      "3     [Nice, Combat system can be better, All the fu...   \n",
+      "4     [9/10, 10/10, 9/10, 10/10, -99999/10, 9/10, id...   \n",
+      "...                                                 ...   \n",
+      "1189  [RDM simulator, 192 cubic tonnes of mud simula...   \n",
+      "1190  [do I have a brain?, 18+, relationship of Park...   \n",
+      "1191  [For DC, Любителям Аниме., 123, Ну такое., Hor...   \n",
+      "1192  [Peed all over my dog then I started the game....   \n",
+      "1193  [Great for a few hours until you beat it., Fun...   \n",
+      "\n",
+      "                                            next_review  \n",
+      "0     HUGE improvements on graphics and gameplay and...  \n",
+      "1                                                   ok.  \n",
+      "2                                              Unstable  \n",
+      "3     You've probably already heard that this game i...  \n",
+      "4                                                 10/10  \n",
+      "...                                                 ...  \n",
+      "1189                      best game ever made simulator  \n",
+      "1190                                new vision, new way  \n",
+      "1191    Кратко: мужик живёт с мамой в маленьком фургоне  \n",
+      "1192  Story mode kinda sucks but the rest of the gam...  \n",
+      "1193  While being free-to-play, Robocraft seems very...  \n",
+      "\n",
+      "[1194 rows x 5 columns]\n"
+     ]
+    }
+   ],
+   "source": [
+    "# 读取pickle文件\n",
+    "df = pd.read_pickle(\"./Test_data.df\")\n",
+    "\n",
+    "# 打印数据框\n",
+    "print(df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "interaction_pairs = []\n",
+    "\n",
+    "for idx, row in df.iterrows():\n",
+    "    user_id = idx  # 如果你有显式的用户ID，这里应该使用那个用户ID\n",
+    "    items = row['seq'] + [row['next']]\n",
+    "    for item in items:\n",
+    "        interaction_pairs.append((user_id, item))\n",
+    "\n",
+    "# 转换成DataFrame\n",
+    "interaction_df = pd.DataFrame(interaction_pairs, columns=['user_id', 'item_id'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Interaction Density: 0.0031\n"
+     ]
+    }
+   ],
+   "source": [
+    "# 创建用户-项矩阵\n",
+    "user_item_matrix = pd.crosstab(interaction_df['user_id'], interaction_df['item_id'])\n",
+    "\n",
+    "# 计算非零元素的比例来确定稠密度\n",
+    "non_zero_count = user_item_matrix.astype(bool).sum().sum()  # 计算非零元素的总数\n",
+    "total_elements = user_item_matrix.size  # 矩阵中元素的总数\n",
+    "\n",
+    "interaction_density = non_zero_count / total_elements\n",
+    "print(f\"Interaction Density: {interaction_density:.4f}\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "MOE4REC",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.19"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

data/ref/steam/train_data.df

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90f8e2ec827427d271cf0b03db45ed5ac3231840fbd022ac57cfae6061ed5b63
+size 7322362

data/steam_data.py

@@ -0,0 +1,103 @@
+import torch
+import os.path as op
+import numpy as np
+import pickle as pkl
+import torch.utils.data as data
+
+import pandas as pd
+import random
+
+class SteamData(data.Dataset):
+    def __init__(self, data_dir=r'data/ref/steam',
+                 stage=None,
+                 cans_num=10,
+                 sep=", ",
+                 no_augment=True):
+        self.__dict__.update(locals())
+        self.aug = (stage=='train') and not no_augment
+        self.padding_item_id=3581
+        self.check_files()
+
+    # 返回session_data['seq']长度
+    def __len__(self):
+        return len(self.session_data['seq'])
+
+    # 获取索引i的样本(批次i)
+    def __getitem__(self, i):
+        temp = self.session_data.iloc[i]
+        candidates = self.negative_sampling(temp['seq_unpad'],temp['next'])
+        cans_name=[self.item_id2name[can] for can in candidates]
+        sample = {
+            'seq': temp['seq'],
+            'seq_name': temp['seq_title'],
+            'len_seq': temp['len_seq'],
+            'seq_str': self.sep.join(temp['seq_title']),
+            'cans': candidates,
+            'cans_name': cans_name,
+            'cans_str': self.sep.join(cans_name),
+            'len_cans': self.cans_num,
+            'item_id': temp['next'],
+            'item_name': temp['next_item_name'],
+            'correct_answer': temp['next_item_name']
+        }
+        return sample
+    
+    # 进行负采样, 返回序列ID列表
+    def negative_sampling(self,seq_unpad,next_item):
+        # canset: 所有游戏id中不在seq_unpad中的游戏id
+        canset=[i for i in list(self.item_id2name.keys()) if i not in seq_unpad and i!=next_item]
+        # 随机选择cans_num-1个游戏id, 加上next_item
+        candidates=random.sample(canset, self.cans_num-1)+[next_item]
+        random.shuffle(candidates)
+        return candidates  
+
+    # 检查并加载数据文件
+    def check_files(self):
+        self.item_id2name=self.get_game_id2name()
+        if self.stage=='train':
+            filename="train_data.df"
+        elif self.stage=='val':
+            filename="Val_data.df"
+        elif self.stage=='test':
+            filename="Test_data.df"
+        data_path=op.join(self.data_dir, filename)
+        # 根据data_path和id2name字典加载数据
+        self.session_data = self.session_data4frame(data_path, self.item_id2name)  
+
+    # 获取游戏id到游戏名的映射, 返回字典
+    def get_game_id2name(self):
+        game_id2name = dict()
+        item_path=op.join(self.data_dir, 'id2name.txt')
+        with open(item_path, 'r') as f:
+            for l in f.readlines():
+                ll = l.strip('\n').split('::')
+                game_id2name[int(ll[0])] = ll[1].strip()
+        return game_id2name
+    
+    # 对数据进行预处理
+    def session_data4frame(self, datapath, game_id2name):
+        # 根据datapath读取pd数据
+        train_data = pd.read_pickle(datapath)
+        train_data = train_data[train_data['len_seq'] >= 3]
+        # 从序列中移除填充项
+        def remove_padding(xx):
+            x = xx[:]
+            for i in range(10):
+                try:
+                    x.remove(self.padding_item_id)
+                except:
+                    break
+            return x
+        # 去除pad的train_data序列 -> train_data['seq_unpad']
+        train_data['seq_unpad'] = train_data['seq'].apply(remove_padding)
+        # 序列号 -> 游戏名
+        def seq_to_title(x): 
+            return [game_id2name[x_i] for x_i in x]
+        # 转换train_data ID序列为游戏名序列 -> train_data['seq_title']
+        train_data['seq_title'] = train_data['seq_unpad'].apply(seq_to_title)
+        # 单个序列 -> 游戏名
+        def next_item_title(x): 
+            return game_id2name[x]
+        # 转换train_data['next'] ID序列为游戏名序列 -> train_data['next_item_name']
+        train_data['next_item_name'] = train_data['next'].apply(next_item_title)
+        return train_data

debug/modeling_llama.py

@@ -0,0 +1,886 @@
+# coding=utf-8
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
+#
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
+# and OPT implementations in this library. It has been modified from its
+# original forms to accommodate minor architectural differences compared
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" PyTorch LLaMA model."""
+import math
+from typing import List, Optional, Tuple, Union
+
+import torch
+import torch.utils.checkpoint
+from torch import nn
+from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
+
+from ...activations import ACT2FN
+from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast, SequenceClassifierOutputWithPast
+from ...modeling_utils import PreTrainedModel
+from ...utils import add_start_docstrings, add_start_docstrings_to_model_forward, logging, replace_return_docstrings
+from .configuration_llama import LlamaConfig
+
+
+logger = logging.get_logger(__name__)
+
+_CONFIG_FOR_DOC = "LlamaConfig"
+
+
+# Copied from transformers.models.bart.modeling_bart._make_causal_mask
+def _make_causal_mask(
+    input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0
+):
+    """
+    Make causal mask used for bi-directional self-attention.
+    """
+    bsz, tgt_len = input_ids_shape
+    mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min, device=device), device=device)
+    mask_cond = torch.arange(mask.size(-1), device=device)
+    mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0)
+    mask = mask.to(dtype)
+
+    if past_key_values_length > 0:
+        mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1)
+    return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length)
+
+
+# Copied from transformers.models.bart.modeling_bart._expand_mask
+def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None):
+    """
+    Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`.
+    """
+    bsz, src_len = mask.size()
+    tgt_len = tgt_len if tgt_len is not None else src_len
+
+    expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype)
+
+    inverted_mask = 1.0 - expanded_mask
+
+    return inverted_mask.masked_fill(inverted_mask.to(torch.bool), torch.finfo(dtype).min)
+
+
+class LlamaRMSNorm(nn.Module):
+    def __init__(self, hidden_size, eps=1e-6):
+        """
+        LlamaRMSNorm is equivalent to T5LayerNorm
+        """
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(hidden_size))
+        self.variance_epsilon = eps
+
+    def forward(self, hidden_states):
+        variance = hidden_states.to(torch.float32).pow(2).mean(-1, keepdim=True)
+        hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
+
+        # convert into half-precision if necessary
+        if self.weight.dtype in [torch.float16, torch.bfloat16]:
+            hidden_states = hidden_states.to(self.weight.dtype)
+
+        return self.weight * hidden_states
+
+
+class LlamaRotaryEmbedding(torch.nn.Module):
+    def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None):
+        super().__init__()
+        inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2).float().to(device) / dim))
+        self.register_buffer("inv_freq", inv_freq)
+
+        # Build here to make `torch.jit.trace` work.
+        self.max_seq_len_cached = max_position_embeddings
+        t = torch.arange(self.max_seq_len_cached, device=self.inv_freq.device, dtype=self.inv_freq.dtype)
+        freqs = torch.einsum("i,j->ij", t, self.inv_freq)
+        # Different from paper, but it uses a different permutation in order to obtain the same calculation
+        emb = torch.cat((freqs, freqs), dim=-1)
+        self.register_buffer("cos_cached", emb.cos()[None, None, :, :], persistent=False)
+        self.register_buffer("sin_cached", emb.sin()[None, None, :, :], persistent=False)
+
+    def forward(self, x, seq_len=None):
+        # x: [bs, num_attention_heads, seq_len, head_size]
+        # This `if` block is unlikely to be run after we build sin/cos in `__init__`. Keep the logic here just in case.
+        if seq_len > self.max_seq_len_cached:
+            self.max_seq_len_cached = seq_len
+            t = torch.arange(self.max_seq_len_cached, device=x.device, dtype=self.inv_freq.dtype)
+            freqs = torch.einsum("i,j->ij", t, self.inv_freq)
+            # Different from paper, but it uses a different permutation in order to obtain the same calculation
+            emb = torch.cat((freqs, freqs), dim=-1).to(x.device)
+            self.register_buffer("cos_cached", emb.cos()[None, None, :, :], persistent=False)
+            self.register_buffer("sin_cached", emb.sin()[None, None, :, :], persistent=False)
+        return (
+            self.cos_cached[:, :, :seq_len, ...].to(dtype=x.dtype),
+            self.sin_cached[:, :, :seq_len, ...].to(dtype=x.dtype),
+        )
+
+
+def rotate_half(x):
+    """Rotates half the hidden dims of the input."""
+    x1 = x[..., : x.shape[-1] // 2]
+    x2 = x[..., x.shape[-1] // 2 :]
+    return torch.cat((-x2, x1), dim=-1)
+
+
+def apply_rotary_pos_emb(q, k, cos, sin, position_ids):
+    gather_indices = position_ids[:, None, :, None]  # [bs, 1, seq_len, 1]
+    gather_indices = gather_indices.repeat(1, cos.shape[1], 1, cos.shape[3])
+    cos = torch.gather(cos.repeat(gather_indices.shape[0], 1, 1, 1), 2, gather_indices)
+    sin = torch.gather(sin.repeat(gather_indices.shape[0], 1, 1, 1), 2, gather_indices)
+    q_embed = (q * cos) + (rotate_half(q) * sin)
+    k_embed = (k * cos) + (rotate_half(k) * sin)
+    return q_embed, k_embed
+
+
+class LlamaMLP(nn.Module):
+    def __init__(
+        self,
+        hidden_size: int,
+        intermediate_size: int,
+        hidden_act: str,
+    ):
+        super().__init__()
+        self.gate_proj = nn.Linear(hidden_size, intermediate_size, bias=False)
+        self.down_proj = nn.Linear(intermediate_size, hidden_size, bias=False)
+        self.up_proj = nn.Linear(hidden_size, intermediate_size, bias=False)
+        self.act_fn = ACT2FN[hidden_act]
+
+    def forward(self, x):
+        return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
+
+
+class LlamaAttention(nn.Module):
+    """Multi-headed attention from 'Attention Is All You Need' paper"""
+
+    def __init__(self, config: LlamaConfig):
+        super().__init__()
+        self.config = config
+        self.hidden_size = config.hidden_size
+        self.num_heads = config.num_attention_heads
+        self.head_dim = self.hidden_size // self.num_heads
+        self.max_position_embeddings = config.max_position_embeddings
+
+        if (self.head_dim * self.num_heads) != self.hidden_size:
+            raise ValueError(
+                f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}"
+                f" and `num_heads`: {self.num_heads})."
+            )
+        self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False)
+        self.k_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False)
+        self.v_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False)
+        self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False)
+        self.rotary_emb = LlamaRotaryEmbedding(self.head_dim, max_position_embeddings=self.max_position_embeddings)
+
+    def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
+        return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous()
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_value: Optional[Tuple[torch.Tensor]] = None,
+        output_attentions: bool = False,
+        use_cache: bool = False,
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+        bsz, q_len, _ = hidden_states.size()
+
+        query_states = self.q_proj(hidden_states).view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+        key_states = self.k_proj(hidden_states).view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+        value_states = self.v_proj(hidden_states).view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+
+        kv_seq_len = key_states.shape[-2]
+        if past_key_value is not None:
+            kv_seq_len += past_key_value[0].shape[-2]
+        cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
+        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)
+        # [bsz, nh, t, hd]
+
+        if past_key_value is not None:
+            # reuse k, v, self_attention
+            key_states = torch.cat([past_key_value[0], key_states], dim=2)
+            value_states = torch.cat([past_key_value[1], value_states], dim=2)
+
+        past_key_value = (key_states, value_states) if use_cache else None
+
+        attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim)
+
+        if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len):
+            raise ValueError(
+                f"Attention weights should be of size {(bsz * self.num_heads, q_len, kv_seq_len)}, but is"
+                f" {attn_weights.size()}"
+            )
+
+        if attention_mask is not None:
+            if attention_mask.size() != (bsz, 1, q_len, kv_seq_len):
+                raise ValueError(
+                    f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}"
+                )
+            attn_weights = attn_weights + attention_mask
+            attn_weights = torch.max(attn_weights, torch.tensor(torch.finfo(attn_weights.dtype).min))
+
+        # upcast attention to fp32
+        attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)
+        attn_output = torch.matmul(attn_weights, value_states)
+
+        if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim):
+            raise ValueError(
+                f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is"
+                f" {attn_output.size()}"
+            )
+
+        attn_output = attn_output.transpose(1, 2)
+        attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)
+
+        attn_output = self.o_proj(attn_output)
+
+        if not output_attentions:
+            attn_weights = None
+
+        return attn_output, attn_weights, past_key_value
+
+
+class LlamaDecoderLayer(nn.Module):
+    def __init__(self, config: LlamaConfig):
+        super().__init__()
+        self.hidden_size = config.hidden_size
+        self.self_attn = LlamaAttention(config=config)
+        self.mlp = LlamaMLP(
+            hidden_size=self.hidden_size,
+            intermediate_size=config.intermediate_size,
+            hidden_act=config.hidden_act,
+        )
+        self.input_layernorm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.post_attention_layernorm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_value: Optional[Tuple[torch.Tensor]] = None,
+        output_attentions: Optional[bool] = False,
+        use_cache: Optional[bool] = False,
+    ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]:
+        """
+        Args:
+            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
+            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
+                `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
+            output_attentions (`bool`, *optional*):
+                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
+                returned tensors for more detail.
+            use_cache (`bool`, *optional*):
+                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
+                (see `past_key_values`).
+            past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states
+        """
+
+        residual = hidden_states
+
+        hidden_states = self.input_layernorm(hidden_states)
+
+        # Self Attention
+        hidden_states, self_attn_weights, present_key_value = self.self_attn(
+            hidden_states=hidden_states,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_value=past_key_value,
+            output_attentions=output_attentions,
+            use_cache=use_cache,
+        )
+        hidden_states = residual + hidden_states
+
+        # Fully Connected
+        residual = hidden_states
+        hidden_states = self.post_attention_layernorm(hidden_states)
+        hidden_states = self.mlp(hidden_states)
+        hidden_states = residual + hidden_states
+
+        outputs = (hidden_states,)
+
+        if output_attentions:
+            outputs += (self_attn_weights,)
+
+        if use_cache:
+            outputs += (present_key_value,)
+
+        return outputs
+
+
+LLAMA_START_DOCSTRING = r"""
+    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+    etc.)
+
+    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+    and behavior.
+
+    Parameters:
+        config ([`LlamaConfig`]):
+            Model configuration class with all the parameters of the model. Initializing with a config file does not
+            load the weights associated with the model, only the configuration. Check out the
+            [`~PreTrainedModel.from_pretrained`] method to load the model weights.
+"""
+
+
+@add_start_docstrings(
+    "The bare LLaMA Model outputting raw hidden-states without any specific head on top.",
+    LLAMA_START_DOCSTRING,
+)
+class LlamaPreTrainedModel(PreTrainedModel):
+    config_class = LlamaConfig
+    base_model_prefix = "model"
+    supports_gradient_checkpointing = True
+    _no_split_modules = ["LlamaDecoderLayer"]
+    _keys_to_ignore_on_load_unexpected = [r"decoder\.version"]
+
+    def _init_weights(self, module):
+        std = self.config.initializer_range
+        if isinstance(module, nn.Linear):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.bias is not None:
+                module.bias.data.zero_()
+        elif isinstance(module, nn.Embedding):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.padding_idx is not None:
+                module.weight.data[module.padding_idx].zero_()
+
+    def _set_gradient_checkpointing(self, module, value=False):
+        if isinstance(module, LlamaModel):
+            module.gradient_checkpointing = value
+
+
+LLAMA_INPUTS_DOCSTRING = r"""
+    Args:
+        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
+            Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
+            it.
+
+            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+            [`PreTrainedTokenizer.__call__`] for details.
+
+            [What are input IDs?](../glossary#input-ids)
+        attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
+
+            - 1 for tokens that are **not masked**,
+            - 0 for tokens that are **masked**.
+
+            [What are attention masks?](../glossary#attention-mask)
+
+            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+            [`PreTrainedTokenizer.__call__`] for details.
+
+            If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
+            `past_key_values`).
+
+            If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`]
+            and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more
+            information on the default strategy.
+
+            - 1 indicates the head is **not masked**,
+            - 0 indicates the head is **masked**.
+        position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+            config.n_positions - 1]`.
+
+            [What are position IDs?](../glossary#position-ids)
+        past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
+            Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
+            `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape
+            `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
+
+            Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
+            blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
+
+            If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+            don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+            `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
+            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+            model's internal embedding lookup matrix.
+        use_cache (`bool`, *optional*):
+            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+            `past_key_values`).
+        output_attentions (`bool`, *optional*):
+            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
+            tensors for more detail.
+        output_hidden_states (`bool`, *optional*):
+            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
+            more detail.
+        return_dict (`bool`, *optional*):
+            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+"""
+
+
+@add_start_docstrings(
+    "The bare LLaMA Model outputting raw hidden-states without any specific head on top.",
+    LLAMA_START_DOCSTRING,
+)
+class LlamaModel(LlamaPreTrainedModel):
+    """
+    Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`LlamaDecoderLayer`]
+
+    Args:
+        config: LlamaConfig
+    """
+
+    def __init__(self, config: LlamaConfig):
+        super().__init__(config)
+        self.padding_idx = config.pad_token_id
+        self.vocab_size = config.vocab_size
+
+        self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
+        self.layers = nn.ModuleList([LlamaDecoderLayer(config) for _ in range(config.num_hidden_layers)])
+        self.norm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+
+        self.gradient_checkpointing = False
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.embed_tokens
+
+    def set_input_embeddings(self, value):
+        self.embed_tokens = value
+
+    # Copied from transformers.models.bart.modeling_bart.BartDecoder._prepare_decoder_attention_mask
+    def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_embeds, past_key_values_length):
+        # create causal mask
+        # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
+        combined_attention_mask = None
+        if input_shape[-1] > 1:
+            combined_attention_mask = _make_causal_mask(
+                input_shape,
+                inputs_embeds.dtype,
+                device=inputs_embeds.device,
+                past_key_values_length=past_key_values_length,
+            )
+
+        if attention_mask is not None:
+            # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
+            expanded_attn_mask = _expand_mask(attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]).to(
+                inputs_embeds.device
+            )
+            combined_attention_mask = (
+                expanded_attn_mask if combined_attention_mask is None else expanded_attn_mask + combined_attention_mask
+            )
+
+        return combined_attention_mask
+
+    @add_start_docstrings_to_model_forward(LLAMA_INPUTS_DOCSTRING)
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, BaseModelOutputWithPast]:
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        use_cache = use_cache if use_cache is not None else self.config.use_cache
+
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        # retrieve input_ids and inputs_embeds
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError("You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time")
+        elif input_ids is not None:
+            batch_size, seq_length = input_ids.shape
+        elif inputs_embeds is not None:
+            batch_size, seq_length, _ = inputs_embeds.shape
+        else:
+            raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds")
+
+        seq_length_with_past = seq_length
+        past_key_values_length = 0
+
+        if past_key_values is not None:
+            past_key_values_length = past_key_values[0][0].shape[2]
+            seq_length_with_past = seq_length_with_past + past_key_values_length
+
+        if position_ids is None:
+            device = input_ids.device if input_ids is not None else inputs_embeds.device
+            position_ids = torch.arange(
+                past_key_values_length, seq_length + past_key_values_length, dtype=torch.long, device=device
+            )
+            position_ids = position_ids.unsqueeze(0).view(-1, seq_length)
+        else:
+            position_ids = position_ids.view(-1, seq_length).long()
+
+        if inputs_embeds is None:
+            inputs_embeds = self.embed_tokens(input_ids)
+        # embed positions
+        if attention_mask is None:
+            attention_mask = torch.ones(
+                (batch_size, seq_length_with_past), dtype=torch.bool, device=inputs_embeds.device
+            )
+        attention_mask = self._prepare_decoder_attention_mask(
+            attention_mask, (batch_size, seq_length), inputs_embeds, past_key_values_length
+        )
+
+        hidden_states = inputs_embeds
+
+        if self.gradient_checkpointing and self.training:
+            if use_cache:
+                logger.warning_once(
+                    "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
+                )
+                use_cache = False
+
+        # decoder layers
+        all_hidden_states = () if output_hidden_states else None
+        all_self_attns = () if output_attentions else None
+        next_decoder_cache = () if use_cache else None
+
+        for idx, decoder_layer in enumerate(self.layers):
+            if output_hidden_states:
+                all_hidden_states += (hidden_states,)
+
+            past_key_value = past_key_values[idx] if past_key_values is not None else None
+
+            if self.gradient_checkpointing and self.training:
+
+                def create_custom_forward(module):
+                    def custom_forward(*inputs):
+                        # None for past_key_value
+                        return module(*inputs, output_attentions, None)
+
+                    return custom_forward
+
+                layer_outputs = torch.utils.checkpoint.checkpoint(
+                    create_custom_forward(decoder_layer),
+                    hidden_states,
+                    attention_mask,
+                    position_ids,
+                    None,
+                )
+            else:
+                layer_outputs = decoder_layer(
+                    hidden_states,
+                    attention_mask=attention_mask,
+                    position_ids=position_ids,
+                    past_key_value=past_key_value,
+                    output_attentions=output_attentions,
+                    use_cache=use_cache,
+                )
+
+            hidden_states = layer_outputs[0]
+
+            if use_cache:
+                next_decoder_cache += (layer_outputs[2 if output_attentions else 1],)
+
+            if output_attentions:
+                all_self_attns += (layer_outputs[1],)
+
+        hidden_states = self.norm(hidden_states)
+
+        # add hidden states from the last decoder layer
+        if output_hidden_states:
+            all_hidden_states += (hidden_states,)
+
+        next_cache = next_decoder_cache if use_cache else None
+        if not return_dict:
+            return tuple(v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] if v is not None)
+        return BaseModelOutputWithPast(
+            last_hidden_state=hidden_states,
+            past_key_values=next_cache,
+            hidden_states=all_hidden_states,
+            attentions=all_self_attns,
+        )
+
+
+class LlamaForCausalLM(LlamaPreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+        self.model = LlamaModel(config)
+
+        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.model.embed_tokens
+
+    def set_input_embeddings(self, value):
+        self.model.embed_tokens = value
+
+    def get_output_embeddings(self):
+        return self.lm_head
+
+    def set_output_embeddings(self, new_embeddings):
+        self.lm_head = new_embeddings
+
+    def set_decoder(self, decoder):
+        self.model = decoder
+
+    def get_decoder(self):
+        return self.model
+
+    @add_start_docstrings_to_model_forward(LLAMA_INPUTS_DOCSTRING)
+    @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC)
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+        # !!!
+        **kwargs
+    ) -> Union[Tuple, CausalLMOutputWithPast]:
+        r"""
+        Args:
+            labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+                Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+                config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+                (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
+
+        Returns:
+
+        Example:
+
+        ```python
+        >>> from transformers import AutoTokenizer, LlamaForCausalLM
+
+        >>> model = LlamaForCausalLM.from_pretrained(PATH_TO_CONVERTED_WEIGHTS)
+        >>> tokenizer = AutoTokenizer.from_pretrained(PATH_TO_CONVERTED_TOKENIZER)
+
+        >>> prompt = "Hey, are you consciours? Can you talk to me?"
+        >>> inputs = tokenizer(prompt, return_tensors="pt")
+
+        >>> # Generate
+        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
+        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
+        "Hey, are you consciours? Can you talk to me?\nI'm not consciours, but I can talk to you."
+        ```"""
+
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
+        outputs = self.model(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        hidden_states = outputs[0]
+        logits = self.lm_head(hidden_states)
+
+        loss = None
+        if labels is not None:
+            # Shift so that tokens < n predict n
+            shift_logits = logits[..., :-1, :].contiguous()
+            shift_labels = labels[..., 1:].contiguous()
+            # Flatten the tokens
+            loss_fct = CrossEntropyLoss()
+            shift_logits = shift_logits.view(-1, self.config.vocab_size)
+            shift_labels = shift_labels.view(-1)
+            # Enable model parallelism
+            shift_labels = shift_labels.to(shift_logits.device)
+            loss = loss_fct(shift_logits, shift_labels)
+
+        if not return_dict:
+            output = (logits,) + outputs[1:]
+            return (loss,) + output if loss is not None else output
+
+        return CausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+    def prepare_inputs_for_generation(
+        self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs
+    ):
+        if past_key_values:
+            input_ids = input_ids[:, -1:]
+
+        position_ids = kwargs.get("position_ids", None)
+        if attention_mask is not None and position_ids is None:
+            # create position_ids on the fly for batch generation
+            position_ids = attention_mask.long().cumsum(-1) - 1
+            position_ids.masked_fill_(attention_mask == 0, 1)
+            if past_key_values:
+                position_ids = position_ids[:, -1].unsqueeze(-1)
+
+        # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
+        if inputs_embeds is not None and past_key_values is None:
+            model_inputs = {"inputs_embeds": inputs_embeds}
+        else:
+            model_inputs = {"input_ids": input_ids}
+
+        model_inputs.update(
+            {
+                "position_ids": position_ids,
+                "past_key_values": past_key_values,
+                "use_cache": kwargs.get("use_cache"),
+                "attention_mask": attention_mask,
+            }
+        )
+        return model_inputs
+
+    @staticmethod
+    def _reorder_cache(past_key_values, beam_idx):
+        reordered_past = ()
+        for layer_past in past_key_values:
+            reordered_past += (tuple(past_state.index_select(0, beam_idx) for past_state in layer_past),)
+        return reordered_past
+
+
+@add_start_docstrings(
+    """
+    The LLaMa Model transformer with a sequence classification head on top (linear layer).
+
+    [`LlamaForSequenceClassification`] uses the last token in order to do the classification, as other causal models
+    (e.g. GPT-2) do.
+
+    Since it does classification on the last token, it requires to know the position of the last token. If a
+    `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
+    no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
+    padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
+    each row of the batch).
+    """,
+    LLAMA_START_DOCSTRING,
+)
+class LlamaForSequenceClassification(LlamaPreTrainedModel):
+    _keys_to_ignore_on_load_missing = [r"lm_head.weight"]
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.num_labels = config.num_labels
+        self.model = LlamaModel(config)
+        self.score = nn.Linear(config.hidden_size, self.num_labels, bias=False)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.model.embed_tokens
+
+    def set_input_embeddings(self, value):
+        self.model.embed_tokens = value
+
+    @add_start_docstrings_to_model_forward(LLAMA_INPUTS_DOCSTRING)
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, SequenceClassifierOutputWithPast]:
+        r"""
+        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
+            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+        """
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        transformer_outputs = self.model(
+            input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+        hidden_states = transformer_outputs[0]
+        logits = self.score(hidden_states)
+
+        if input_ids is not None:
+            batch_size = input_ids.shape[0]
+        else:
+            batch_size = inputs_embeds.shape[0]
+
+        if self.config.pad_token_id is None and batch_size != 1:
+            raise ValueError("Cannot handle batch sizes > 1 if no padding token is defined.")
+        if self.config.pad_token_id is None:
+            sequence_lengths = -1
+        else:
+            if input_ids is not None:
+                sequence_lengths = (torch.ne(input_ids, self.config.pad_token_id).sum(-1) - 1).to(logits.device)
+            else:
+                sequence_lengths = -1
+
+        pooled_logits = logits[torch.arange(batch_size, device=logits.device), sequence_lengths]
+
+        loss = None
+        if labels is not None:
+            labels = labels.to(logits.device)
+            if self.config.problem_type is None:
+                if self.num_labels == 1:
+                    self.config.problem_type = "regression"
+                elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
+                    self.config.problem_type = "single_label_classification"
+                else:
+                    self.config.problem_type = "multi_label_classification"
+
+            if self.config.problem_type == "regression":
+                loss_fct = MSELoss()
+                if self.num_labels == 1:
+                    loss = loss_fct(pooled_logits.squeeze(), labels.squeeze())
+                else:
+                    loss = loss_fct(pooled_logits, labels)
+            elif self.config.problem_type == "single_label_classification":
+                loss_fct = CrossEntropyLoss()
+                loss = loss_fct(pooled_logits.view(-1, self.num_labels), labels.view(-1))
+            elif self.config.problem_type == "multi_label_classification":
+                loss_fct = BCEWithLogitsLoss()
+                loss = loss_fct(pooled_logits, labels)
+        if not return_dict:
+            output = (pooled_logits,) + transformer_outputs[1:]
+            return ((loss,) + output) if loss is not None else output
+
+        return SequenceClassifierOutputWithPast(
+            loss=loss,
+            logits=pooled_logits,
+            past_key_values=transformer_outputs.past_key_values,
+            hidden_states=transformer_outputs.hidden_states,
+            attentions=transformer_outputs.attentions,
+        )

main.py

@@ -0,0 +1,149 @@
+import os
+import pytorch_lightning as pl
+from argparse import ArgumentParser
+from pytorch_lightning import Trainer
+import pytorch_lightning.callbacks as plc
+from pytorch_lightning.loggers import TensorBoardLogger, CSVLogger
+
+from model.model_interface import MInterface
+from data.data_interface import DInterface
+from recommender.A_SASRec_final_bce_llm import SASRec, Caser, GRU
+from SASRecModules_ori import *
+from transformers import LlamaForCausalLM, LlamaTokenizer
+
+def load_callbacks(args):
+    callbacks = []
+    callbacks.append(plc.EarlyStopping(
+        monitor='metric',
+        mode='max',
+        patience=10,
+        min_delta=0.001
+    ))
+
+    callbacks.append(plc.ModelCheckpoint(
+        monitor='metric',
+        dirpath=args.ckpt_dir,
+        filename='{epoch:02d}-{metric:.3f}',
+        save_top_k=-1,
+        mode='max',
+        save_last=True,
+        #train_time_interval=args.val_check_interval
+        every_n_epochs=1
+    ))
+    
+    if args.lr_scheduler:
+        callbacks.append(plc.LearningRateMonitor(
+            logging_interval='step'))
+    return callbacks
+
+def main(args):
+    pl.seed_everything(args.seed)
+    model = MInterface(**vars(args))
+    if args.ckpt_path:
+        ckpt = torch.load(args.ckpt_path, map_location='cpu')
+
+        model.load_state_dict(ckpt['state_dict'], strict=False)
+        print("load checkpoints from {}".format(args.ckpt_path))
+
+    data_module = DInterface(llm_tokenizer=model.llama_tokenizer,**vars(args))
+
+    args.max_steps=len(data_module.trainset) * args.max_epochs // (args.accumulate_grad_batches * args.batch_size)
+    logger = TensorBoardLogger(save_dir='./log/', name=args.log_dir)
+    args.callbacks = load_callbacks(args)
+    args.logger = logger
+    if not os.path.exists(args.ckpt_dir):
+        os.makedirs(args.ckpt_dir)
+
+    trainer = Trainer.from_argparse_args(args)
+
+    if args.auto_lr_find:
+        lr_finder=trainer.tuner.lr_find(model=model, datamodule=data_module, min_lr=1e-10, max_lr=1e-3, num_training=100)
+        fig=lr_finder.plot(suggest=True)
+        fig_path="lr_finder.png"
+        fig.savefig(fig_path)
+        print("Saving to {}".format(fig_path))
+        model.hparams.lr=lr_finder.suggestion()
+
+    if args.mode == 'train':
+        trainer.fit(model=model, datamodule=data_module)
+    else:
+        trainer.test(model=model, datamodule=data_module)
+
+
+if __name__ == '__main__':
+    torch.multiprocessing.set_start_method('spawn')
+    parser = ArgumentParser()
+
+    parser.add_argument('--accelerator', default='gpu', type=str)
+    parser.add_argument('--devices', default=-1, type=list)
+    parser.add_argument('--precision', default=16, type=int)
+    parser.add_argument('--amp_backend', default="native", type=str)
+
+    parser.add_argument('--batch_size', default=4, type=int)
+    parser.add_argument('--num_workers', default=8, type=int)
+    parser.add_argument('--seed', default=1234, type=int)
+    parser.add_argument('--lr', default=1e-4, type=float)
+    parser.add_argument('--accumulate_grad_batches', default=32, type=int)
+    parser.add_argument('--check_val_every_n_epoch', default=1, type=int)
+
+    parser.add_argument('--lr_scheduler', default='cosine', choices=['cosine'], type=str)
+    parser.add_argument('--lr_decay_min_lr', default=1e-6, type=float)
+    parser.add_argument('--lr_warmup_start_lr', default=1e-6, type=float)
+
+    parser.add_argument('--load_best', action='store_true')
+    parser.add_argument('--load_dir', default=None, type=str)
+    parser.add_argument('--load_ver', default=None, type=str)
+    parser.add_argument('--load_v_num', default=None, type=int)
+
+    parser.add_argument('--dataset', default='steam_data', type=str)
+    parser.add_argument('--data_dir', default='LLaRA_MOE/data/ref/steam', type=str)
+    parser.add_argument('--model_name', default='mlp_projector', type=str)
+    parser.add_argument('--loss', default='lm', type=str)
+    parser.add_argument('--weight_decay', default=1e-5, type=float)
+    parser.add_argument('--no_augment', action='store_true')
+    parser.add_argument('--ckpt_dir', default='LLaRA_MOE/checkpoints/steam/', type=str)
+    parser.add_argument('--log_dir', default='steam_logs', type=str)
+    
+    parser.add_argument('--rec_size', default=64, type=int)
+    parser.add_argument('--padding_item_id', default=3581, type=int)
+    parser.add_argument('--llm_path', default='meta-llama/Llama-2-7b-hf', type=str)
+    parser.add_argument('--rec_model_path', default='LLaRA_MOE/rec_model/SASRec_steam.pt', type=str)
+    parser.add_argument('--prompt_path', default='LLaRA_MOE/prompt/game.txt', type=str)
+    parser.add_argument('--output_dir', default='LLaRA_MOE/output/steam_moe/', type=str)
+    parser.add_argument('--ckpt_path', type=str)
+    parser.add_argument('--rec_embed', default="SASRec", choices=['SASRec', 'Caser','GRU'], type=str)
+
+    parser.add_argument('--aug_prob', default=0.5, type=float)
+    parser.add_argument('--mode', default='test', choices=['train', 'test'], type=str)
+    parser.add_argument('--auto_lr_find', default=False, action='store_true')
+    parser.add_argument('--metric', default='hr', choices=['hr'], type=str)
+    parser.add_argument('--max_epochs', default=5, type=int)
+    parser.add_argument('--save', default='part', choices=['part', 'all'], type=str)
+    parser.add_argument('--cans_num', default=20, type=int)
+
+    # Finetuning
+    parser.add_argument('--llm_tuning', default='moelora', choices=['lora', 'freeze','freeze_lora', 'moelora'], type=str)
+    parser.add_argument('--peft_dir', default=None, type=str)
+    parser.add_argument('--peft_config', default=None, type=str)
+    parser.add_argument('--lora_r', default=8, type=float)
+    parser.add_argument('--lora_alpha', default=32, type=float)
+    parser.add_argument('--lora_dropout', default=0.1, type=float)
+    parser.add_argument('--num_moe', default=4, type=int)
+    parser.add_argument('--gating', default='Dense', type=str)
+    
+    parser.add_argument('--local_rank', default=3, type=int)
+    
+    parser.add_argument('--if_rand', default=False, type=bool)
+    
+    parser.add_argument('--router', default='unshare', choices=['share', 'unshare'], type=str)
+
+    args = parser.parse_args()
+    
+    if 'movielens' in args.data_dir:
+        args.padding_item_id = 1682
+    elif 'steam' in args.data_dir:
+        args.padding_item_id = 3581
+    elif 'lastfm' in args.data_dir:
+        args.padding_item_id = 4606
+
+    main(args)

model/mlp_projector.py

@@ -0,0 +1,14 @@
+from torch import nn
+
+class MlpProjector(nn.Module):
+    def __init__(self, rec_size=64, llm_size=4096):
+        super().__init__()
+        self.mlp_proj = nn.Sequential(
+            nn.Linear(rec_size, llm_size),
+            nn.GELU(),
+            nn.Linear(llm_size, llm_size)
+        )
+
+    def forward(self, x):
+        x = self.mlp_proj(x)
+        return x

model/model_interface.py

@@ -0,0 +1,492 @@
+import inspect
+import torch
+import importlib
+from torch import nn
+from torch.nn import functional as F
+import torch.optim.lr_scheduler as lrs
+
+import pytorch_lightning as pl
+
+from transformers import LlamaForCausalLM, LlamaTokenizer
+import random
+from pandas.core.frame import DataFrame
+import os.path as op
+import os
+from optims import LinearWarmupCosineLRScheduler
+import numpy as np
+from .peft import get_peft_config, get_peft_model, get_peft_model_state_dict, LoraConfig, TaskType, PeftModel, MoeLoraConfig, MoeLoraModel
+import pickle
+from .router.nlpr import LambdaLayer, ResidualBlock, GateFunction, NLPRecommendationRouter, build_router
+
+
+# from peft import get_peft_config, get_peft_model, get_peft_model_state_dict, LoraConfig, TaskType, PeftModel
+class MInterface(pl.LightningModule):
+    def __init__(self, 
+                 **kargs):
+        super().__init__()
+        self.save_hyperparameters()
+        self.load_llm(self.hparams.llm_path)
+        
+        if self.hparams.router == 'share':
+            self.router = build_router()
+
+        self.load_rec_model(self.hparams.rec_model_path)
+        self.load_projector()
+        self.gradient_storage = {}
+    
+    def forward(self, batch):
+        targets = batch["tokens"].input_ids.masked_fill(
+            batch["tokens"].input_ids == self.llama_tokenizer.pad_token_id, -100
+        ) # [batch_size, max_len]
+        
+        targets = targets.masked_fill((batch["tokens"].token_type_ids == 0)[:,1:], -100)
+        # targets = targets.masked_fill((batch["tokens"].token_type_ids == 0)[:,:], -100)
+        
+        input_embeds, user_embeds = self.wrap_emb(batch)
+
+        if self.hparams.router == 'share':
+            gate_weights = self.router(user_embeds)
+            outputs = self.llama_model(
+                inputs_embeds=input_embeds,
+                attention_mask=batch["tokens"].attention_mask,
+                return_dict=True,
+                labels=targets,
+                use_cache=False,
+                user_embeds=user_embeds,
+                gate_weights=gate_weights
+            )
+            return outputs
+        
+        outputs = self.llama_model(
+            inputs_embeds=input_embeds,
+            attention_mask=batch["tokens"].attention_mask,
+            return_dict=True,
+            labels=targets,
+            use_cache=False,
+            user_embeds=user_embeds
+        )
+        return outputs
+
+    def generate(self, batch,temperature=0.8,do_sample=False,num_beams=1,max_gen_length=64,min_gen_length=1,repetition_penalty=1.0,length_penalty=1.0, num_return_sequences=1):
+        input_embeds, user_embeds = self.wrap_emb(batch)
+        if self.hparams.router == 'share':
+            gate_weights = self.router(user_embeds)
+            generate_ids = self.llama_model.generate(
+                inputs_embeds=input_embeds,
+                attention_mask=batch["tokens"].attention_mask,
+                temperature=temperature,
+                do_sample=do_sample,
+                num_beams=num_beams,
+                max_new_tokens=max_gen_length,
+                min_new_tokens=min_gen_length,
+                pad_token_id=self.llama_tokenizer.pad_token_id,
+                repetition_penalty=repetition_penalty,
+                length_penalty=length_penalty,
+                num_return_sequences=num_return_sequences,
+                user_embeds=user_embeds,
+                gate_weights = gate_weights
+            )
+            output_text=self.llama_tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)
+            outputs=[text.strip() for text in output_text]
+            return outputs
+            
+        gate_weights = self.router(user_embeds)
+        
+        generate_ids = self.llama_model.generate(
+            inputs_embeds=input_embeds,
+            attention_mask=batch["tokens"].attention_mask,
+            temperature=temperature,
+            do_sample=do_sample,
+            num_beams=num_beams,
+            max_new_tokens=max_gen_length,
+            min_new_tokens=min_gen_length,
+            pad_token_id=self.llama_tokenizer.pad_token_id,
+            repetition_penalty=repetition_penalty,
+            length_penalty=length_penalty,
+            num_return_sequences=num_return_sequences,
+            
+            user_embeds=user_embeds, 
+            gate_weights = gate_weights
+            )
+        output_text=self.llama_tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)
+        outputs=[text.strip() for text in output_text]
+        return outputs
+    
+    def capture_and_store_gradients(self):
+        for name, param in self.llama_model.named_parameters():
+            if "lora" in name and param.grad is not None:
+                if name not in self.gradient_storage:
+                    self.gradient_storage[name] = []
+                self.gradient_storage[name].append(param.grad.clone().detach())
+        
+        if self.trainer.global_step % 10 == 0: 
+            self.save_gradients_to_file()
+            
+    def save_gradients_to_file(self):
+        directory = self.hparams.capture_dir
+        if not os.path.exists(directory):
+            os.makedirs(directory)
+        file_path = os.path.join(directory, f'gradients_step_{self.trainer.global_step}.pkl')
+        with open(file_path, 'wb') as f:
+            pickle.dump(self.gradient_storage, f)
+        self.gradient_storage = {} 
+            
+    def training_step(self, batch, batch_idx):
+        if self.scheduler:
+            self.scheduler.step(self.trainer.global_step, self.current_epoch, self.trainer.max_steps)
+        if batch["flag"]:
+            for name, param in self.projector.named_parameters():
+                param.requires_grad = False
+        else:
+            for name, param in self.projector.named_parameters():
+                param.requires_grad = True
+        out = self(batch)
+        loss = self.configure_loss(out)
+        self.log('loss', loss, on_step=True, on_epoch=True, prog_bar=True)
+        self.log('lr', self.scheduler.optimizer.param_groups[0]['lr'], on_step=True, on_epoch=True, prog_bar=True)
+        self.log('global_step_num', self.trainer.global_step, on_step=True, on_epoch=True, prog_bar=True)
+        
+        return loss
+            
+    def on_validation_epoch_start(self):
+        self.val_content={
+            "generate":[],
+            "real":[],
+            "cans":[],
+        }
+
+    @torch.no_grad()
+    def validation_step(self, batch, batch_idx):
+        generate_output = self.generate(batch)
+        output=[]
+        for i,generate in enumerate(generate_output):
+            real=batch['correct_answer'][i]
+            cans=batch['cans_name'][i]
+            generate=generate.strip().split("\n")[0]
+            output.append((generate,real,cans))
+        return output
+
+    def on_validation_batch_end(self, outputs, batch, batch_idx, dataloader_idx):
+        for generate,real,cans in outputs:
+            self.val_content["generate"].append(generate)
+            self.val_content["real"].append(real)
+            self.val_content["cans"].append(cans)
+
+    def on_validation_epoch_end(self):
+        df=DataFrame(self.val_content)
+        if not os.path.exists(self.hparams.output_dir):
+            os.makedirs(self.hparams.output_dir)
+        df.to_csv(op.join(self.hparams.output_dir, 'valid.csv'))
+        prediction_valid_ratio,hr=self.calculate_hr1(self.val_content)
+        metric=hr*prediction_valid_ratio
+        self.log('val_prediction_valid', prediction_valid_ratio, on_step=False, on_epoch=True, prog_bar=True)
+        self.log('val_hr', hr, on_step=False, on_epoch=True, prog_bar=True)
+        self.log('metric', metric, on_step=False, on_epoch=True, prog_bar=True)
+
+    def on_test_epoch_start(self):
+        self.test_content={
+            "generate":[],
+            "real":[],
+            "cans":[],
+        }
+
+    @torch.no_grad()
+    def test_step(self, batch, batch_idx):
+        generate_output = self.generate(batch)
+        output=[]
+        for i,generate in enumerate(generate_output):
+            real=batch['correct_answer'][i]
+            cans=batch['cans_name'][i]
+            generate=generate.strip().split("\n")[0]
+            output.append((generate,real,cans))
+        return output
+    
+    def on_test_batch_end(self, outputs, batch, batch_idx, dataloader_idx):
+        for generate,real,cans in outputs:
+            self.test_content["generate"].append(generate)
+            self.test_content["real"].append(real)
+            self.test_content["cans"].append(cans)
+
+    def on_test_epoch_end(self):
+        df=DataFrame(self.test_content)
+        if not os.path.exists(self.hparams.output_dir):
+            os.makedirs(self.hparams.output_dir)
+        df.to_csv(op.join(self.hparams.output_dir, 'test.csv'))
+        prediction_valid_ratio,hr=self.calculate_hr1(self.test_content)
+        metric=hr*prediction_valid_ratio
+
+        self.log('test_prediction_valid', prediction_valid_ratio, on_step=False, on_epoch=True, prog_bar=True)
+        self.log('test_hr', hr, on_step=False, on_epoch=True, prog_bar=True)
+        self.log('metric', metric, on_step=False, on_epoch=True, prog_bar=True)
+
+    def configure_optimizers(self):
+        if hasattr(self.hparams, 'weight_decay'):
+            weight_decay = self.hparams.weight_decay
+        else:
+            weight_decay = 0
+        optimizer = torch.optim.SGD([
+            {'params': self.projector.parameters(), 'lr': self.hparams.lr, 'weight_decay':weight_decay},
+            
+            {'params': self.router.parameters(), 'lr': self.hparams.lr * 0.3, 'weight_decay':weight_decay},
+            
+            {'params': [p for n, p in self.llama_model.named_parameters() if "gating" not in n], 'lr': self.hparams.lr},
+            # {'params': [p for n, p in self.llama_model.named_parameters() if "gating" in n], 'lr': self.hparams.lr * 1, 'weight_decay':weight_decay}
+           
+            # {'params': self.llama_model.parameters(), 'lr': self.hparams.lr},
+        ])
+        
+        
+        for i, param_group in enumerate(optimizer.param_groups):
+            print(f"Initial LR for group {i}: {param_group['lr']}")   
+            total_params = sum(p.numel() for p in param_group['params'])
+            print(f"Parameter Group {i}: {total_params} parameters")
+
+        if self.hparams.lr_scheduler is None:
+            return optimizer
+        else:
+            max_step = self.trainer.max_steps
+            warmup_steps = max_step // 20
+            print(f'max_step: {max_step}')
+            print(f'warmup_steps: {warmup_steps}')
+            if self.hparams.lr_scheduler == 'cosine':
+                
+                init_lr_list = [
+                    self.hparams.lr,  
+                    self.hparams.lr * 0.3, 
+                    self.hparams.lr * 1
+                ]
+                min_lr_list = [
+                    self.hparams.lr_decay_min_lr,  
+                    self.hparams.lr_decay_min_lr * 0.3,  
+                    self.hparams.lr_decay_min_lr * 1  
+                ]
+                warmup_start_lr_list = [
+                    self.hparams.lr_warmup_start_lr, 
+                    self.hparams.lr_warmup_start_lr * 0.3, 
+                    self.hparams.lr_warmup_start_lr * 1 
+                ]
+                self.scheduler = LinearWarmupCosineLRScheduler(
+                    optimizer=optimizer,
+                    max_step=max_step,
+                    min_lr_list=min_lr_list,
+                    init_lr_list=init_lr_list,
+                    warmup_steps=warmup_steps,
+                    warmup_start_lr_list=warmup_start_lr_list
+                )
+                                
+                
+                for i, param_group in enumerate(optimizer.param_groups):
+                    print(f"Initial LR for group {i}: {param_group['lr']}")   
+                    total_params = sum(p.numel() for p in param_group['params'])
+                    print(f"Parameter Group {i}: {total_params} parameters")
+                    
+                    
+            else:
+                self.scheduler = None
+                raise ValueError('Invalid lr_scheduler type!')
+            return optimizer
+
+    def configure_loss(self, out, labels=None):
+        loss = self.hparams.loss.lower()
+        if loss == 'lm':
+            return out.loss
+        else:
+            raise ValueError("Invalid Loss Type!")
+
+    def on_save_checkpoint(self, checkpoint):
+        if self.hparams.save == 'part':
+            checkpoint.pop('optimizer_states')
+            to_be_removed = []
+            for key, value in checkpoint['state_dict'].items():
+                try:
+                    if not self.get_parameter(key).requires_grad:
+                        to_be_removed.append(key)
+                except AttributeError:
+                    to_be_removed.append(key)
+            for key in to_be_removed:
+                checkpoint['state_dict'].pop(key)
+        elif self.hparams.save == 'all':
+            pass
+        
+    def load_llm(self, llm_path):
+        print('Loading LLAMA')
+        self.llama_tokenizer = LlamaTokenizer.from_pretrained(llm_path, use_fast=False)
+        self.llama_tokenizer.pad_token = self.llama_tokenizer.eos_token
+        self.llama_tokenizer.add_special_tokens({'pad_token': '[PAD]'})
+        self.llama_tokenizer.padding_side = "right"
+        self.llama_tokenizer.add_special_tokens({'additional_special_tokens': ['[PH]','[HistoryEmb]','[CansEmb]','[ItemEmb]']})
+        self.llama_model = LlamaForCausalLM.from_pretrained(llm_path, device_map="auto",load_in_8bit=True)
+        self.llama_model.resize_token_embeddings(len(self.llama_tokenizer))
+        if self.hparams.llm_tuning == 'lora':
+            if self.hparams.peft_dir:
+                self.llama_model = PeftModel.from_pretrained(self.llm_model, self.hparams.peft_dir, is_trainable=True)
+            else:
+                if self.hparams.peft_config:
+                    peft_config = LoraConfig(**LoraConfig.from_json_file(self.hparams.peft_config))
+                else:
+                    peft_config = LoraConfig(task_type=TaskType.CAUSAL_LM,
+                                             inference_mode=False,
+                                             r=self.hparams.lora_r,
+                                             lora_alpha=self.hparams.lora_alpha,
+                                             lora_dropout=self.hparams.lora_dropout,
+                                             target_modules=['k_proj', 'v_proj', 'q_proj', 'o_proj', 'gate_proj', 'up_proj', 'down_proj'])
+                self.peft_config = peft_config
+                self.llama_model = get_peft_model(self.llama_model, peft_config)
+            self.llama_model.print_trainable_parameters()
+        elif self.hparams.llm_tuning == 'freeze':
+            for name, param in self.llama_model.named_parameters():
+                param.requires_grad = False
+        elif self.hparams.llm_tuning == 'freeze_lora':
+            if self.hparams.peft_dir:
+                self.llama_model = PeftModel.from_pretrained(self.llm_model, self.hparams.peft_dir, is_trainable=True)
+            else:
+                if self.hparams.peft_config:
+                    peft_config = LoraConfig(**LoraConfig.from_json_file(self.hparams.peft_config))
+                else:
+                    peft_config = LoraConfig(task_type=TaskType.CAUSAL_LM,
+                                             inference_mode=False,
+                                             r=self.hparams.lora_r,
+                                             lora_alpha=self.hparams.lora_alpha,
+                                             lora_dropout=self.hparams.lora_dropout,
+                                             target_modules=['k_proj', 'v_proj', 'q_proj', 'o_proj', 'gate_proj', 'up_proj', 'down_proj'])
+                self.peft_config = peft_config
+                self.llama_model = get_peft_model(self.llama_model, peft_config)
+            for name, param in self.llama_model.named_parameters():
+                param.requires_grad = False
+            self.llama_model.print_trainable_parameters()
+        elif self.hparams.llm_tuning == 'moelora':
+            if self.hparams.peft_dir:
+                self.llama_model = PeftModel.from_pretrained(self.llm_model, self.hparams.peft_dir, is_trainable=True)
+            else:
+                if self.hparams.peft_config:
+                    peft_config = MoeLoraConfig(**MoeLoraConfig.from_json_file(self.hparams.peft_config))
+                else:
+                    peft_config = MoeLoraConfig(task_type=TaskType.CAUSAL_LM,
+                                                inference_mode=False,
+                                                r=self.hparams.lora_r,
+                                                lora_alpha=self.hparams.lora_alpha,
+                                                lora_dropout=self.hparams.lora_dropout,
+                                                target_modules=['k_proj', 'v_proj', 'q_proj', 'o_proj', 'gate_proj', 'up_proj', 'down_proj'],
+                                                num_moe=self.hparams.num_moe,
+                                                gating=self.hparams.gating)
+                self.peft_config = peft_config
+                self.llama_model = get_peft_model(self.llama_model, peft_config)
+                
+                """for name, param in self.llama_model.named_parameters():
+                    if "gating" not in name:
+                        param.requires_grad = False"""
+            self.llama_model.print_trainable_parameters()
+        else:
+            raise NotImplementedError()
+ 
+        print('Loading LLAMA Done')
+
+    def load_projector(self):
+        name = self.hparams.model_name
+        camel_name = ''.join([i.capitalize() for i in name.split('_')])
+        try:
+            Model = getattr(importlib.import_module(
+                '.'+name, package=__package__), camel_name)
+        except:
+            raise ValueError(
+                f'Invalid Module File Name or Invalid Class Name {name}.{camel_name}!')
+        self.projector = self.instancialize(Model, rec_size=self.hparams.rec_size, llm_size=self.llama_model.config.hidden_size)
+
+    def instancialize(self, Model, **other_args):
+        class_args = inspect.getargspec(Model.__init__).args[1:]
+        inkeys = self.hparams.keys()
+        args1 = {}
+        for arg in class_args:
+            if arg in inkeys:
+                args1[arg] = getattr(self.hparams, arg)
+        args1.update(other_args)
+        # args1: args在hparams中有的部分
+        return Model(**args1)
+
+    def load_rec_model(self, rec_model_path):
+        print('Loading Rec Model')
+        self.rec_model = torch.load(rec_model_path, map_location="cpu")
+        self.rec_model.eval()
+        for name, param in self.rec_model.named_parameters():
+            param.requires_grad = False
+        print('Loding Rec model Done')
+
+    def encode_items(self, seq):
+        if self.hparams.rec_embed=="SASRec":
+            item_rec_embs=self.rec_model.cacu_x(seq)
+        elif self.hparams.rec_embed in ['Caser','GRU']:
+            item_rec_embs=self.rec_model.item_embeddings(seq)
+        item_txt_embs=self.projector(item_rec_embs)
+        return item_txt_embs
+
+    def encode_users(self, seq, len_seq):
+        if self.hparams.rec_embed=="SASRec":
+            user_rec_embs=self.rec_model.cacul_h(seq, len_seq)
+        elif self.hparams.rec_embed in ['Caser','GRU']:
+            user_rec_embs=self.rec_model.item_embeddings(seq)
+        
+        user_txt_embs=self.projector(user_rec_embs)    
+        return user_rec_embs
+    
+    def embed_tokens(self, token_ids):
+        embeds = self.llama_model.base_model.embed_tokens(token_ids)
+        return embeds
+
+    # batch -> embeds
+    def wrap_emb(self, batch):
+        input_embeds = self.llama_model.get_input_embeddings()(batch["tokens"].input_ids)
+        
+           
+        
+        his_token_id=self.llama_tokenizer("[HistoryEmb]", return_tensors="pt",add_special_tokens=False).input_ids.item()
+        cans_token_id=self.llama_tokenizer("[CansEmb]", return_tensors="pt",add_special_tokens=False).input_ids.item()
+        item_token_id=self.llama_tokenizer("[ItemEmb]", return_tensors="pt",add_special_tokens=False).input_ids.item()
+         
+        
+        his_item_embeds = self.encode_items(batch["seq"])
+        cans_item_embeds = self.encode_items(batch["cans"])
+        item_embeds=self.encode_items(batch["item_id"])
+        
+        
+        user_embeds=self.encode_users(batch["seq"], batch["len_seq"])
+
+        for i in range(len(batch["len_seq"])):
+            if (batch["tokens"].input_ids[i]==his_token_id).nonzero().shape[0]>0:
+                idx_tensor=(batch["tokens"].input_ids[i]==his_token_id).nonzero().view(-1)
+                for idx, item_emb in zip(idx_tensor,his_item_embeds[i,:batch["len_seq"][i].item()]):
+                    input_embeds[i,idx]=item_emb
+            if (batch["tokens"].input_ids[i]==cans_token_id).nonzero().shape[0]>0:
+                idx_tensor=(batch["tokens"].input_ids[i]==cans_token_id).nonzero().view(-1)
+                for idx, item_emb in zip(idx_tensor,cans_item_embeds[i,:batch["len_cans"][i].item()]):
+                    input_embeds[i,idx]=item_emb
+            if (batch["tokens"].input_ids[i]==item_token_id).nonzero().shape[0]>0:
+                idx=(batch["tokens"].input_ids[i]==item_token_id).nonzero().item()
+                input_embeds[i,idx]=item_embeds[i]
+        
+        return input_embeds, user_embeds
+     
+    def calculate_hr1(self,eval_content):
+        correct_num=0
+        valid_num=0
+        total_num=0
+        for i,generate in enumerate(eval_content["generate"]):
+            real=eval_content["real"][i]
+            cans=eval_content["cans"][i]
+            total_num+=1
+            generate=generate.strip().lower().strip()
+            real=real.strip().lower().strip()
+            cans=[item.strip().lower().strip() for item in cans]
+            gen_cans_list=[]
+            for cans_item in cans:
+                if cans_item in generate:
+                    gen_cans_list.append(cans_item)
+            if len(gen_cans_list)==1:
+                valid_num+=1
+                if real == gen_cans_list[0]:
+                    correct_num+=1
+        valid_ratio=valid_num/total_num
+        if valid_num>0:
+            hr1=correct_num/valid_num
+        else:
+            hr1=0
+        return valid_ratio,hr1

model/peft/__init__.py

@@ -0,0 +1,61 @@
+# flake8: noqa
+# There's no way to ignore "F401 '...' imported but unused" warnings in this
+# module, but to preserve other warnings. So, don't check this module at all.
+
+# coding=utf-8
+# Copyright 2023-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__version__ = "0.4.0.dev0"
+
+from .mapping import MODEL_TYPE_TO_PEFT_MODEL_MAPPING, PEFT_TYPE_TO_CONFIG_MAPPING, get_peft_config, get_peft_model
+from .peft_model import (
+    PeftModel,
+    PeftModelForCausalLM,
+    PeftModelForSeq2SeqLM,
+    PeftModelForSequenceClassification,
+    PeftModelForTokenClassification,
+    PeftModelForQuestionAnswering,
+)
+from .tuners import (
+    AdaptionPromptConfig,
+    AdaptionPromptModel,
+    LoraConfig,
+    LoraModel,
+    AdaLoraConfig,
+    AdaLoraModel,
+    PrefixEncoder,
+    PrefixTuningConfig,
+    PromptEmbedding,
+    PromptEncoder,
+    PromptEncoderConfig,
+    PromptEncoderReparameterizationType,
+    PromptTuningConfig,
+    PromptTuningInit,
+    MoeLoraConfig,
+    MoeLoraModel,
+)
+from .utils import (
+    TRANSFORMERS_MODELS_TO_PREFIX_TUNING_POSTPROCESS_MAPPING,
+    PeftConfig,
+    PeftType,
+    PromptLearningConfig,
+    TaskType,
+    bloom_model_postprocess_past_key_value,
+    get_peft_model_state_dict,
+    prepare_model_for_int8_training,
+    prepare_model_for_kbit_training,
+    set_peft_model_state_dict,
+    shift_tokens_right,
+)

model/peft/import_utils.py

@@ -0,0 +1,28 @@
+# coding=utf-8
+# Copyright 2023-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import importlib
+
+
+def is_bnb_available():
+    return importlib.util.find_spec("bitsandbytes") is not None
+
+
+def is_bnb_4bit_available():
+    if not is_bnb_available():
+        return False
+
+    import bitsandbytes as bnb
+
+    return hasattr(bnb.nn, "Linear4bit")

model/peft/mapping.py

@@ -0,0 +1,134 @@
+# coding=utf-8
+# Copyright 2023-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Dict
+
+from .peft_model import (
+    PeftModel,
+    PeftModelForCausalLM,
+    PeftModelForQuestionAnswering,
+    PeftModelForSeq2SeqLM,
+    PeftModelForSequenceClassification,
+    PeftModelForTokenClassification,
+)
+from .tuners import (
+    AdaLoraConfig,
+    AdaptionPromptConfig,
+    LoraConfig,
+    PrefixTuningConfig,
+    PromptEncoderConfig,
+    PromptTuningConfig,
+    MoeLoraConfig,
+)
+from .utils import PromptLearningConfig
+
+
+if TYPE_CHECKING:
+    from transformers import PreTrainedModel
+
+    from .utils.config import PeftConfig
+
+
+MODEL_TYPE_TO_PEFT_MODEL_MAPPING = {
+    "SEQ_CLS": PeftModelForSequenceClassification,
+    "SEQ_2_SEQ_LM": PeftModelForSeq2SeqLM,
+    "CAUSAL_LM": PeftModelForCausalLM,
+    "TOKEN_CLS": PeftModelForTokenClassification,
+    "QUESTION_ANS": PeftModelForQuestionAnswering,
+}
+
+PEFT_TYPE_TO_CONFIG_MAPPING = {
+    "ADAPTION_PROMPT": AdaptionPromptConfig,
+    "PROMPT_TUNING": PromptTuningConfig,
+    "PREFIX_TUNING": PrefixTuningConfig,
+    "P_TUNING": PromptEncoderConfig,
+    "LORA": LoraConfig,
+    "ADALORA": AdaLoraConfig,
+    "MOELORA": MoeLoraConfig,
+}
+
+
+def get_peft_config(config_dict: Dict[str, Any]):
+    """
+    Returns a Peft config object from a dictionary.
+
+    Args:
+        config_dict (`Dict[str, Any]`): Dictionary containing the configuration parameters.
+    """
+
+    return PEFT_TYPE_TO_CONFIG_MAPPING[config_dict["peft_type"]](**config_dict)
+
+
+def _prepare_prompt_learning_config(peft_config: PeftConfig, model_config: Dict[str, Any]):
+    if peft_config.num_layers is None:
+        if "num_hidden_layers" in model_config:
+            num_layers = model_config["num_hidden_layers"]
+        elif "num_layers" in model_config:
+            num_layers = model_config["num_layers"]
+        elif "n_layer" in model_config:
+            num_layers = model_config["n_layer"]
+        else:
+            raise ValueError("Please specify `num_layers` in `peft_config`")
+        peft_config.num_layers = num_layers
+
+    if peft_config.token_dim is None:
+        if "hidden_size" in model_config:
+            token_dim = model_config["hidden_size"]
+        elif "n_embd" in model_config:
+            token_dim = model_config["n_embd"]
+        elif "d_model" in model_config:
+            token_dim = model_config["d_model"]
+        else:
+            raise ValueError("Please specify `token_dim` in `peft_config`")
+        peft_config.token_dim = token_dim
+
+    if peft_config.num_attention_heads is None:
+        if "num_attention_heads" in model_config:
+            num_attention_heads = model_config["num_attention_heads"]
+        elif "n_head" in model_config:
+            num_attention_heads = model_config["n_head"]
+        elif "num_heads" in model_config:
+            num_attention_heads = model_config["num_heads"]
+        elif "encoder_attention_heads" in model_config:
+            num_attention_heads = model_config["encoder_attention_heads"]
+        else:
+            raise ValueError("Please specify `num_attention_heads` in `peft_config`")
+        peft_config.num_attention_heads = num_attention_heads
+
+    if getattr(peft_config, "encoder_hidden_size", None) is None:
+        setattr(peft_config, "encoder_hidden_size", peft_config.token_dim)
+
+    return peft_config
+
+
+def get_peft_model(model: PreTrainedModel, peft_config: PeftConfig, adapter_name: str = "default") -> PeftModel:
+    """
+    Returns a Peft model object from a model and a config.
+
+    Args:
+        model ([`transformers.PreTrainedModel`]): Model to be wrapped.
+        peft_config ([`PeftConfig`]): Configuration object containing the parameters of the Peft model.
+    """
+    model_config = model.config.to_dict() if hasattr(model.config, "to_dict") else model.config
+    peft_config.base_model_name_or_path = model.__dict__.get("name_or_path", None)
+    if peft_config.task_type not in MODEL_TYPE_TO_PEFT_MODEL_MAPPING.keys() and not isinstance(
+        peft_config, PromptLearningConfig
+    ):
+        return PeftModel(model, peft_config, adapter_name=adapter_name)
+    if isinstance(peft_config, PromptLearningConfig):
+        peft_config = _prepare_prompt_learning_config(peft_config, model_config)
+    return MODEL_TYPE_TO_PEFT_MODEL_MAPPING[peft_config.task_type](model, peft_config, adapter_name=adapter_name)

model/peft/peft_model.py

@@ -0,0 +1,1619 @@
+# coding=utf-8
+# Copyright 2023-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import inspect
+import os
+import warnings
+from contextlib import contextmanager
+from copy import deepcopy
+from typing import Any, Dict, Optional, Union
+
+import torch
+from accelerate import dispatch_model, infer_auto_device_map
+from accelerate.hooks import AlignDevicesHook, add_hook_to_module, remove_hook_from_submodules
+from accelerate.utils import get_balanced_memory
+from huggingface_hub import hf_hub_download
+from huggingface_hub.utils import EntryNotFoundError
+from safetensors.torch import load_file as safe_load_file
+from safetensors.torch import save_file as safe_save_file
+from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
+from transformers import PreTrainedModel
+from transformers.modeling_outputs import QuestionAnsweringModelOutput, SequenceClassifierOutput, TokenClassifierOutput
+from transformers.utils import PushToHubMixin
+
+from . import __version__
+from .tuners import (
+    AdaLoraModel,
+    AdaptionPromptModel,
+    LoraModel,
+    PrefixEncoder,
+    PromptEmbedding,
+    PromptEncoder,
+    MoeLoraModel,
+)
+from .utils import (
+    SAFETENSORS_WEIGHTS_NAME,
+    TRANSFORMERS_MODELS_TO_PREFIX_TUNING_POSTPROCESS_MAPPING,
+    WEIGHTS_NAME,
+    PeftConfig,
+    PeftType,
+    PromptLearningConfig,
+    TaskType,
+    _set_adapter,
+    _set_trainable,
+    add_library_to_model_card,
+    get_peft_model_state_dict,
+    hub_file_exists,
+    set_peft_model_state_dict,
+    shift_tokens_right,
+)
+
+
+PEFT_TYPE_TO_MODEL_MAPPING = {
+    PeftType.LORA: LoraModel,
+    PeftType.PROMPT_TUNING: PromptEmbedding,
+    PeftType.P_TUNING: PromptEncoder,
+    PeftType.PREFIX_TUNING: PrefixEncoder,
+    PeftType.ADALORA: AdaLoraModel,
+    PeftType.ADAPTION_PROMPT: AdaptionPromptModel,
+    PeftType.MOELORA: MoeLoraModel,
+}
+
+
+class PeftModel(PushToHubMixin, torch.nn.Module):
+    """
+    Base model encompassing various Peft methods.
+
+    Args:
+        model ([`~transformers.PreTrainedModel`]): The base transformer model used for Peft.
+        peft_config ([`PeftConfig`]): The configuration of the Peft model.
+
+
+    **Attributes**:
+        - **base_model** ([`~transformers.PreTrainedModel`]) -- The base transformer model used for Peft.
+        - **peft_config** ([`PeftConfig`]) -- The configuration of the Peft model.
+        - **modules_to_save** (`list` of `str`) -- The list of sub-module names to save when
+        saving the model.
+        - **prompt_encoder** ([`PromptEncoder`]) -- The prompt encoder used for Peft if
+        using [`PromptLearningConfig`].
+        - **prompt_tokens** (`torch.Tensor`) -- The virtual prompt tokens used for Peft if
+        using [`PromptLearningConfig`].
+        - **transformer_backbone_name** (`str`) -- The name of the transformer
+        backbone in the base model if using [`PromptLearningConfig`].
+        - **word_embeddings** (`torch.nn.Embedding`) -- The word embeddings of the transformer backbone
+        in the base model if using [`PromptLearningConfig`].
+    """
+
+    def __init__(self, model: PreTrainedModel, peft_config: PeftConfig, adapter_name: str = "default"):
+        super().__init__()
+        self.base_model = model
+        self.config = self.base_model.config
+        self.modules_to_save = None
+        self.peft_config = {}
+        self.active_adapter = adapter_name
+        self.peft_type = peft_config.peft_type
+        self.base_model_torch_dtype = getattr(model, "dtype", None)
+        if not isinstance(peft_config, PromptLearningConfig):
+            self.peft_config[adapter_name] = peft_config
+            self.base_model = PEFT_TYPE_TO_MODEL_MAPPING[peft_config.peft_type](
+                self.base_model, self.peft_config, adapter_name
+            )
+            self.set_additional_trainable_modules(peft_config, adapter_name)
+        else:
+            self.add_adapter(adapter_name, peft_config)
+
+        if getattr(model, "is_gradient_checkpointing", True):
+            model = self._prepare_model_for_gradient_checkpointing(model)
+
+    def save_pretrained(self, save_directory: str, safe_serialization: bool = False, **kwargs: Any):
+        r"""
+        This function saves the adapter model and the adapter configuration files to a directory, so that it can be
+        reloaded using the [`LoraModel.from_pretrained`] class method, and also used by the [`LoraModel.push_to_hub`]
+        method.
+
+        Args:
+            save_directory (`str`):
+                Directory where the adapter model and configuration files will be saved (will be created if it does not
+                exist).
+            kwargs (additional keyword arguments, *optional*):
+                Additional keyword arguments passed along to the `push_to_hub` method.
+        """
+        if os.path.isfile(save_directory):
+            raise ValueError(f"Provided path ({save_directory}) should be a directory, not a file")
+        os.makedirs(save_directory, exist_ok=True)
+        self.create_or_update_model_card(save_directory)
+
+        for adapter_name, peft_config in self.peft_config.items():
+            # save only the trainable weights
+            output_state_dict = get_peft_model_state_dict(
+                self, state_dict=kwargs.get("state_dict", None), adapter_name=adapter_name
+            )
+            output_dir = os.path.join(save_directory, adapter_name) if adapter_name != "default" else save_directory
+            os.makedirs(output_dir, exist_ok=True)
+
+            if safe_serialization:
+                safe_save_file(
+                    output_state_dict, os.path.join(output_dir, SAFETENSORS_WEIGHTS_NAME), metadata={"format": "pt"}
+                )
+            else:
+                torch.save(output_state_dict, os.path.join(output_dir, WEIGHTS_NAME))
+
+            # save the config and change the inference mode to `True`
+            if peft_config.base_model_name_or_path is None:
+                peft_config.base_model_name_or_path = (
+                    self.base_model.__dict__.get("name_or_path", None)
+                    if isinstance(peft_config, PromptLearningConfig)
+                    else self.base_model.model.__dict__.get("name_or_path", None)
+                )
+            inference_mode = peft_config.inference_mode
+            peft_config.inference_mode = True
+            peft_config.save_pretrained(output_dir)
+            peft_config.inference_mode = inference_mode
+
+    @classmethod
+    def from_pretrained(
+        cls,
+        model: PreTrainedModel,
+        model_id: Union[str, os.PathLike],
+        adapter_name: str = "default",
+        is_trainable: bool = False,
+        config: Optional[PeftConfig] = None,
+        **kwargs: Any,
+    ):
+        r"""
+        Instantiate a [`LoraModel`] from a pretrained Lora configuration and weights.
+
+        Args:
+            model ([`~transformers.PreTrainedModel`]):
+                The model to be adapted. The model should be initialized with the
+                [`~transformers.PreTrainedModel.from_pretrained`] method from the 🤗 Transformers library.
+            model_id (`str` or `os.PathLike`):
+                The name of the Lora configuration to use. Can be either:
+                    - A string, the `model id` of a Lora configuration hosted inside a model repo on the Hugging Face
+                      Hub.
+                    - A path to a directory containing a Lora configuration file saved using the `save_pretrained`
+                      method (`./my_lora_config_directory/`).
+            adapter_name (`str`, *optional*, defaults to `"default"`):
+                The name of the adapter to be loaded. This is useful for loading multiple adapters.
+            is_trainable (`bool`, *optional*, defaults to `False`):
+                Whether the adapter should be trainable or not. If `False`, the adapter will be frozen and use for
+                inference
+            config ([`~peft.PeftConfig`], *optional*):
+                The configuration object to use instead of an automatically loaded configuation. This configuration
+                object is mutually exclusive with `model_id` and `kwargs`. This is useful when configuration is already
+                loaded before calling `from_pretrained`.
+            kwargs: (`optional`):
+                Additional keyword arguments passed along to the specific Lora configuration class.
+        """
+        from .mapping import MODEL_TYPE_TO_PEFT_MODEL_MAPPING, PEFT_TYPE_TO_CONFIG_MAPPING
+
+        # load the config
+        if config is None:
+            config = PEFT_TYPE_TO_CONFIG_MAPPING[
+                PeftConfig._get_peft_type(
+                    model_id,
+                    subfolder=kwargs.get("subfolder", None),
+                    revision=kwargs.get("revision", None),
+                    cache_dir=kwargs.get("cache_dir", None),
+                )
+            ].from_pretrained(model_id, subfolder=kwargs.get("subfolder", None), **kwargs)
+        elif isinstance(config, PeftConfig):
+            config.inference_mode = not is_trainable
+        else:
+            raise ValueError(f"The input config must be a PeftConfig, got {config.__class__}")
+
+        if (getattr(model, "hf_device_map", None) is not None) and len(
+            set(model.hf_device_map.values()).intersection({"cpu", "disk"})
+        ) > 0:
+            remove_hook_from_submodules(model)
+
+        if isinstance(config, PromptLearningConfig) and is_trainable:
+            raise ValueError("Cannot set a prompt learning adapter to trainable when loading pretrained adapter.")
+        else:
+            config.inference_mode = not is_trainable
+
+        if config.task_type not in MODEL_TYPE_TO_PEFT_MODEL_MAPPING.keys():
+            model = cls(model, config, adapter_name)
+        else:
+            model = MODEL_TYPE_TO_PEFT_MODEL_MAPPING[config.task_type](model, config, adapter_name)
+        model.load_adapter(model_id, adapter_name, is_trainable=is_trainable, **kwargs)
+        return model
+
+    def _setup_prompt_encoder(self, adapter_name: str):
+        config = self.peft_config[adapter_name]
+        self.prompt_encoder = torch.nn.ModuleDict({})
+        self.prompt_tokens = {}
+        transformer_backbone = None
+        for name, module in self.base_model.named_children():
+            for param in module.parameters():
+                param.requires_grad = False
+            if isinstance(module, PreTrainedModel):
+                # Make sure to freeze Tranformers model
+                if transformer_backbone is None:
+                    transformer_backbone = module
+                    self.transformer_backbone_name = name
+
+        if config.num_transformer_submodules is None:
+            config.num_transformer_submodules = 2 if config.task_type == TaskType.SEQ_2_SEQ_LM else 1
+
+        for named_param, value in list(transformer_backbone.named_parameters()):
+            if value.shape[0] == self.base_model.config.vocab_size:
+                self.word_embeddings = transformer_backbone.get_submodule(named_param.replace(".weight", ""))
+                break
+
+        if config.peft_type == PeftType.PROMPT_TUNING:
+            prompt_encoder = PromptEmbedding(config, self.word_embeddings)
+        elif config.peft_type == PeftType.P_TUNING:
+            prompt_encoder = PromptEncoder(config)
+        elif config.peft_type == PeftType.PREFIX_TUNING:
+            prompt_encoder = PrefixEncoder(config)
+        else:
+            raise ValueError("Not supported")
+        self.prompt_encoder.update(torch.nn.ModuleDict({adapter_name: prompt_encoder}))
+        self.prompt_tokens[adapter_name] = torch.arange(
+            config.num_virtual_tokens * config.num_transformer_submodules
+        ).long()
+
+    def _prepare_model_for_gradient_checkpointing(self, model: PreTrainedModel):
+        r"""
+        Prepares the model for gradient checkpointing if necessary
+        """
+        if not (getattr(model, "is_loaded_in_8bit", False) or getattr(model, "is_loaded_in_4bit", False)):
+            if hasattr(model, "enable_input_require_grads"):
+                model.enable_input_require_grads()
+            else:
+
+                def make_inputs_require_grad(module, input, output):
+                    output.requires_grad_(True)
+
+                model.get_input_embeddings().register_forward_hook(make_inputs_require_grad)
+        return model
+
+    def get_prompt_embedding_to_save(self, adapter_name: str):
+        """
+        Returns the prompt embedding to save when saving the model. Only applicable when `peft_config.peft_type !=
+        PeftType.LORA`.
+        """
+        prompt_encoder = self.prompt_encoder[adapter_name]
+        prompt_tokens = (
+            self.prompt_tokens[adapter_name].unsqueeze(0).expand(1, -1).to(prompt_encoder.embedding.weight.device)
+        )
+        if self.peft_config[adapter_name].peft_type == PeftType.PREFIX_TUNING:
+            prompt_tokens = prompt_tokens[:, : self.peft_config[adapter_name].num_virtual_tokens]
+        prompt_embeddings = prompt_encoder(prompt_tokens)
+        return prompt_embeddings[0].detach().cpu()
+
+    def get_prompt(self, batch_size: int):
+        """
+        Returns the virtual prompts to use for Peft. Only applicable when `peft_config.peft_type != PeftType.LORA`.
+        """
+        peft_config = self.active_peft_config
+        prompt_encoder = self.prompt_encoder[self.active_adapter]
+        prompt_tokens = (
+            self.prompt_tokens[self.active_adapter]
+            .unsqueeze(0)
+            .expand(batch_size, -1)
+            .to(prompt_encoder.embedding.weight.device)
+        )
+        if peft_config.peft_type == PeftType.PREFIX_TUNING:
+            prompt_tokens = prompt_tokens[:, : peft_config.num_virtual_tokens]
+            if peft_config.inference_mode:
+                past_key_values = prompt_encoder.embedding.weight.repeat(batch_size, 1, 1)
+            else:
+                past_key_values = prompt_encoder(prompt_tokens)
+            if self.base_model_torch_dtype is not None:
+                past_key_values = past_key_values.to(self.base_model_torch_dtype)
+            past_key_values = past_key_values.view(
+                batch_size,
+                peft_config.num_virtual_tokens,
+                peft_config.num_layers * 2,
+                peft_config.num_attention_heads,
+                peft_config.token_dim // peft_config.num_attention_heads,
+            )
+            if peft_config.num_transformer_submodules == 2:
+                past_key_values = torch.cat([past_key_values, past_key_values], dim=2)
+            past_key_values = past_key_values.permute([2, 0, 3, 1, 4]).split(
+                peft_config.num_transformer_submodules * 2
+            )
+            if TRANSFORMERS_MODELS_TO_PREFIX_TUNING_POSTPROCESS_MAPPING.get(self.config.model_type, None) is not None:
+                post_process_fn = TRANSFORMERS_MODELS_TO_PREFIX_TUNING_POSTPROCESS_MAPPING[self.config.model_type]
+                past_key_values = post_process_fn(past_key_values)
+            return past_key_values
+        else:
+            if peft_config.inference_mode:
+                prompts = prompt_encoder.embedding.weight.repeat(batch_size, 1, 1)
+            else:
+                prompts = prompt_encoder(prompt_tokens)
+            return prompts
+
+    def print_trainable_parameters(self):
+        """
+        Prints the number of trainable parameters in the model.
+        """
+        trainable_params = 0
+        all_param = 0
+        for _, param in self.named_parameters():
+            num_params = param.numel()
+            # if using DS Zero 3 and the weights are initialized empty
+            if num_params == 0 and hasattr(param, "ds_numel"):
+                num_params = param.ds_numel
+
+            all_param += num_params
+            if param.requires_grad:
+                trainable_params += num_params
+        print(
+            f"trainable params: {trainable_params:,d} || all params: {all_param:,d} || trainable%: {100 * trainable_params / all_param}"
+        )
+
+    def __getattr__(self, name: str):
+        """Forward missing attributes to the wrapped module."""
+        try:
+            return super().__getattr__(name)  # defer to nn.Module's logic
+        except AttributeError:
+            return getattr(self.base_model, name)
+
+    def forward(self, *args: Any, **kwargs: Any):
+        """
+        Forward pass of the model.
+        """
+        return self.get_base_model()(*args, **kwargs)
+
+    @contextmanager
+    def disable_adapter(self):
+        """
+        Disables the adapter module.
+        """
+        try:
+            if isinstance(self.peft_config[self.active_adapter], PromptLearningConfig):
+                old_forward = self.forward
+                self.forward = self.base_model.forward
+            else:
+                self.base_model.disable_adapter_layers()
+            yield
+        finally:
+            if isinstance(self.peft_config[self.active_adapter], PromptLearningConfig):
+                self.forward = old_forward
+            else:
+                self.base_model.enable_adapter_layers()
+
+    def get_base_model(self):
+        """
+        Returns the base model.
+        """
+        return self.base_model if isinstance(self.active_peft_config, PromptLearningConfig) else self.base_model.model
+
+    def add_adapter(self, adapter_name: str, peft_config: PeftConfig):
+        if peft_config.peft_type != self.peft_type:
+            raise ValueError(
+                f"Cannot combine adapters with different peft types. "
+                f"Found {self.peft_type} and {peft_config.peft_type}."
+            )
+        self.peft_config[adapter_name] = peft_config
+        if isinstance(peft_config, PromptLearningConfig):
+            self._setup_prompt_encoder(adapter_name)
+        else:
+            self.base_model.add_adapter(adapter_name, peft_config)
+
+        self.set_additional_trainable_modules(peft_config, adapter_name)
+
+    def set_additional_trainable_modules(self, peft_config, adapter_name):
+        if getattr(peft_config, "modules_to_save", None) is not None:
+            if self.modules_to_save is None:
+                self.modules_to_save = set(peft_config.modules_to_save)
+            else:
+                self.modules_to_save.update(peft_config.modules_to_save)
+            _set_trainable(self, adapter_name)
+
+    @classmethod
+    def _split_kwargs(cls, kwargs: Dict[str, Any]):
+        hf_hub_download_kwargs = {}
+        other_kwargs = {}
+
+        for key, value in kwargs.items():
+            if key in inspect.signature(hf_hub_download).parameters:
+                hf_hub_download_kwargs[key] = value
+            else:
+                other_kwargs[key] = value
+
+        return hf_hub_download_kwargs, other_kwargs
+
+    def load_adapter(self, model_id: str, adapter_name: str, is_trainable: bool = False, **kwargs: Any):
+        from .mapping import PEFT_TYPE_TO_CONFIG_MAPPING
+
+        hf_hub_download_kwargs, kwargs = self._split_kwargs(kwargs)
+
+        if adapter_name not in self.peft_config:
+            # load the config
+            peft_config = PEFT_TYPE_TO_CONFIG_MAPPING[
+                PeftConfig._get_peft_type(
+                    model_id,
+                    subfolder=kwargs.get("subfolder", None),
+                    revision=kwargs.get("revision", None),
+                    cache_dir=kwargs.get("cache_dir", None),
+                )
+            ].from_pretrained(
+                model_id,
+                subfolder=kwargs.get("subfolder", None),
+                revision=kwargs.get("revision", None),
+                cache_dir=kwargs.get("cache_dir", None),
+            )
+            if isinstance(peft_config, PromptLearningConfig) and is_trainable:
+                raise ValueError("Cannot set a prompt learning adapter to trainable when loading pretrained adapter.")
+            else:
+                peft_config.inference_mode = not is_trainable
+            self.add_adapter(adapter_name, peft_config)
+
+        # load weights if any
+        path = os.path.join(model_id, kwargs["subfolder"]) if kwargs.get("subfolder", None) is not None else model_id
+
+        if os.path.exists(os.path.join(path, SAFETENSORS_WEIGHTS_NAME)):
+            filename = os.path.join(path, SAFETENSORS_WEIGHTS_NAME)
+            use_safetensors = True
+        elif os.path.exists(os.path.join(path, WEIGHTS_NAME)):
+            filename = os.path.join(path, WEIGHTS_NAME)
+            use_safetensors = False
+        else:
+            has_remote_safetensors_file = hub_file_exists(
+                model_id, SAFETENSORS_WEIGHTS_NAME, revision=kwargs.get("revision", None)
+            )
+            use_safetensors = has_remote_safetensors_file
+
+            if has_remote_safetensors_file:
+                # Priority 1: load safetensors weights
+                filename = hf_hub_download(
+                    model_id,
+                    SAFETENSORS_WEIGHTS_NAME,
+                    subfolder=kwargs.get("subfolder", None),
+                    **hf_hub_download_kwargs,
+                )
+            else:
+                try:
+                    filename = hf_hub_download(
+                        model_id, WEIGHTS_NAME, subfolder=kwargs.get("subfolder", None), **hf_hub_download_kwargs
+                    )
+                except EntryNotFoundError:
+                    raise ValueError(
+                        f"Can't find weights for {model_id} in {model_id} or in the Hugging Face Hub. "
+                        f"Please check that the file {WEIGHTS_NAME} or {SAFETENSORS_WEIGHTS_NAME} is present at {model_id}."
+                    )
+
+        if use_safetensors:
+            adapters_weights = safe_load_file(filename, device="cuda" if torch.cuda.is_available() else "cpu")
+        else:
+            adapters_weights = torch.load(
+                filename, map_location=torch.device("cuda" if torch.cuda.is_available() else "cpu")
+            )
+
+        # load the weights into the model
+        load_result = set_peft_model_state_dict(self, adapters_weights, adapter_name=adapter_name)
+        if (
+            (getattr(self, "hf_device_map", None) is not None)
+            and (len(set(self.hf_device_map.values()).intersection({"cpu", "disk"})) > 0)
+            and len(self.peft_config) == 1
+        ):
+            device_map = kwargs.get("device_map", "auto")
+            max_memory = kwargs.get("max_memory", None)
+            offload_dir = kwargs.get("offload_folder", None)
+            offload_index = kwargs.get("offload_index", None)
+
+            dispatch_model_kwargs = {}
+            # Safety checker for previous `accelerate` versions
+            # `offload_index` was introduced in https://github.com/huggingface/accelerate/pull/873/
+            if "offload_index" in inspect.signature(dispatch_model).parameters:
+                dispatch_model_kwargs["offload_index"] = offload_index
+
+            no_split_module_classes = self._no_split_modules
+
+            if device_map != "sequential":
+                max_memory = get_balanced_memory(
+                    self,
+                    max_memory=max_memory,
+                    no_split_module_classes=no_split_module_classes,
+                    low_zero=(device_map == "balanced_low_0"),
+                )
+            if isinstance(device_map, str):
+                device_map = infer_auto_device_map(
+                    self, max_memory=max_memory, no_split_module_classes=no_split_module_classes
+                )
+            dispatch_model(
+                self,
+                device_map=device_map,
+                offload_dir=offload_dir,
+                **dispatch_model_kwargs,
+            )
+            hook = AlignDevicesHook(io_same_device=True)
+            if isinstance(self.peft_config[adapter_name], PromptLearningConfig):
+                remove_hook_from_submodules(self.prompt_encoder)
+            add_hook_to_module(self.get_base_model(), hook)
+
+        # Set model in evaluation mode to deactivate Dropout modules by default
+        if not is_trainable:
+            self.eval()
+        return load_result
+
+    def set_adapter(self, adapter_name: str):
+        """
+        Sets the active adapter.
+        """
+        if adapter_name not in self.peft_config:
+            raise ValueError(f"Adapter {adapter_name} not found.")
+        self.active_adapter = adapter_name
+        if not isinstance(self.peft_config[adapter_name], PromptLearningConfig):
+            self.base_model.set_adapter(adapter_name)
+        _set_adapter(self, adapter_name)
+
+    @property
+    def active_peft_config(self):
+        return self.peft_config[self.active_adapter]
+
+    def create_or_update_model_card(self, output_dir: str):
+        """
+        Updates or create model card to include information about peft:
+        1. Adds `peft` library tag
+        2. Adds peft version
+        3. Adds quantization information if it was used
+        """
+        # Adds `peft` library tag
+        add_library_to_model_card(output_dir)
+
+        with open(os.path.join(output_dir, "README.md"), "r") as f:
+            lines = f.readlines()
+
+        quantization_config = None
+        if hasattr(self.config, "quantization_config"):
+            quantization_config = self.config.quantization_config.to_dict()
+        training_config_text = ""
+        # Adds quantization information if it was used
+        if quantization_config is not None:
+            training_config_text += "\nThe following `bitsandbytes` quantization config was used during training:\n"
+            training_config_text += "\n".join([f"- {name}: {value}" for name, value in quantization_config.items()])
+            training_config_text += "\n"
+
+        training_procedure_heading = "## Training procedure\n"
+        if training_procedure_heading in lines:
+            lines.insert(lines.index(training_procedure_heading) + 2, training_config_text)
+        else:
+            lines.append(f"{training_procedure_heading}\n{training_config_text}")
+
+        # Adds peft version
+        framework_block_heading = "### Framework versions\n"
+        if framework_block_heading in lines:
+            lines.insert(lines.index(framework_block_heading) + 2, f"- PEFT {__version__}\n")
+        else:
+            lines.append(f"{framework_block_heading}\n\n- PEFT {__version__}\n")
+
+        # write the lines back to README.md
+        with open(os.path.join(output_dir, "README.md"), "w") as f:
+            f.writelines(lines)
+
+
+class PeftModelForSequenceClassification(PeftModel):
+    """
+    Peft model for sequence classification tasks.
+
+    Args:
+        model ([`~transformers.PreTrainedModel`]): Base transformer model.
+        peft_config ([`PeftConfig`]): Peft config.
+
+    **Attributes**:
+        - **config** ([`~transformers.PretrainedConfig`]) -- The configuration object of the base model.
+        - **cls_layer_name** (`str`) -- The name of the classification layer.
+
+    Example:
+
+        ```py
+        >>> from transformers import AutoModelForSequenceClassification
+        >>> from peft import PeftModelForSequenceClassification, get_peft_config
+
+        >>> config = {
+        ...     "peft_type": "PREFIX_TUNING",
+        ...     "task_type": "SEQ_CLS",
+        ...     "inference_mode": False,
+        ...     "num_virtual_tokens": 20,
+        ...     "token_dim": 768,
+        ...     "num_transformer_submodules": 1,
+        ...     "num_attention_heads": 12,
+        ...     "num_layers": 12,
+        ...     "encoder_hidden_size": 768,
+        ...     "prefix_projection": False,
+        ...     "postprocess_past_key_value_function": None,
+        ... }
+
+        >>> peft_config = get_peft_config(config)
+        >>> model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased")
+        >>> peft_model = PeftModelForSequenceClassification(model, peft_config)
+        >>> peft_model.print_trainable_parameters()
+        trainable params: 370178 || all params: 108680450 || trainable%: 0.3406113979101117
+        ```
+    """
+
+    def __init__(self, model, peft_config: PeftConfig, adapter_name="default"):
+        super().__init__(model, peft_config, adapter_name)
+        if self.modules_to_save is None:
+            self.modules_to_save = {"classifier", "score"}
+        else:
+            self.modules_to_save.update({"classifier", "score"})
+
+        for name, _ in self.base_model.named_children():
+            if any(module_name in name for module_name in self.modules_to_save):
+                self.cls_layer_name = name
+                break
+
+        # to make sure classifier layer is trainable
+        _set_trainable(self, adapter_name)
+
+    def forward(
+        self,
+        input_ids=None,
+        attention_mask=None,
+        inputs_embeds=None,
+        labels=None,
+        output_attentions=None,
+        output_hidden_states=None,
+        return_dict=None,
+        **kwargs,
+    ):
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+        peft_config = self.active_peft_config
+        if not isinstance(peft_config, PromptLearningConfig):
+            return self.base_model(
+                input_ids=input_ids,
+                attention_mask=attention_mask,
+                inputs_embeds=inputs_embeds,
+                labels=labels,
+                output_attentions=output_attentions,
+                output_hidden_states=output_hidden_states,
+                return_dict=return_dict,
+                **kwargs,
+            )
+
+        batch_size = input_ids.shape[0]
+        if attention_mask is not None:
+            # concat prompt attention mask
+            prefix_attention_mask = torch.ones(batch_size, peft_config.num_virtual_tokens).to(attention_mask.device)
+            attention_mask = torch.cat((prefix_attention_mask, attention_mask), dim=1)
+        if kwargs.get("position_ids", None) is not None:
+            warnings.warn("Position ids are not supported for parameter efficient tuning. Ignoring position ids.")
+            kwargs["position_ids"] = None
+        kwargs.update(
+            {
+                "attention_mask": attention_mask,
+                "labels": labels,
+                "output_attentions": output_attentions,
+                "output_hidden_states": output_hidden_states,
+                "return_dict": return_dict,
+            }
+        )
+
+        if peft_config.peft_type == PeftType.PREFIX_TUNING:
+            return self._prefix_tuning_forward(input_ids=input_ids, **kwargs)
+        else:
+            if kwargs.get("token_type_ids", None) is not None:
+                kwargs["token_type_ids"] = torch.cat(
+                    (
+                        torch.zeros(batch_size, peft_config.num_virtual_tokens).to(self.word_embeddings.weight.device),
+                        kwargs["token_type_ids"],
+                    ),
+                    dim=1,
+                ).long()
+            if inputs_embeds is None:
+                inputs_embeds = self.word_embeddings(input_ids)
+            prompts = self.get_prompt(batch_size=batch_size)
+            prompts = prompts.to(inputs_embeds.dtype)
+            inputs_embeds = torch.cat((prompts, inputs_embeds), dim=1)
+            return self.base_model(inputs_embeds=inputs_embeds, **kwargs)
+
+    def _prefix_tuning_forward(
+        self,
+        input_ids=None,
+        attention_mask=None,
+        inputs_embeds=None,
+        labels=None,
+        output_attentions=None,
+        output_hidden_states=None,
+        return_dict=None,
+        **kwargs,
+    ):
+        batch_size = input_ids.shape[0]
+        past_key_values = self.get_prompt(batch_size)
+        fwd_params = list(inspect.signature(self.base_model.forward).parameters.keys())
+        kwargs.update(
+            {
+                "input_ids": input_ids,
+                "attention_mask": attention_mask,
+                "inputs_embeds": inputs_embeds,
+                "output_attentions": output_attentions,
+                "output_hidden_states": output_hidden_states,
+                "return_dict": return_dict,
+                "past_key_values": past_key_values,
+            }
+        )
+        if "past_key_values" in fwd_params:
+            return self.base_model(labels=labels, **kwargs)
+        else:
+            transformer_backbone_name = self.base_model.get_submodule(self.transformer_backbone_name)
+            fwd_params = list(inspect.signature(transformer_backbone_name.forward).parameters.keys())
+            if "past_key_values" not in fwd_params:
+                raise ValueError("Model does not support past key values which are required for prefix tuning.")
+            outputs = transformer_backbone_name(**kwargs)
+            pooled_output = outputs[1] if len(outputs) > 1 else outputs[0]
+            if "dropout" in [name for name, _ in list(self.base_model.named_children())]:
+                pooled_output = self.base_model.dropout(pooled_output)
+            logits = self.base_model.get_submodule(self.cls_layer_name)(pooled_output)
+
+            loss = None
+            if labels is not None:
+                if self.config.problem_type is None:
+                    if self.base_model.num_labels == 1:
+                        self.config.problem_type = "regression"
+                    elif self.base_model.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
+                        self.config.problem_type = "single_label_classification"
+                    else:
+                        self.config.problem_type = "multi_label_classification"
+
+                if self.config.problem_type == "regression":
+                    loss_fct = MSELoss()
+                    if self.base_model.num_labels == 1:
+                        loss = loss_fct(logits.squeeze(), labels.squeeze())
+                    else:
+                        loss = loss_fct(logits, labels)
+                elif self.config.problem_type == "single_label_classification":
+                    loss_fct = CrossEntropyLoss()
+                    loss = loss_fct(logits.view(-1, self.base_model.num_labels), labels.view(-1))
+                elif self.config.problem_type == "multi_label_classification":
+                    loss_fct = BCEWithLogitsLoss()
+                    loss = loss_fct(logits, labels)
+            if not return_dict:
+                output = (logits,) + outputs[2:]
+                return ((loss,) + output) if loss is not None else output
+
+            return SequenceClassifierOutput(
+                loss=loss,
+                logits=logits,
+                hidden_states=outputs.hidden_states,
+                attentions=outputs.attentions,
+            )
+
+
+class PeftModelForCausalLM(PeftModel):
+    """
+    Peft model for causal language modeling.
+
+    Args:
+        model ([`~transformers.PreTrainedModel`]): Base transformer model.
+        peft_config ([`PeftConfig`]): Peft config.
+
+
+    Example:
+
+        ```py
+        >>> from transformers import AutoModelForCausalLM
+        >>> from peft import PeftModelForCausalLM, get_peft_config
+
+        >>> config = {
+        ...     "peft_type": "PREFIX_TUNING",
+        ...     "task_type": "CAUSAL_LM",
+        ...     "inference_mode": False,
+        ...     "num_virtual_tokens": 20,
+        ...     "token_dim": 1280,
+        ...     "num_transformer_submodules": 1,
+        ...     "num_attention_heads": 20,
+        ...     "num_layers": 36,
+        ...     "encoder_hidden_size": 1280,
+        ...     "prefix_projection": False,
+        ...     "postprocess_past_key_value_function": None,
+        ... }
+
+        >>> peft_config = get_peft_config(config)
+        >>> model = AutoModelForCausalLM.from_pretrained("gpt2-large")
+        >>> peft_model = PeftModelForCausalLM(model, peft_config)
+        >>> peft_model.print_trainable_parameters()
+        trainable params: 1843200 || all params: 775873280 || trainable%: 0.23756456724479544
+        ```
+    """
+
+    def __init__(self, model, peft_config: PeftConfig, adapter_name="default"):
+        super().__init__(model, peft_config, adapter_name)
+        # 备份self.base_model_prepare_inputs_for_generation
+        self.base_model_prepare_inputs_for_generation = self.base_model.prepare_inputs_for_generation
+        self.base_model._validate_model_kwargs = self.base_model_validate_model_kwargs
+
+    def forward(
+        self,
+        input_ids=None,
+        attention_mask=None,
+        inputs_embeds=None,
+        labels=None,
+        output_attentions=None,
+        output_hidden_states=None,
+        return_dict=None,
+        **kwargs,
+    ):
+        peft_config = self.active_peft_config
+        
+        # 添加
+        # print("kwargs = ", kwargs)
+        
+        if not isinstance(peft_config, PromptLearningConfig):
+            if self.base_model.config.model_type == "mpt":
+                if inputs_embeds is not None:
+                    raise AssertionError("forward in MPTForCausalLM does not support inputs_embeds")
+                return self.base_model(
+                    input_ids=input_ids,
+                    attention_mask=attention_mask,
+                    labels=labels,
+                    output_attentions=output_attentions,
+                    output_hidden_states=output_hidden_states,
+                    return_dict=return_dict,
+                    **kwargs,
+                )
+
+            return self.base_model(
+                input_ids=input_ids,
+                attention_mask=attention_mask,
+                inputs_embeds=inputs_embeds,
+                labels=labels,
+                output_attentions=output_attentions,
+                output_hidden_states=output_hidden_states,
+                return_dict=return_dict,
+                **kwargs,
+            )
+
+        batch_size = input_ids.shape[0]
+        if attention_mask is not None:
+            # concat prompt attention mask
+            prefix_attention_mask = torch.ones(batch_size, peft_config.num_virtual_tokens).to(attention_mask.device)
+            attention_mask = torch.cat((prefix_attention_mask, attention_mask), dim=1)
+
+        if kwargs.get("position_ids", None) is not None:
+            warnings.warn("Position ids are not supported for parameter efficient tuning. Ignoring position ids.")
+            kwargs["position_ids"] = None
+        if kwargs.get("token_type_ids", None) is not None:
+            warnings.warn("Token type ids are not supported for parameter efficient tuning. Ignoring token type ids")
+            kwargs["token_type_ids"] = None
+        kwargs.update(
+            {
+                "attention_mask": attention_mask,
+                "labels": labels,
+                "output_attentions": output_attentions,
+                "output_hidden_states": output_hidden_states,
+                "return_dict": return_dict,
+            }
+        )
+
+        if peft_config.peft_type == PeftType.PREFIX_TUNING:
+            past_key_values = self.get_prompt(batch_size)
+            return self.base_model(input_ids=input_ids, past_key_values=past_key_values, **kwargs)
+        else:
+            if inputs_embeds is None:
+                inputs_embeds = self.word_embeddings(input_ids)
+            # concat prompt labels
+            if labels is not None:
+                prefix_labels = torch.full((batch_size, peft_config.num_virtual_tokens), -100).to(labels.device)
+                kwargs["labels"] = torch.cat((prefix_labels, labels), dim=1)
+            prompts = self.get_prompt(batch_size=batch_size)
+            prompts = prompts.to(inputs_embeds.dtype)
+            inputs_embeds = torch.cat((prompts, inputs_embeds), dim=1)
+            return self.base_model(inputs_embeds=inputs_embeds, **kwargs)
+
+    def generate(self, **kwargs):
+        self.base_model.prepare_inputs_for_generation = self.prepare_inputs_for_generation
+        if hasattr(self.base_model, "model"):
+            self.base_model.model.generation_config = self.generation_config
+        else:
+            self.base_model.generation_config = self.generation_config
+        try:
+            # MoeLoRAModel.generate
+            outputs = self.base_model.generate(**kwargs)
+        except:
+            # 引发异常
+            self.base_model.prepare_inputs_for_generation = self.base_model_prepare_inputs_for_generation
+            raise
+        else:
+            self.base_model.prepare_inputs_for_generation = self.base_model_prepare_inputs_for_generation
+            return outputs
+
+    def prepare_inputs_for_generation(self, *args, **kwargs):
+        peft_config = self.active_peft_config
+        model_kwargs = self.base_model_prepare_inputs_for_generation(*args, **kwargs)
+        if isinstance(peft_config, PromptLearningConfig):
+            if model_kwargs.get("attention_mask", None) is not None:
+                prefix_attention_mask = torch.ones(
+                    model_kwargs["input_ids"].shape[0], peft_config.num_virtual_tokens
+                ).to(model_kwargs["input_ids"].device)
+                model_kwargs["attention_mask"] = torch.cat(
+                    (prefix_attention_mask, model_kwargs["attention_mask"]), dim=1
+                )
+
+            if model_kwargs.get("position_ids", None) is not None:
+                warnings.warn("Position ids are not supported for parameter efficient tuning. Ignoring position ids.")
+                model_kwargs["position_ids"] = None
+
+            if kwargs.get("token_type_ids", None) is not None:
+                warnings.warn(
+                    "Token type ids are not supported for parameter efficient tuning. Ignoring token type ids"
+                )
+                kwargs["token_type_ids"] = None
+
+            if model_kwargs["past_key_values"] is None and peft_config.peft_type == PeftType.PREFIX_TUNING:
+                past_key_values = self.get_prompt(batch_size=model_kwargs["input_ids"].shape[0])
+                model_kwargs["past_key_values"] = past_key_values
+            else:
+                if model_kwargs["past_key_values"] is None:
+                    inputs_embeds = self.word_embeddings(model_kwargs["input_ids"])
+                    prompts = self.get_prompt(batch_size=model_kwargs["input_ids"].shape[0])
+                    prompts = prompts.to(inputs_embeds.dtype)
+                    model_kwargs["inputs_embeds"] = torch.cat((prompts, inputs_embeds), dim=1)
+                    model_kwargs["input_ids"] = None
+        # !!!
+        model_kwargs["user_embeds"] = None
+        return model_kwargs
+
+    # !!!
+    def base_model_validate_model_kwargs(self, model_kwargs: Dict[str, Any]):
+        """Validates model kwargs for generation. Generate argument typos will also be caught here."""
+        pass
+        # If a `Cache` instance is passed, checks whether the model is compatible with it
+        if isinstance(model_kwargs.get("past_key_values", None), Cache) and not self._supports_cache_class:
+            raise ValueError(
+                f"{self.__class__.__name__} does not support an instance of `Cache` as `past_key_values`. Please "
+                "check the model documentation for supported cache formats."
+            )
+
+        # Excludes arguments that are handled before calling any model function
+        if self.config.is_encoder_decoder:
+            for key in ["decoder_input_ids"]:
+                model_kwargs.pop(key, None)
+
+        unused_model_args = []
+        model_args = set(inspect.signature(self.prepare_inputs_for_generation).parameters)
+        # `kwargs`/`model_kwargs` is often used to handle optional forward pass inputs like `attention_mask`. If
+        # `prepare_inputs_for_generation` doesn't accept them, then a stricter check can be made ;)
+        if "kwargs" in model_args or "model_kwargs" in model_args:
+            model_args |= set(inspect.signature(self.forward).parameters)
+
+        # Encoder-Decoder models may also need Encoder arguments from `model_kwargs`
+        if self.config.is_encoder_decoder:
+            base_model = getattr(self, self.base_model_prefix, None)
+
+            # allow encoder kwargs
+            encoder = getattr(self, "encoder", None)
+            # `MusicgenForConditionalGeneration` has `text_encoder` and `audio_encoder`.
+            # Also, it has `base_model_prefix = "encoder_decoder"` but there is no `self.encoder_decoder`
+            # TODO: A better way to handle this.
+            if encoder is None and base_model is not None:
+                encoder = getattr(base_model, "encoder", None)
+
+            if encoder is not None:
+                encoder_model_args = set(inspect.signature(encoder.forward).parameters)
+                model_args |= encoder_model_args
+
+            # allow decoder kwargs
+            decoder = getattr(self, "decoder", None)
+            if decoder is None and base_model is not None:
+                decoder = getattr(base_model, "decoder", None)
+
+            if decoder is not None:
+                decoder_model_args = set(inspect.signature(decoder.forward).parameters)
+                model_args |= {f"decoder_{x}" for x in decoder_model_args}
+
+            # allow assistant_encoder_outputs to be passed if we're doing assisted generating
+            if "assistant_encoder_outputs" in model_kwargs:
+                model_args |= {"assistant_encoder_outputs"}
+
+        for key, value in model_kwargs.items():
+            if value is not None and key not in model_args:
+                unused_model_args.append(key)
+        
+        if unused_model_args:
+            raise ValueError(
+                f"The following `model_kwargs` are not used by the model: {unused_model_args} (note: typos in the"
+                " generate arguments will also show up in this list)"
+            )
+
+class PeftModelForSeq2SeqLM(PeftModel):
+    """
+    Peft model for sequence-to-sequence language modeling.
+
+    Args:
+        model ([`~transformers.PreTrainedModel`]): Base transformer model.
+        peft_config ([`PeftConfig`]): Peft config.
+
+
+    Example:
+
+        ```py
+        >>> from transformers import AutoModelForSeq2SeqLM
+        >>> from peft import PeftModelForSeq2SeqLM, get_peft_config
+
+        >>> config = {
+        ...     "peft_type": "LORA",
+        ...     "task_type": "SEQ_2_SEQ_LM",
+        ...     "inference_mode": False,
+        ...     "r": 8,
+        ...     "target_modules": ["q", "v"],
+        ...     "lora_alpha": 32,
+        ...     "lora_dropout": 0.1,
+        ...     "fan_in_fan_out": False,
+        ...     "enable_lora": None,
+        ...     "bias": "none",
+        ... }
+
+        >>> peft_config = get_peft_config(config)
+        >>> model = AutoModelForSeq2SeqLM.from_pretrained("t5-base")
+        >>> peft_model = PeftModelForSeq2SeqLM(model, peft_config)
+        >>> peft_model.print_trainable_parameters()
+        trainable params: 884736 || all params: 223843584 || trainable%: 0.3952474242013566
+        ```
+    """
+
+    def __init__(self, model, peft_config: PeftConfig, adapter_name="default"):
+        super().__init__(model, peft_config, adapter_name)
+        self.base_model_prepare_inputs_for_generation = self.base_model.prepare_inputs_for_generation
+        self.base_model_prepare_encoder_decoder_kwargs_for_generation = (
+            self.base_model._prepare_encoder_decoder_kwargs_for_generation
+        )
+
+    def forward(
+        self,
+        input_ids=None,
+        attention_mask=None,
+        inputs_embeds=None,
+        decoder_input_ids=None,
+        decoder_attention_mask=None,
+        decoder_inputs_embeds=None,
+        labels=None,
+        output_attentions=None,
+        output_hidden_states=None,
+        return_dict=None,
+        **kwargs,
+    ):
+        peft_config = self.active_peft_config
+        if not isinstance(peft_config, PromptLearningConfig):
+            return self.base_model(
+                input_ids=input_ids,
+                attention_mask=attention_mask,
+                inputs_embeds=inputs_embeds,
+                decoder_input_ids=decoder_input_ids,
+                decoder_attention_mask=decoder_attention_mask,
+                decoder_inputs_embeds=decoder_inputs_embeds,
+                labels=labels,
+                output_attentions=output_attentions,
+                output_hidden_states=output_hidden_states,
+                return_dict=return_dict,
+                **kwargs,
+            )
+
+        batch_size = input_ids.shape[0]
+        if decoder_attention_mask is not None:
+            # concat prompt attention mask
+            prefix_attention_mask = torch.ones(batch_size, peft_config.num_virtual_tokens).to(
+                decoder_attention_mask.device
+            )
+            decoder_attention_mask = torch.cat((prefix_attention_mask, decoder_attention_mask), dim=1)
+
+        if kwargs.get("position_ids", None) is not None:
+            warnings.warn("Position ids are not supported for parameter efficient tuning. Ignoring position ids.")
+            kwargs["position_ids"] = None
+        if kwargs.get("token_type_ids", None) is not None:
+            warnings.warn("Token type ids are not supported for parameter efficient tuning. Ignoring token type ids")
+            kwargs["token_type_ids"] = None
+        kwargs.update(
+            {
+                "attention_mask": attention_mask,
+                "decoder_attention_mask": decoder_attention_mask,
+                "labels": labels,
+                "output_attentions": output_attentions,
+                "output_hidden_states": output_hidden_states,
+                "return_dict": return_dict,
+            }
+        )
+
+        if peft_config.peft_type == PeftType.PREFIX_TUNING:
+            past_key_values = self.get_prompt(batch_size)
+            return self.base_model(
+                input_ids=input_ids, decoder_input_ids=decoder_input_ids, past_key_values=past_key_values, **kwargs
+            )
+        elif peft_config.peft_type in [PeftType.PROMPT_TUNING, PeftType.P_TUNING]:
+            if inputs_embeds is None:
+                inputs_embeds = self.word_embeddings(input_ids)
+
+            if attention_mask is not None:
+                # concat prompt attention mask
+                prefix_attention_mask = torch.ones(batch_size, peft_config.num_virtual_tokens).to(
+                    attention_mask.device
+                )
+                kwargs["attention_mask"] = torch.cat((prefix_attention_mask, attention_mask), dim=1)
+
+            prompts = self.get_prompt(batch_size=batch_size)
+            prompts = prompts.to(inputs_embeds.dtype)
+            inputs_embeds = torch.cat((prompts[:, : peft_config.num_virtual_tokens], inputs_embeds), dim=1)
+
+            return self.base_model(inputs_embeds=inputs_embeds, **kwargs)
+        else:
+            if inputs_embeds is None:
+                inputs_embeds = self.word_embeddings(input_ids)
+            if decoder_inputs_embeds is None and decoder_input_ids is None:
+                decoder_input_ids = shift_tokens_right(
+                    labels, self.config.pad_token_id, self.config.decoder_start_token_id
+                )
+                decoder_inputs_embeds = self.word_embeddings(decoder_input_ids)
+
+            if attention_mask is not None:
+                # concat prompt attention mask
+                prefix_attention_mask = torch.ones(batch_size, peft_config.num_virtual_tokens).to(
+                    attention_mask.device
+                )
+                kwargs["attention_mask"] = torch.cat((prefix_attention_mask, attention_mask), dim=1)
+            # concat prompt labels
+            if labels is not None:
+                if peft_config.num_transformer_submodules == 1:
+                    kwargs["labels"] = labels
+                elif peft_config.num_transformer_submodules == 2:
+                    prefix_labels = torch.full((batch_size, peft_config.num_virtual_tokens), -100).to(labels.device)
+                    kwargs["labels"] = torch.cat((prefix_labels, labels), dim=1)
+            prompts = self.get_prompt(batch_size=batch_size)
+            prompts = prompts.to(inputs_embeds.dtype)
+            inputs_embeds = torch.cat((prompts[:, : peft_config.num_virtual_tokens], inputs_embeds), dim=1)
+            if peft_config.num_transformer_submodules == 1:
+                return self.base_model(inputs_embeds=inputs_embeds, **kwargs)
+            elif peft_config.num_transformer_submodules == 2:
+                decoder_inputs_embeds = torch.cat(
+                    (prompts[:, peft_config.num_virtual_tokens :], decoder_inputs_embeds), dim=1
+                )
+                return self.base_model(
+                    inputs_embeds=inputs_embeds, decoder_inputs_embeds=decoder_inputs_embeds, **kwargs
+                )
+
+    def generate(self, **kwargs):
+        peft_config = self.active_peft_config
+        self.base_model.prepare_inputs_for_generation = self.prepare_inputs_for_generation
+        self.base_model._prepare_encoder_decoder_kwargs_for_generation = (
+            self._prepare_encoder_decoder_kwargs_for_generation
+        )
+        try:
+            if not isinstance(peft_config, PromptLearningConfig):
+                outputs = self.base_model.generate(**kwargs)
+            else:
+                if "input_ids" not in kwargs:
+                    raise ValueError("input_ids must be provided for Peft model generation")
+                if kwargs.get("position_ids", None) is not None:
+                    warnings.warn(
+                        "Position ids are not supported for parameter efficient tuning. Ignoring position ids."
+                    )
+                    kwargs["position_ids"] = None
+                if kwargs.get("token_type_ids", None) is not None:
+                    warnings.warn(
+                        "Token type ids are not supported for parameter efficient tuning. Ignoring token type ids"
+                    )
+                    kwargs["token_type_ids"] = None
+
+                if peft_config.peft_type == PeftType.PREFIX_TUNING:
+                    outputs = self.base_model.generate(**kwargs)
+                elif peft_config.peft_type in [PeftType.PROMPT_TUNING, PeftType.P_TUNING]:
+                    kwargs = deepcopy(kwargs)
+
+                    if "encoder_outputs" in kwargs:
+                        del kwargs["encoder_ouputs"]
+                        warnings.warn(
+                            "`encoder_outputs` should not be passed to `generate` when using prompt tuning. Ignoring it."
+                        )
+
+                    input_ids = kwargs.pop("input_ids")
+                    inputs_embeds = self.word_embeddings(input_ids)
+                    batch_size = inputs_embeds.shape[0]
+                    prompts = self.get_prompt(batch_size=batch_size)
+                    prompts = prompts.to(inputs_embeds.dtype)
+
+                    inputs_embeds = torch.cat((prompts[:, : peft_config.num_virtual_tokens], inputs_embeds), dim=1)
+                    kwargs["inputs_embeds"] = inputs_embeds
+
+                    if "attention_mask" in kwargs:
+                        prefix_attention_mask = torch.ones(batch_size, peft_config.num_virtual_tokens).to(
+                            kwargs["attention_mask"].device
+                        )
+                        kwargs["attention_mask"] = torch.cat((prefix_attention_mask, kwargs["attention_mask"]), dim=1)
+
+                    return self.base_model.generate(**kwargs)
+                else:
+                    raise NotImplementedError
+        except:
+            self.base_model.prepare_inputs_for_generation = self.base_model_prepare_inputs_for_generation
+            self.base_model._prepare_encoder_decoder_kwargs_for_generation = (
+                self.base_model_prepare_encoder_decoder_kwargs_for_generation
+            )
+            raise
+        else:
+            self.base_model.prepare_inputs_for_generation = self.base_model_prepare_inputs_for_generation
+            self.base_model._prepare_encoder_decoder_kwargs_for_generation = (
+                self.base_model_prepare_encoder_decoder_kwargs_for_generation
+            )
+            return outputs
+
+    def prepare_inputs_for_generation(self, *args, **kwargs):
+        peft_config = self.active_peft_config
+        model_kwargs = self.base_model_prepare_inputs_for_generation(*args, **kwargs)
+        if model_kwargs["past_key_values"] is None and peft_config.peft_type == PeftType.PREFIX_TUNING:
+            batch_size = model_kwargs["decoder_input_ids"].shape[0]
+            past_key_values = self.get_prompt(batch_size)
+            model_kwargs["past_key_values"] = past_key_values
+
+        return model_kwargs
+
+
+class PeftModelForTokenClassification(PeftModel):
+    """
+    Peft model for token classification tasks.
+
+    Args:
+        model ([`~transformers.PreTrainedModel`]): Base transformer model.
+        peft_config ([`PeftConfig`]): Peft config.
+
+    **Attributes**:
+        - **config** ([`~transformers.PretrainedConfig`]) -- The configuration object of the base model.
+        - **cls_layer_name** (`str`) -- The name of the classification layer.
+
+    Example:
+
+        ```py
+        >>> from transformers import AutoModelForSequenceClassification
+        >>> from peft import PeftModelForTokenClassification, get_peft_config
+
+        >>> config = {
+        ...     "peft_type": "PREFIX_TUNING",
+        ...     "task_type": "TOKEN_CLS",
+        ...     "inference_mode": False,
+        ...     "num_virtual_tokens": 20,
+        ...     "token_dim": 768,
+        ...     "num_transformer_submodules": 1,
+        ...     "num_attention_heads": 12,
+        ...     "num_layers": 12,
+        ...     "encoder_hidden_size": 768,
+        ...     "prefix_projection": False,
+        ...     "postprocess_past_key_value_function": None,
+        ... }
+
+        >>> peft_config = get_peft_config(config)
+        >>> model = AutoModelForTokenClassification.from_pretrained("bert-base-cased")
+        >>> peft_model = PeftModelForTokenClassification(model, peft_config)
+        >>> peft_model.print_trainable_parameters()
+        trainable params: 370178 || all params: 108680450 || trainable%: 0.3406113979101117
+        ```
+    """
+
+    def __init__(self, model, peft_config: PeftConfig = None, adapter_name="default"):
+        super().__init__(model, peft_config, adapter_name)
+        if self.modules_to_save is None:
+            self.modules_to_save = {"classifier", "score"}
+        else:
+            self.modules_to_save.update({"classifier", "score"})
+
+        for name, _ in self.base_model.named_children():
+            if any(module_name in name for module_name in self.modules_to_save):
+                self.cls_layer_name = name
+                break
+
+        # to make sure classifier layer is trainable
+        _set_trainable(self, adapter_name)
+
+    def forward(
+        self,
+        input_ids=None,
+        attention_mask=None,
+        inputs_embeds=None,
+        labels=None,
+        output_attentions=None,
+        output_hidden_states=None,
+        return_dict=None,
+        **kwargs,
+    ):
+        peft_config = self.active_peft_config
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        if not isinstance(peft_config, PromptLearningConfig):
+            return self.base_model(
+                input_ids=input_ids,
+                attention_mask=attention_mask,
+                inputs_embeds=inputs_embeds,
+                labels=labels,
+                output_attentions=output_attentions,
+                output_hidden_states=output_hidden_states,
+                return_dict=return_dict,
+                **kwargs,
+            )
+
+        batch_size = input_ids.shape[0]
+        if attention_mask is not None:
+            # concat prompt attention mask
+            prefix_attention_mask = torch.ones(batch_size, peft_config.num_virtual_tokens).to(attention_mask.device)
+            attention_mask = torch.cat((prefix_attention_mask, attention_mask), dim=1)
+        if kwargs.get("position_ids", None) is not None:
+            warnings.warn("Position ids are not supported for parameter efficient tuning. Ignoring position ids.")
+            kwargs["position_ids"] = None
+        kwargs.update(
+            {
+                "attention_mask": attention_mask,
+                "labels": labels,
+                "output_attentions": output_attentions,
+                "output_hidden_states": output_hidden_states,
+                "return_dict": return_dict,
+            }
+        )
+
+        if peft_config.peft_type == PeftType.PREFIX_TUNING:
+            return self._prefix_tuning_forward(input_ids=input_ids, **kwargs)
+        else:
+            if kwargs.get("token_type_ids", None) is not None:
+                kwargs["token_type_ids"] = torch.cat(
+                    (
+                        torch.zeros(batch_size, peft_config.num_virtual_tokens).to(self.word_embeddings.weight.device),
+                        kwargs["token_type_ids"],
+                    ),
+                    dim=1,
+                ).long()
+            if inputs_embeds is None:
+                inputs_embeds = self.word_embeddings(input_ids)
+            prompts = self.get_prompt(batch_size=batch_size)
+            prompts = prompts.to(inputs_embeds.dtype)
+            inputs_embeds = torch.cat((prompts, inputs_embeds), dim=1)
+            return self.base_model(inputs_embeds=inputs_embeds, **kwargs)
+
+    def _prefix_tuning_forward(
+        self,
+        input_ids=None,
+        attention_mask=None,
+        inputs_embeds=None,
+        labels=None,
+        output_attentions=None,
+        output_hidden_states=None,
+        return_dict=None,
+        **kwargs,
+    ):
+        batch_size = input_ids.shape[0]
+        past_key_values = self.get_prompt(batch_size)
+        fwd_params = list(inspect.signature(self.base_model.forward).parameters.keys())
+        kwargs.update(
+            {
+                "input_ids": input_ids,
+                "attention_mask": attention_mask,
+                "inputs_embeds": inputs_embeds,
+                "output_attentions": output_attentions,
+                "output_hidden_states": output_hidden_states,
+                "return_dict": return_dict,
+                "past_key_values": past_key_values,
+            }
+        )
+        if "past_key_values" in fwd_params:
+            return self.base_model(labels=labels, **kwargs)
+        else:
+            transformer_backbone_name = self.base_model.get_submodule(self.transformer_backbone_name)
+            fwd_params = list(inspect.signature(transformer_backbone_name.forward).parameters.keys())
+            if "past_key_values" not in fwd_params:
+                raise ValueError("Model does not support past key values which are required for prefix tuning.")
+            outputs = transformer_backbone_name(**kwargs)
+            sequence_output = outputs[0]
+            if "dropout" in [name for name, _ in list(self.base_model.named_children())]:
+                sequence_output = self.base_model.dropout(sequence_output)
+            logits = self.base_model.get_submodule(self.cls_layer_name)(sequence_output)
+
+            loss = None
+            if labels is not None:
+                loss_fct = CrossEntropyLoss()
+                loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
+
+            if not return_dict:
+                output = (logits,) + outputs[2:]
+                return ((loss,) + output) if loss is not None else output
+
+            return TokenClassifierOutput(
+                loss=loss,
+                logits=logits,
+                hidden_states=outputs.hidden_states,
+                attentions=outputs.attentions,
+            )
+
+
+class PeftModelForQuestionAnswering(PeftModel):
+    """
+    Peft model for extractive question answering.
+
+    Args:
+        model ([`~transformers.PreTrainedModel`]): Base transformer model.
+        peft_config ([`PeftConfig`]): Peft config.
+
+    **Attributes**:
+        - **config** ([`~transformers.PretrainedConfig`]) -- The configuration object of the base model.
+        - **cls_layer_name** (`str`) -- The name of the classification layer.
+
+    Example:
+
+        ```py
+        >>> from transformers import AutoModelForQuestionAnswering
+        >>> from peft import PeftModelForQuestionAnswering, get_peft_config
+
+        >>> config = {
+        ...     "peft_type": "LORA",
+        ...     "task_type": "QUESTION_ANS",
+        ...     "inference_mode": False,
+        ...     "r": 16,
+        ...     "target_modules": ["query", "value"],
+        ...     "lora_alpha": 32,
+        ...     "lora_dropout": 0.05,
+        ...     "fan_in_fan_out": False,
+        ...     "bias": "none",
+        ... }
+
+        >>> peft_config = get_peft_config(config)
+        >>> model = AutoModelForQuestionAnswering.from_pretrained("bert-base-cased")
+        >>> peft_model = PeftModelForQuestionAnswering(model, peft_config)
+        >>> peft_model.print_trainable_parameters()
+        trainable params: 592900 || all params: 108312580 || trainable%: 0.5473971721475013
+        ```
+    """
+
+    def __init__(self, model, peft_config: PeftConfig = None, adapter_name="default"):
+        super().__init__(model, peft_config, adapter_name)
+        if self.modules_to_save is None:
+            self.modules_to_save = {"qa_outputs"}
+        else:
+            self.modules_to_save.update({"qa_outputs"})
+
+        for name, _ in self.base_model.named_children():
+            if any(module_name in name for module_name in self.modules_to_save):
+                self.cls_layer_name = name
+                break
+
+        # to make sure classifier layer is trainable
+        _set_trainable(self, adapter_name)
+
+    def forward(
+        self,
+        input_ids=None,
+        attention_mask=None,
+        token_type_ids=None,
+        position_ids=None,
+        inputs_embeds=None,
+        start_positions=None,
+        end_positions=None,
+        output_attentions=None,
+        output_hidden_states=None,
+        return_dict=None,
+        **kwargs,
+    ):
+        peft_config = self.active_peft_config
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        if not isinstance(peft_config, PromptLearningConfig):
+            return self.base_model(
+                input_ids=input_ids,
+                attention_mask=attention_mask,
+                inputs_embeds=inputs_embeds,
+                start_positions=start_positions,
+                end_positions=end_positions,
+                output_attentions=output_attentions,
+                output_hidden_states=output_hidden_states,
+                return_dict=return_dict,
+                **kwargs,
+            )
+
+        batch_size = input_ids.shape[0]
+        if attention_mask is not None:
+            # concat prompt attention mask
+            prefix_attention_mask = torch.ones(batch_size, peft_config.num_virtual_tokens).to(attention_mask.device)
+            attention_mask = torch.cat((prefix_attention_mask, attention_mask), dim=1)
+        if kwargs.get("position_ids", None) is not None:
+            warnings.warn("Position ids are not supported for parameter efficient tuning. Ignoring position ids.")
+            kwargs["position_ids"] = None
+        kwargs.update(
+            {
+                "attention_mask": attention_mask,
+                "start_positions": start_positions,
+                "end_positions": end_positions,
+                "output_attentions": output_attentions,
+                "output_hidden_states": output_hidden_states,
+                "return_dict": return_dict,
+            }
+        )
+
+        if peft_config.peft_type == PeftType.PREFIX_TUNING:
+            return self._prefix_tuning_forward(input_ids=input_ids, **kwargs)
+        else:
+            if kwargs.get("token_type_ids", None) is not None:
+                kwargs["token_type_ids"] = torch.cat(
+                    (
+                        torch.zeros(batch_size, peft_config.num_virtual_tokens).to(self.word_embeddings.weight.device),
+                        kwargs["token_type_ids"],
+                    ),
+                    dim=1,
+                ).long()
+            if inputs_embeds is None:
+                inputs_embeds = self.word_embeddings(input_ids)
+            prompts = self.get_prompt(batch_size=batch_size)
+            prompts = prompts.to(inputs_embeds.dtype)
+            inputs_embeds = torch.cat((prompts, inputs_embeds), dim=1)
+            return self.base_model(inputs_embeds=inputs_embeds, **kwargs)
+
+    def _prefix_tuning_forward(
+        self,
+        input_ids=None,
+        attention_mask=None,
+        inputs_embeds=None,
+        start_positions=None,
+        end_positions=None,
+        output_attentions=None,
+        output_hidden_states=None,
+        return_dict=None,
+        **kwargs,
+    ):
+        batch_size = input_ids.shape[0]
+        past_key_values = self.get_prompt(batch_size)
+        fwd_params = list(inspect.signature(self.base_model.forward).parameters.keys())
+        kwargs.update(
+            {
+                "input_ids": input_ids,
+                "attention_mask": attention_mask,
+                "inputs_embeds": inputs_embeds,
+                "output_attentions": output_attentions,
+                "output_hidden_states": output_hidden_states,
+                "return_dict": return_dict,
+                "past_key_values": past_key_values,
+            }
+        )
+        if "past_key_values" in fwd_params:
+            return self.base_model(start_positions=start_positions, end_positions=end_positions, **kwargs)
+        else:
+            transformer_backbone_name = self.base_model.get_submodule(self.transformer_backbone_name)
+            fwd_params = list(inspect.signature(transformer_backbone_name.forward).parameters.keys())
+            if "past_key_values" not in fwd_params:
+                raise ValueError("Model does not support past key values which are required for prefix tuning.")
+            outputs = transformer_backbone_name(**kwargs)
+            sequence_output = outputs[0]
+            if "dropout" in [name for name, _ in list(self.base_model.named_children())]:
+                sequence_output = self.base_model.dropout(sequence_output)
+            logits = self.base_model.get_submodule(self.cls_layer_name)(sequence_output)
+            start_logits, end_logits = logits.split(1, dim=-1)
+            start_logits = start_logits.squeeze(-1).contiguous()
+            end_logits = end_logits.squeeze(-1).contiguous()
+
+            total_loss = None
+            if start_positions is not None and end_positions is not None:
+                # If we are on multi-GPU, split add a dimension
+                if len(start_positions.size()) > 1:
+                    start_positions = start_positions.squeeze(-1)
+                if len(end_positions.size()) > 1:
+                    end_positions = end_positions.squeeze(-1)
+                # sometimes the start/end positions are outside our model inputs, we ignore these terms
+                ignored_index = start_logits.size(1)
+                start_positions = start_positions.clamp(0, ignored_index)
+                end_positions = end_positions.clamp(0, ignored_index)
+
+                loss_fct = CrossEntropyLoss(ignore_index=ignored_index)
+                start_loss = loss_fct(start_logits, start_positions)
+                end_loss = loss_fct(end_logits, end_positions)
+                total_loss = (start_loss + end_loss) / 2
+
+            if not return_dict:
+                output = (start_logits, end_logits) + outputs[2:]
+                return ((total_loss,) + output) if total_loss is not None else output
+
+            return QuestionAnsweringModelOutput(
+                loss=total_loss,
+                start_logits=start_logits,
+                end_logits=end_logits,
+                hidden_states=outputs.hidden_states,
+                attentions=outputs.attentions,
+            )

model/peft/tuners/__init__.py

@@ -0,0 +1,27 @@
+# flake8: noqa
+# There's no way to ignore "F401 '...' imported but unused" warnings in this
+# module, but to preserve other warnings. So, don't check this module at all
+
+# coding=utf-8
+# Copyright 2023-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .adaption_prompt import AdaptionPromptConfig, AdaptionPromptModel
+from .lora import LoraConfig, LoraModel
+from .adalora import AdaLoraConfig, AdaLoraModel
+from .p_tuning import PromptEncoder, PromptEncoderConfig, PromptEncoderReparameterizationType
+from .prefix_tuning import PrefixEncoder, PrefixTuningConfig
+from .prompt_tuning import PromptEmbedding, PromptTuningConfig, PromptTuningInit
+from .moelora import MoeLoraConfig, MoeLoraModel
+from .gating import GATING_TO_MODEL_MAPPING

model/peft/tuners/adalora.py

@@ -0,0 +1,751 @@
+import re
+import warnings
+from dataclasses import dataclass, field
+from typing import Optional
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from transformers.pytorch_utils import Conv1D
+
+from ..import_utils import is_bnb_4bit_available, is_bnb_available
+from ..utils import (
+    TRANSFORMERS_MODELS_TO_ADALORA_TARGET_MODULES_MAPPING,
+    PeftType,
+    _freeze_adapter,
+    _get_submodules,
+    transpose,
+)
+from .lora import (
+    LoraConfig,
+    LoraLayer,
+    LoraModel,
+    mark_only_lora_as_trainable,
+)
+
+
+if is_bnb_available():
+    import bitsandbytes as bnb
+
+
+@dataclass
+class AdaLoraConfig(LoraConfig):
+    """
+    This is the configuration class to store the configuration of a [`~peft.AdaLora`].
+
+    Args:
+        target_r (`int`): The target average rank of incremental matrix.
+        init_r (`int`): The initial rank for each incremental matrix.
+        tinit (`int`): The steps of initial fine-tuning warmup.
+        tfinal (`int`): The step of final fine-tuning.
+        deltaT (`int`): The time internval between two budget allocations.
+        beta1 (`float`): The hyperparameter of EMA for sensitivity smoothing.
+        beta2 (`float`): The hyperparameter of EMA for undertainty quantification.
+        orth_reg_weight (`float`): The coefficient of orthogonal regularization.
+        total_step (`int`): The total training steps that should be specified before training.
+        rank_pattern (`list`): The allocated rank for each weight matrix by RankAllocator.
+    """
+
+    target_r: int = field(default=8, metadata={"help": "Target Lora matrix dimension."})
+    init_r: int = field(default=12, metadata={"help": "Intial Lora matrix dimension."})
+    tinit: int = field(default=0, metadata={"help": "The steps of initial warmup."})
+    tfinal: int = field(default=0, metadata={"help": "The steps of final warmup."})
+    deltaT: int = field(default=1, metadata={"help": "Step interval of rank allocation."})
+    beta1: float = field(default=0.85, metadata={"help": "Hyperparameter of EMA."})
+    beta2: float = field(default=0.85, metadata={"help": "Hyperparameter of EMA."})
+    orth_reg_weight: float = field(default=0.5, metadata={"help": "The orthogonal regularization coefficient."})
+    total_step: Optional[int] = field(default=None, metadata={"help": "The total training steps."})
+    rank_pattern: Optional[dict] = field(default=None, metadata={"help": "The saved rank pattern."})
+
+    def __post_init__(self):
+        self.peft_type = PeftType.ADALORA
+
+
+class AdaLoraModel(LoraModel):
+    """
+    Creates AdaLoRA (Adaptive LoRA) model from a pretrained transformers model. Paper:
+    https://openreview.net/pdf?id=lq62uWRJjiY
+
+    Args:
+        model ([`transformers.PreTrainedModel`]): The model to be adapted.
+        config ([`AdaLoraConfig`]): The configuration of the AdaLora model.
+
+    Returns:
+        `torch.nn.Module`: The AdaLora model.
+
+    Example::
+
+        >>> from transformers import AutoModelForSeq2SeqLM, LoraConfig >>> from peft import AdaLoraModel, AdaLoraConfig
+        >>> config = AdaLoraConfig(
+                peft_type="ADALORA", task_type="SEQ_2_SEQ_LM", r=8, lora_alpha=32, target_modules=["q", "v"],
+                lora_dropout=0.01,
+            )
+        >>> model = AutoModelForSeq2SeqLM.from_pretrained("t5-base") >>> model = AdaLoraModel(config, model)
+
+    **Attributes**:
+        - **model** ([`transformers.PreTrainedModel`]) -- The model to be adapted.
+        - **peft_config** ([`AdaLoraConfig`]): The configuration of the AdaLora model.
+    """
+
+    def __init__(self, model, config, adapter_name):
+        nn.Module.__init__(self)
+        self.model = model
+        self.peft_config = config
+        self.add_adapter(adapter_name, self.peft_config[adapter_name])
+
+    def add_adapter(self, adapter_name, config=None):
+        if config is not None:
+            model_config = self.model.config.to_dict() if hasattr(self.model.config, "to_dict") else self.model.config
+            config = self._prepare_adalora_config(config, model_config)
+            self.peft_config[adapter_name] = config
+        self._find_and_replace(adapter_name)
+        if len(self.peft_config) > 1 and self.peft_config[adapter_name].bias != "none":
+            raise ValueError(
+                "AdaLoraModel supports only 1 adapter with bias. When using multiple adapters, set bias to 'none' for all adapters."
+            )
+        traininable_mode_counter = 0
+        for config in self.peft_config.values():
+            if not config.inference_mode:
+                traininable_mode_counter += 1
+
+        if traininable_mode_counter > 1:
+            raise ValueError(
+                "AdaLoraModel supports only 1 trainable adapter. "
+                "When using multiple adapters, set inference_mode to True for all adapters except the one you want to train."
+            )
+
+        mark_only_lora_as_trainable(self.model, self.peft_config[adapter_name].bias)
+        if self.peft_config[adapter_name].inference_mode:
+            _freeze_adapter(self.model, adapter_name)
+        else:
+            self.trainable_adapter_name = adapter_name
+            self.rankallocator = RankAllocator(self.model, self.peft_config[adapter_name], self.trainable_adapter_name)
+
+    def _find_and_replace(self, adapter_name):
+        lora_config = self.peft_config[adapter_name]
+        loaded_in_8bit = getattr(self.model, "is_loaded_in_8bit", False)
+        loaded_in_4bit = getattr(self.model, "is_loaded_in_4bit", False)
+
+        if (loaded_in_8bit or loaded_in_4bit) and not is_bnb_available():
+            raise ImportError(
+                "To use Lora with 8-bit quantization, please install the `bitsandbytes` package. "
+                "You can install it with `pip install bitsandbytes`."
+            )
+        is_target_modules_in_base_model = False
+        kwargs = {
+            "r": lora_config.init_r,
+            "lora_alpha": lora_config.lora_alpha,
+            "lora_dropout": lora_config.lora_dropout,
+            "fan_in_fan_out": lora_config.fan_in_fan_out,
+            "init_lora_weights": lora_config.init_lora_weights,
+        }
+        key_list = [key for key, _ in self.model.named_modules()]
+        for key in key_list:
+            if isinstance(lora_config.target_modules, str):
+                target_module_found = re.fullmatch(lora_config.target_modules, key)
+            else:
+                target_module_found = any(key.endswith(target_key) for target_key in lora_config.target_modules)
+            if target_module_found:
+                if not is_target_modules_in_base_model:
+                    is_target_modules_in_base_model = True
+                parent, target, target_name = _get_submodules(self.model, key)
+                bias = target.bias is not None
+                if isinstance(target, LoraLayer):
+                    target.update_layer(
+                        adapter_name,
+                        lora_config.init_r,
+                        lora_config.lora_alpha,
+                        lora_config.lora_dropout,
+                        lora_config.init_lora_weights,
+                    )
+                else:
+                    if loaded_in_8bit and isinstance(target, bnb.nn.Linear8bitLt):
+                        kwargs.update(
+                            {
+                                "has_fp16_weights": target.state.has_fp16_weights,
+                                "memory_efficient_backward": target.state.memory_efficient_backward,
+                                "threshold": target.state.threshold,
+                                "index": target.index,
+                            }
+                        )
+                        new_module = SVDLinear8bitLt(
+                            adapter_name, target.in_features, target.out_features, bias=bias, **kwargs
+                        )
+                    elif loaded_in_4bit and is_bnb_4bit_available() and isinstance(target, bnb.nn.Linear4bit):
+                        fourbit_kwargs = kwargs.copy()
+                        fourbit_kwargs.update(
+                            {
+                                "compute_dtype": target.compute_dtype,
+                                "compress_statistics": target.weight.compress_statistics,
+                                "quant_type": target.weight.quant_type,
+                            }
+                        )
+                        new_module = SVDLinear4bit(
+                            adapter_name, target.in_features, target.out_features, bias=bias, **fourbit_kwargs
+                        )
+                    else:
+                        if isinstance(target, torch.nn.Linear):
+                            in_features, out_features = target.in_features, target.out_features
+                            if kwargs["fan_in_fan_out"]:
+                                warnings.warn(
+                                    "fan_in_fan_out is set to True but the target module is `torch.nn.Linear`. "
+                                    "Setting fan_in_fan_out to False."
+                                )
+                                kwargs["fan_in_fan_out"] = lora_config.fan_in_fan_out = False
+                        elif isinstance(target, Conv1D):
+                            in_features, out_features = (
+                                target.weight.ds_shape if hasattr(target.weight, "ds_shape") else target.weight.shape
+                            )
+                            if not kwargs["fan_in_fan_out"]:
+                                warnings.warn(
+                                    "fan_in_fan_out is set to False but the target module is `Conv1D`. "
+                                    "Setting fan_in_fan_out to True."
+                                )
+                                kwargs["fan_in_fan_out"] = lora_config.fan_in_fan_out = True
+                        else:
+                            raise ValueError(
+                                f"Target module {target} is not supported. "
+                                f"Currently, only `torch.nn.Linear` and `Conv1D` are supported."
+                            )
+                        new_module = SVDLinear(adapter_name, in_features, out_features, bias=bias, **kwargs)
+
+                    self._replace_module(parent, target_name, new_module, target)
+        if not is_target_modules_in_base_model:
+            raise ValueError(
+                f"Target modules {lora_config.target_modules} not found in the base model. "
+                f"Please check the target modules and try again."
+            )
+
+    def __getattr__(self, name: str):
+        """Forward missing attributes to the wrapped module."""
+        try:
+            return super().__getattr__(name)  # defer to nn.Module's logic
+        except AttributeError:
+            return getattr(self.model, name)
+
+    def forward(self, *args, **kwargs):
+        outputs = self.model.forward(*args, **kwargs)
+
+        # Calculate the orthogonal regularization
+        orth_reg_weight = self.peft_config[self.trainable_adapter_name].orth_reg_weight
+        assert orth_reg_weight > 0
+
+        if hasattr(outputs, "loss"):
+            regu_loss = 0
+            num_param = 0
+            for n, p in self.model.named_parameters():
+                if ("lora_A" in n or "lora_B" in n) and self.trainable_adapter_name in n:
+                    para_cov = p @ p.T if "lora_A" in n else p.T @ p
+                    I = torch.eye(*para_cov.size(), out=torch.empty_like(para_cov))
+                    I.requires_grad = False
+                    num_param += 1
+                    regu_loss += torch.norm(para_cov - I, p="fro")
+            if num_param > 0:
+                regu_loss = regu_loss / num_param
+            else:
+                regu_loss = 0
+            outputs.loss += orth_reg_weight * regu_loss
+        return outputs
+
+    def resize_modules_by_rank_pattern(self, rank_pattern, adapter_name):
+        lora_config = self.peft_config[adapter_name]
+        for name, rank_idx in rank_pattern.items():
+            if isinstance(rank_idx, list):
+                rank = sum(rank_idx)
+            elif isinstance(rank_idx, torch.Tensor):
+                rank_idx = rank_idx.view(-1)
+                rank = rank_idx.sum().item()
+            else:
+                raise ValueError("Unexcepted type of rank_idx")
+            key = ".".join(name.split(".")[0:-2]) if adapter_name in name else ".".join(name.split(".")[0:-1])
+            _, target, _ = _get_submodules(self.model, key)
+            lora_E_weights = target.lora_E[adapter_name][rank_idx]
+            lora_A_weights = target.lora_A[adapter_name][rank_idx]
+            lora_B_weights = target.lora_B[adapter_name][:, rank_idx]
+            ranknum = target.ranknum[adapter_name]
+            target.update_layer(
+                adapter_name,
+                rank,
+                lora_config.lora_alpha,
+                lora_config.lora_dropout,
+                lora_config.init_lora_weights,
+            )
+            with torch.no_grad():
+                if rank > 0:
+                    target.lora_E[adapter_name].copy_(lora_E_weights)
+                    target.lora_A[adapter_name].copy_(lora_A_weights)
+                    target.lora_B[adapter_name].copy_(lora_B_weights)
+                    # The scaling is exactly as the previous
+                    target.ranknum[adapter_name].copy_(ranknum)
+
+    def resize_state_dict_by_rank_pattern(self, rank_pattern, state_dict, adapter_name):
+        for name, rank_idx in rank_pattern.items():
+            rank = sum(rank_idx)
+            prefix = ".".join(name.split(".")[0:-2]) if adapter_name in name else ".".join(name.split(".")[0:-1])
+            for layer in ["lora_E", "lora_A", "lora_B"]:
+                key = f"base_model.model.{prefix}.{layer}.{adapter_name}"
+                if layer != "lora_B":
+                    state_dict[key] = (
+                        state_dict[key][rank_idx] if rank != state_dict[key].shape[0] else state_dict[key]
+                    )
+                else:
+                    state_dict[key] = (
+                        state_dict[key][:, rank_idx] if rank != state_dict[key].shape[1] else state_dict[key]
+                    )
+        return state_dict
+
+    def update_and_allocate(self, global_step):
+        lora_config = self.peft_config[self.trainable_adapter_name]
+        # Update the importance score and allocate the budget
+        if global_step < lora_config.total_step - lora_config.tfinal:
+            _, rank_pattern = self.rankallocator.update_and_allocate(self.model, global_step)
+            if rank_pattern:
+                lora_config.rank_pattern = rank_pattern
+        # Finalize the budget allocation
+        elif global_step == lora_config.total_step - lora_config.tfinal:
+            _, rank_pattern = self.rankallocator.update_and_allocate(self.model, global_step, force_mask=True)
+            # for some reason, this freezes the trainable parameters and nothing gets updates
+            # self.resize_modules_by_rank_pattern(rank_pattern, self.trainable_adapter_name)
+            lora_config.rank_pattern = rank_pattern
+            self.rankallocator.reset_ipt()
+        # Currently using inefficient way to mask the unimportant weights using the rank pattern
+        #  due to problem mentioned above
+        elif global_step > lora_config.total_step - lora_config.tfinal:
+            self.rankallocator.mask_using_rank_pattern(self.model, lora_config.rank_pattern)
+        # Pass the function and do forward propagation
+        else:
+            return None
+
+    @staticmethod
+    def _prepare_adalora_config(peft_config, model_config):
+        if peft_config.target_modules is None:
+            if model_config["model_type"] not in TRANSFORMERS_MODELS_TO_ADALORA_TARGET_MODULES_MAPPING:
+                raise ValueError("Please specify `target_modules` in `peft_config`")
+            peft_config.target_modules = TRANSFORMERS_MODELS_TO_ADALORA_TARGET_MODULES_MAPPING[
+                model_config["model_type"]
+            ]
+        return peft_config
+
+
+class AdaLoraLayer(LoraLayer):
+    def __init__(
+        self,
+        in_features: int,
+        out_features: int,
+    ):
+        super().__init__(in_features, out_features)
+        self.lora_E = nn.ParameterDict({})
+        self.lora_A = nn.ParameterDict({})
+        self.lora_B = nn.ParameterDict({})
+        self.ranknum = nn.ParameterDict({})
+
+    def update_layer(self, adapter_name, r, lora_alpha, lora_dropout, init_lora_weights):
+        self.r[adapter_name] = r
+        self.lora_alpha[adapter_name] = lora_alpha
+        if lora_dropout > 0.0:
+            lora_dropout_layer = nn.Dropout(p=lora_dropout)
+        else:
+
+            def lora_dropout_layer(x):
+                return x
+
+        self.lora_dropout.update(nn.ModuleDict({adapter_name: lora_dropout_layer}))
+        # Actual trainable parameters
+        # Right singular vectors
+        self.lora_A.update(nn.ParameterDict({adapter_name: nn.Parameter(torch.zeros(r, self.in_features))}))
+        # Singular values
+        self.lora_E.update(nn.ParameterDict({adapter_name: nn.Parameter(torch.zeros(r, 1))}))
+        # Left singular vectors
+        self.lora_B.update(nn.ParameterDict({adapter_name: nn.Parameter(torch.zeros(self.out_features, r))}))
+        # The current rank
+        self.ranknum.update(nn.ParameterDict({adapter_name: nn.Parameter(torch.zeros(1), requires_grad=False)}))
+        self.ranknum[adapter_name].data.fill_(float(r))
+        self.ranknum[adapter_name].requires_grad = False
+        self.scaling[adapter_name] = lora_alpha if lora_alpha > 0 else float(r)
+        if init_lora_weights:
+            self.reset_lora_parameters(adapter_name)
+        self.to(self.weight.device)
+
+    def reset_lora_parameters(self, adapter_name):
+        if adapter_name in self.lora_A.keys():
+            nn.init.zeros_(self.lora_E[adapter_name])
+            nn.init.normal_(self.lora_A[adapter_name], mean=0.0, std=0.02)
+            nn.init.normal_(self.lora_B[adapter_name], mean=0.0, std=0.02)
+
+
+class SVDLinear(nn.Linear, AdaLoraLayer):
+    # SVD-based adaptation by a dense layer
+    def __init__(
+        self,
+        adapter_name: str,
+        in_features: int,
+        out_features: int,
+        r: int = 0,
+        lora_alpha: int = 1,
+        lora_dropout: float = 0.0,
+        fan_in_fan_out: bool = False,
+        **kwargs,
+    ):
+        init_lora_weights = kwargs.pop("init_lora_weights", True)
+        nn.Linear.__init__(self, in_features, out_features, **kwargs)
+        AdaLoraLayer.__init__(self, in_features=in_features, out_features=out_features)
+        # Freezing the pre-trained weight matrix
+        self.weight.requires_grad = False
+
+        self.fan_in_fan_out = fan_in_fan_out
+        if fan_in_fan_out:
+            self.weight.data = self.weight.data.T
+
+        nn.Linear.reset_parameters(self)
+        self.update_layer(adapter_name, r, lora_alpha, lora_dropout, init_lora_weights)
+        self.active_adapter = adapter_name
+
+    def merge(self):
+        if self.active_adapter not in self.lora_A.keys():
+            return
+        if self.merged:
+            warnings.warn("Already merged. Nothing to do.")
+            return
+        if self.r[self.active_adapter] > 0:
+            self.weight.data += (
+                transpose(
+                    self.lora_B[self.active_adapter]
+                    @ (self.lora_A[self.active_adapter] * self.lora_E[self.active_adapter]),
+                    self.fan_in_fan_out,
+                )
+                * self.scaling[self.active_adapter]
+                / (self.ranknum[self.active_adapter] + 1e-5)
+            )
+            self.merged = True
+
+    def unmerge(self):
+        if self.active_adapter not in self.lora_A.keys():
+            return
+        if not self.merged:
+            warnings.warn("Already unmerged. Nothing to do.")
+            return
+        if self.r[self.active_adapter] > 0:
+            self.weight.data -= (
+                transpose(
+                    self.lora_B[self.active_adapter]
+                    @ (self.lora_A[self.active_adapter] * self.lora_E[self.active_adapter])
+                )
+                * self.scaling[self.active_adapter]
+                / (self.ranknum[self.active_adapter] + 1e-5)
+            )
+            self.merged = False
+
+    def forward(self, x: torch.Tensor):
+        if self.active_adapter not in self.lora_A.keys():
+            return F.linear(x, transpose(self.weight, self.fan_in_fan_out), bias=self.bias)
+        if self.disable_adapters:
+            if self.r[self.active_adapter] > 0 and self.merged:
+                self.unmerge()
+            result = F.linear(x, transpose(self.weight, self.fan_in_fan_out), bias=self.bias)
+        elif self.r[self.active_adapter] > 0 and not self.merged:
+            result = F.linear(x, transpose(self.weight, self.fan_in_fan_out), bias=self.bias)
+            result += (
+                (
+                    self.lora_dropout[self.active_adapter](x)
+                    @ (self.lora_A[self.active_adapter] * self.lora_E[self.active_adapter]).T
+                    @ self.lora_B[self.active_adapter].T
+                )
+                * self.scaling[self.active_adapter]
+                / (self.ranknum[self.active_adapter] + 1e-5)
+            )
+        else:
+            result = F.linear(x, transpose(self.weight, self.fan_in_fan_out), bias=self.bias)
+        return result
+
+
+if is_bnb_available():
+
+    class SVDLinear8bitLt(bnb.nn.Linear8bitLt, AdaLoraLayer):
+        # Low-rank matrix for SVD-based adaptation
+        def __init__(
+            self,
+            adapter_name,
+            in_features,
+            out_features,
+            r: int = 0,
+            lora_alpha: int = 1,
+            lora_dropout: float = 0.0,
+            **kwargs,
+        ):
+            bnb.nn.Linear8bitLt.__init__(
+                self,
+                in_features,
+                out_features,
+                bias=kwargs.get("bias", True),
+                has_fp16_weights=kwargs.get("has_fp16_weights", True),
+                memory_efficient_backward=kwargs.get("memory_efficient_backward", False),
+                threshold=kwargs.get("threshold", 0.0),
+                index=kwargs.get("index", None),
+            )
+            AdaLoraLayer.__init__(self, in_features=in_features, out_features=out_features)
+            # Freezing the pre-trained weight matrix
+            self.weight.requires_grad = False
+
+            init_lora_weights = kwargs.pop("init_lora_weights", True)
+            self.update_layer(adapter_name, r, lora_alpha, lora_dropout, init_lora_weights)
+            self.active_adapter = adapter_name
+
+        def forward(self, x: torch.Tensor):
+            result = super().forward(x)
+
+            if self.disable_adapters or self.active_adapter not in self.lora_A.keys():
+                return result
+            elif self.r[self.active_adapter] > 0:
+                if not torch.is_autocast_enabled():
+                    expected_dtype = result.dtype
+
+                    if x.dtype != torch.float32:
+                        x = x.float()
+                    output = (
+                        (
+                            self.lora_dropout[self.active_adapter](x)
+                            @ (self.lora_A[self.active_adapter] * self.lora_E[self.active_adapter]).T
+                            @ self.lora_B[self.active_adapter].T
+                        ).to(expected_dtype)
+                        * self.scaling[self.active_adapter]
+                        / (self.ranknum[self.active_adapter] + 1e-5)
+                    )
+                else:
+                    output = (
+                        (
+                            self.lora_dropout[self.active_adapter](x)
+                            @ (self.lora_A[self.active_adapter] * self.lora_E[self.active_adapter]).T
+                            @ self.lora_B[self.active_adapter].T
+                        )
+                        * self.scaling[self.active_adapter]
+                        / (self.ranknum[self.active_adapter] + 1e-5)
+                    )
+                result = result + output
+            return result
+
+
+if is_bnb_4bit_available():
+
+    class SVDLinear4bit(bnb.nn.Linear4bit, AdaLoraLayer):
+        # Low-rank matrix for SVD-based adaptation
+        def __init__(
+            self,
+            adapter_name,
+            in_features,
+            out_features,
+            r: int = 0,
+            lora_alpha: int = 1,
+            lora_dropout: float = 0.0,
+            **kwargs,
+        ):
+            bnb.nn.Linear4bit.__init__(
+                self,
+                in_features,
+                out_features,
+                bias=kwargs.get("bias", True),
+                compute_dtype=kwargs.get("compute_dtype", torch.float32),
+                compress_statistics=kwargs.get("compress_statistics", True),
+                quant_type=kwargs.get("quant_type", "nf4"),
+            )
+            AdaLoraLayer.__init__(self, in_features=in_features, out_features=out_features)
+            # Freezing the pre-trained weight matrix
+            self.weight.requires_grad = False
+
+            init_lora_weights = kwargs.pop("init_lora_weights", True)
+            self.update_layer(adapter_name, r, lora_alpha, lora_dropout, init_lora_weights)
+            self.active_adapter = adapter_name
+
+        def forward(self, x: torch.Tensor):
+            result = super().forward(x)
+
+            if self.disable_adapters or self.active_adapter not in self.lora_A.keys():
+                return result
+            elif self.r[self.active_adapter] > 0:
+                if not torch.is_autocast_enabled():
+                    expected_dtype = result.dtype
+
+                    if x.dtype != torch.float32:
+                        x = x.float()
+                    output = (
+                        (
+                            self.lora_dropout[self.active_adapter](x)
+                            @ (self.lora_A[self.active_adapter] * self.lora_E[self.active_adapter]).T
+                            @ self.lora_B[self.active_adapter].T
+                        ).to(expected_dtype)
+                        * self.scaling[self.active_adapter]
+                        / (self.ranknum[self.active_adapter] + 1e-5)
+                    )
+                else:
+                    output = (
+                        (
+                            self.lora_dropout[self.active_adapter](x)
+                            @ (self.lora_A[self.active_adapter] * self.lora_E[self.active_adapter]).T
+                            @ self.lora_B[self.active_adapter].T
+                        )
+                        * self.scaling[self.active_adapter]
+                        / (self.ranknum[self.active_adapter] + 1e-5)
+                    )
+                result = result + output
+            return result
+
+
+class RankAllocator(object):
+    """
+    The RankAllocator for AdaLoraModel. Paper: https://openreview.net/pdf?id=lq62uWRJjiY
+
+    Args:
+        config ([`AdaLoraConfig`]): The configuration of the AdaLora model.
+        model: the model that we apply AdaLoRA to.
+
+    """
+
+    def __init__(self, model, peft_config, adapter_name):
+        self.peft_config = peft_config
+        self.adapter_name = adapter_name
+        self.beta1 = peft_config.beta1
+        self.beta2 = peft_config.beta2
+        assert self.beta1 > 0 and self.beta1 < 1
+        assert self.beta2 > 0 and self.beta2 < 1
+
+        self.reset_ipt()
+        self._set_budget_scheduler(model)
+
+    def set_total_step(self, total_step):
+        self.peft_config.total_step = total_step
+
+    def reset_ipt(self):
+        self.ipt = {}
+        self.exp_avg_ipt = {}
+        self.exp_avg_unc = {}
+
+    def _set_budget_scheduler(self, model):
+        self.init_bgt = 0
+        self.name_set = set()
+        for n, p in model.named_parameters():
+            if f"lora_A.{self.adapter_name}" in n:
+                self.init_bgt += p.size(0)
+                self.name_set.add(n.replace("lora_A", "%s"))
+        self.name_set = sorted(self.name_set)
+        # The total final rank budget
+        self.target_bgt = self.peft_config.target_r * len(self.name_set)
+
+    def budget_schedule(self, step: int):
+        tinit = self.peft_config.tinit
+        tfinal = self.peft_config.tfinal
+        total_step = self.peft_config.total_step
+        # Initial warmup
+        if step <= tinit:
+            budget = self.init_bgt
+            mask_ind = False
+        # Final fine-tuning
+        elif step > total_step - tfinal:
+            budget = self.target_bgt
+            mask_ind = True
+        else:
+            # Budget decreasing with a cubic scheduler
+            mul_coeff = 1 - (step - tinit) / (total_step - tfinal - tinit)
+            budget = int((self.init_bgt - self.target_bgt) * (mul_coeff**3) + self.target_bgt)
+            mask_ind = True if step % self.peft_config.deltaT == 0 else False
+        return budget, mask_ind
+
+    def update_ipt(self, model):
+        # Update the sensitivity and uncertainty for every weight
+        for n, p in model.named_parameters():
+            if "lora_" in n and self.adapter_name in n:
+                if n not in self.ipt:
+                    self.ipt[n] = torch.zeros_like(p)
+                    self.exp_avg_ipt[n] = torch.zeros_like(p)
+                    self.exp_avg_unc[n] = torch.zeros_like(p)
+                with torch.no_grad():
+                    self.ipt[n] = (p * p.grad).abs().detach()
+                    # Sensitivity smoothing
+                    self.exp_avg_ipt[n] = self.beta1 * self.exp_avg_ipt[n] + (1 - self.beta1) * self.ipt[n]
+                    # Uncertainty quantification
+                    self.exp_avg_unc[n] = (
+                        self.beta2 * self.exp_avg_unc[n] + (1 - self.beta2) * (self.ipt[n] - self.exp_avg_ipt[n]).abs()
+                    )
+
+    def _element_score(self, n):
+        return self.exp_avg_ipt[n] * self.exp_avg_unc[n]
+
+    def _combine_ipt(self, ipt_E, ipt_AB):
+        ipt_AB = ipt_AB.sum(dim=1, keepdim=False)
+        sum_ipt = ipt_E.view(-1) + ipt_AB.view(-1)
+        return sum_ipt
+
+    def mask_to_budget(self, model, budget):
+        value_ipt = {}
+        vector_ipt = {}
+        triplet_ipt = {}
+        # Get the importance score for A, E, B
+        for n, p in model.named_parameters():
+            if f"lora_A.{self.adapter_name}" in n:
+                entry_ipt = self._element_score(n)
+                comb_ipt = torch.mean(entry_ipt, dim=1, keepdim=True)
+                name_m = n.replace("lora_A", "%s")
+                if name_m not in vector_ipt:
+                    vector_ipt[name_m] = [comb_ipt]
+                else:
+                    vector_ipt[name_m].append(comb_ipt)
+            if f"lora_B.{self.adapter_name}" in n:
+                entry_ipt = self._element_score(n)
+                comb_ipt = torch.mean(entry_ipt, dim=0, keepdim=False).view(-1, 1)
+                name_m = n.replace("lora_B", "%s")
+                if name_m not in vector_ipt:
+                    vector_ipt[name_m] = [comb_ipt]
+                else:
+                    vector_ipt[name_m].append(comb_ipt)
+            if f"lora_E.{self.adapter_name}" in n:
+                entry_ipt = self._element_score(n)
+                name_m = n.replace("lora_E", "%s")
+                value_ipt[name_m] = entry_ipt
+
+        all_score = []
+        # Calculate the score for each triplet
+        for name_m in vector_ipt:
+            ipt_E = value_ipt[name_m]
+            ipt_AB = torch.cat(vector_ipt[name_m], dim=1)
+            sum_ipt = self._combine_ipt(ipt_E, ipt_AB)
+            name_E = name_m % "lora_E"
+            triplet_ipt[name_E] = sum_ipt.view(-1, 1)
+            all_score.append(sum_ipt.view(-1))
+
+        # Get the threshold by ranking ipt
+        mask_threshold = torch.kthvalue(
+            torch.cat(all_score),
+            k=self.init_bgt - budget,
+        )[0].item()
+
+        rank_pattern = {}
+        # Mask the unimportant triplets
+        with torch.no_grad():
+            for n, p in model.named_parameters():
+                if f"lora_E.{self.adapter_name}" in n:
+                    p.masked_fill_(triplet_ipt[n] <= mask_threshold, 0.0)
+                    rank_pattern[n] = (~(triplet_ipt[n] <= mask_threshold)).view(-1).tolist()
+        return rank_pattern
+
+    def update_and_allocate(self, model, global_step, force_mask=False):
+        # # Update the importance score and allocate the budget
+        if global_step < self.peft_config.total_step - self.peft_config.tfinal:
+            self.update_ipt(model)
+        budget, mask_ind = self.budget_schedule(global_step)
+        # Allocate the budget according to importance scores
+        if mask_ind or force_mask:
+            rank_pattern = self.mask_to_budget(model, budget)
+        else:
+            rank_pattern = None
+        return budget, rank_pattern
+
+    def mask_using_rank_pattern(self, model, rank_pattern):
+        # Mask the unimportant triplets
+        is_adapter_name_truncated = False
+        if self.adapter_name not in next(iter(rank_pattern.keys())):
+            is_adapter_name_truncated = True
+
+        with torch.no_grad():
+            for n, p in model.named_parameters():
+                if f"lora_E.{self.adapter_name}" in n:
+                    key = n if not is_adapter_name_truncated else n.replace(f".{self.adapter_name}", "")
+                    mask = torch.Tensor(rank_pattern[key]).unsqueeze(-1).to(p.device)
+                    p.masked_fill_(~mask.bool(), 0.0)

model/peft/tuners/adaption_prompt.py

@@ -0,0 +1,368 @@
+# coding=utf-8
+# Copyright 2023-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+from collections import namedtuple
+from dataclasses import dataclass, field
+from typing import Dict, List
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from peft.utils.config import PeftConfig, PeftType
+from peft.utils.other import _freeze_adapter, _get_submodules
+
+
+def llama_rotate_half(x: torch.Tensor) -> torch.Tensor:
+    """
+    Rotate half the hidden dims of the input.
+
+    This function was duplicated verbatim from:
+    https://github.com/huggingface/transformers/blob/1de8ce9ee1191ba761a593ac15d9ccbf5851bfc5/src/transformers/models/llama/modeling_llama.py#L126
+
+    This was done to eliminate the Llama transformers implementation as a dependency of this file. Note that some other
+    functions were also adapted from the transformers implementation but were modified.
+    """
+    x1 = x[..., : x.shape[-1] // 2]
+    x2 = x[..., x.shape[-1] // 2 :]
+    return torch.cat((-x2, x1), dim=-1)
+
+
+def llama_apply_rotary_pos_emb(q, cos, sin, position_ids):
+    """
+    Apply rotary position embedding to query states in the Llama model.
+
+    This function was adapted from:
+    https://github.com/huggingface/transformers/blob/1de8ce9ee1191ba761a593ac15d9ccbf5851bfc5/src/transformers/models/llama/modeling_llama.py#L133
+
+    It was modified to remove unnecessary processing of key states.
+    """
+    gather_indices = position_ids[:, None, :, None]  # [bs, 1, seq_len, 1]
+    gather_indices = gather_indices.repeat(1, cos.shape[1], 1, cos.shape[3])
+    cos = torch.gather(cos.repeat(gather_indices.shape[0], 1, 1, 1), 2, gather_indices)
+    sin = torch.gather(sin.repeat(gather_indices.shape[0], 1, 1, 1), 2, gather_indices)
+    q_embed = (q * cos) + (llama_rotate_half(q) * sin)
+    return q_embed
+
+
+def llama_compute_query_states(model: nn.Module, **kwargs) -> torch.Tensor:
+    """
+    Compute query states for Llama models specifically.
+
+    They need to be recomputed as the forward() method of the original LlamaModel in the transformers library does not
+    return them. See the related discussion in the PR: https://github.com/huggingface/peft/pull/268
+    """
+    hidden_states = kwargs.get("hidden_states")
+    position_ids = kwargs.get("position_ids")
+    past_key_value = kwargs.get("past_key_value")
+    bsz, q_len, _ = hidden_states.size()
+    query_states = model.q_proj(hidden_states).view(bsz, q_len, model.num_heads, model.head_dim).transpose(1, 2)
+    value_states = model.v_proj(hidden_states).view(bsz, q_len, model.num_heads, model.head_dim).transpose(1, 2)
+
+    seq_len = q_len
+    if past_key_value is not None:
+        seq_len += past_key_value[0].shape[-2]
+    cos, sin = model.rotary_emb(value_states, seq_len=seq_len)
+
+    return llama_apply_rotary_pos_emb(query_states, cos, sin, position_ids)
+
+
+# Contains the config that is specific to a transformers model type.
+ModelTypeConfig = namedtuple(
+    "ModelTypeConfig", ["compute_query_states", "target_modules", "k_proj_layer", "v_proj_layer", "o_proj_layer"]
+)
+# Mapping of transformers model types to their specific configuration.
+TRANSFORMERS_MODEL_CONFIG = {
+    "llama": ModelTypeConfig(
+        compute_query_states=llama_compute_query_states,
+        target_modules="self_attn",
+        k_proj_layer="k_proj",
+        v_proj_layer="v_proj",
+        o_proj_layer="o_proj",
+    ),
+}
+
+
+def is_adaption_prompt_trainable(params: str) -> bool:
+    """Return True if module is trainable under adaption prompt fine-tuning."""
+    return params.split(".")[-1].startswith("adaption_")
+
+
+@dataclass
+class AdaptionPromptConfig(PeftConfig):
+    """Stores the configuration of an [`AdaptionPromptModel`]."""
+
+    target_modules: str = field(
+        default=None, metadata={"help": "Name of the attention submodules to insert adaption prompts into."}
+    )
+    adapter_len: int = field(default=None, metadata={"help": "Number of adapter tokens to insert"})
+    adapter_layers: int = field(default=None, metadata={"help": "Number of adapter layers (from the top)"})
+
+    def __post_init__(self):
+        self.peft_type = PeftType.ADAPTION_PROMPT
+
+
+def prepare_config(
+    peft_config: AdaptionPromptConfig,
+    model,
+) -> AdaptionPromptConfig:
+    """Prepare the config based on the llama model type."""
+    if model.config.model_type not in TRANSFORMERS_MODEL_CONFIG:
+        raise ValueError("Unsupported model type for adaption prompt: '{model.config.model_type}'.")
+
+    model_config = TRANSFORMERS_MODEL_CONFIG[model.config.model_type]
+
+    if peft_config.target_modules is None:
+        peft_config.target_modules = model_config.target_modules
+
+    return peft_config
+
+
+class AdaptionPromptModel(nn.Module):
+    """
+    Implements adaption prompts as described in https://arxiv.org/pdf/2303.16199.pdf.
+
+    The top L attention modules are replaced with AdaptedAttention modules that wrap the original ones, but insert
+    trainable prompts with gates (for zero init).
+
+    Notes on the multi-adapter pattern:
+    - We store the states of different adapters by keeping a dictionary of AdaptedAttention modules indexed by adapter
+      name.
+    - Every time we switch adapters, we remove the modules of the currently active adapter from the model, store them
+      in the dictionary, and replace them with the modules of the new adapter.
+    - To avoid duplicated and potentially inconsistent state, the currently active adapter is always removed from the
+      dictionary.
+    - Disabling the adapter would also result in the modules being removed from the model.
+    """
+
+    def __init__(self, model, configs: Dict, adapter_name: str):
+        super().__init__()
+        self.model = model
+        # Store adapter configs by name.
+        self._configs: Dict[str, AdaptionPromptConfig] = {}
+        # Store lists of the parents of the affected attention modules by adapter name.
+        # We keep references to the parents so we can swap the adapters in-and-out of the model.
+        self._parents: Dict[str, List[nn.Module]] = {}
+        # Store lists of cached AdaptedAttention modules by name.
+        self._cached_adapters: Dict[str, List] = {}
+        # The name of the currently active adapter.
+        self._active_adapter = None
+        # Whether the adapter is enabled.
+        self._enabled = True
+        self.forward = self.model.forward
+        self.add_adapter(adapter_name, configs[adapter_name])
+        self._mark_only_adaption_prompts_as_trainable()
+
+    def add_adapter(self, adapter_name: str, config: AdaptionPromptConfig) -> None:
+        """Add an adapter with the given name and config."""
+        config = prepare_config(config, self.model)
+        if adapter_name in self._configs:
+            raise ValueError(f"Adapter with name '{adapter_name}' already exists.")
+
+        parents = []
+        for name, _ in self.model.named_modules():
+            if name.endswith(config.target_modules):
+                par, _, _ = _get_submodules(self.model, name)
+                parents.append(par)
+        if len(parents) < config.adapter_layers:
+            raise ValueError(
+                f"Config specifies more adapter layers '{config.adapter_layers}'"
+                f" than the model has '{len(parents)}'."
+            )
+        # Note that if the target modules are not in Sequential, ModuleList, or
+        # some other PyTorch ordered container, the behavior is undefined as we
+        # assume here that the order of the modules is the same as the order of
+        # the transformer decoder layers.
+        parents = parents[-config.adapter_layers :]
+        self._parents[adapter_name] = parents
+
+        # It is only None during initialization.
+        # If it is disabled, we don't have to remove the modules.
+        if self._active_adapter is not None and self._enabled:
+            self._remove_adapted_attentions(self._active_adapter)
+        self._active_adapter = adapter_name
+        self._configs[adapter_name] = config
+        self._create_adapted_attentions(config, parents)
+        if not self._enabled:
+            self._remove_adapted_attentions(self._active_adapter)
+
+        if config.inference_mode:
+            _freeze_adapter(self.model, adapter_name)
+
+    def set_adapter(self, adapter_name: str) -> None:
+        """Set the model to use the adapter with the given name."""
+        if self._active_adapter == adapter_name:
+            return
+        if adapter_name not in self._configs:
+            raise ValueError(f"Adapter with name '{adapter_name}' does not exist.")
+
+        if self._enabled:
+            self._remove_adapted_attentions(self._active_adapter)
+            self._set_adapted_attentions(adapter_name)
+
+        self._active_adapter = adapter_name
+
+    def enable_adapter_layers(self):
+        """Enable adapter layers by swapping in cached AdaptedAttention modules."""
+        self._enabled = True
+        self._set_adapted_attentions(self._active_adapter)
+
+    def disable_adapter_layers(self):
+        """Disable adapter layers by swapping out AdaptedAttention modules."""
+        self._enabled = False
+        self._remove_adapted_attentions(self._active_adapter)
+
+    def _create_adapted_attentions(self, config: AdaptionPromptConfig, parents: List[nn.Module]) -> None:
+        """Wrap LlamaAttention modules with newly created AdaptedAttention modules."""
+        for par in parents:
+            attn = AdaptedAttention(
+                model_type=self.model.config.model_type,
+                adapter_len=config.adapter_len,
+                model=getattr(par, config.target_modules),
+            )
+            setattr(par, config.target_modules, attn)
+
+    def _set_adapted_attentions(self, adapter_name: str) -> None:
+        """Replace LlamaAttention modules with cached AdaptedAttention modules."""
+        cached = self._cached_adapters[adapter_name]
+        del self._cached_adapters[adapter_name]
+        config = self._configs[adapter_name]
+        for i, par in enumerate(self._parents[adapter_name]):
+            setattr(par, config.target_modules, cached[i])
+
+    def _remove_adapted_attentions(self, adapter_name: str) -> None:
+        """Remove AdaptedAttention modules from the model and store them in the cache."""
+        config = self._configs[adapter_name]
+        adapted_attentions = []
+        for par in self._parents[adapter_name]:
+            attn = getattr(par, config.target_modules)
+            adapted_attentions.append(attn)
+            setattr(par, config.target_modules, attn.model)
+        self._cached_adapters[adapter_name] = adapted_attentions
+
+    def _mark_only_adaption_prompts_as_trainable(self) -> None:
+        """Freeze all parameters of the model except the adaption prompts."""
+        for n, p in self.model.named_parameters():
+            if not is_adaption_prompt_trainable(n):
+                p.requires_grad = False
+
+    def __getattr__(self, name: str):
+        """Forward missing attributes to the wrapped module."""
+        try:
+            return super().__getattr__(name)  # defer to nn.Module's logic
+        except AttributeError:
+            # This is necessary as e.g. causal models have various methods that we
+            # don't want to re-implement here.
+            return getattr(self.model, name)
+
+
+class AdaptedAttention(nn.Module):
+    """This module wraps a LLamaAttention module and injects adaption prompts."""
+
+    def __init__(self, model_type: str, adapter_len: int, model):
+        """
+        Initialize object.
+
+        Args:
+            model_type: The transformer model type. This is used to retrieve the right method to
+                compute query states.
+            adapter_len: The length of the adaption prompt to insert.
+            model: The original transformer attention module that is being wrapped.
+        """
+        assert not isinstance(model, AdaptedAttention)
+        super().__init__()
+        self.model_type = model_type
+        self.model = model
+        self.adapter_len = adapter_len
+        # Assume all parameters of the attention model we are wrapping are on the same device.
+        device = next(model.parameters()).device
+        # Don't think this was specified in the paper, but we follow the official repo which used an Embedding
+        # which initializes the tokens with standard normal values.
+        # https://github.com/ZrrSkywalker/LLaMA-Adapter/blob/41c3546fe1997ab8a65809dc8d8f9252b19d9faf/llama/model.py#L234
+        # (bsz, adapter_len, hidden_size)
+        target_dtype = (
+            model.q_proj.weight.dtype if model.q_proj.weight.dtype not in [torch.int8, torch.uint8] else torch.float32
+        )
+        self.adaption_prompt = nn.Parameter(
+            torch.empty(1, adapter_len, self.model.hidden_size, device=device, dtype=target_dtype).normal_()
+        )
+        # Initialize the gate to 0 as this is "zero-init".
+        self.adaption_gate = nn.Parameter(torch.zeros(1, device=device, dtype=target_dtype))
+
+    def forward(self, **kwargs):
+        """
+        Forward pass for the adapter which wraps the original LlamaAttention module.
+
+        "Official" paper implementation:
+        https://github.com/ZrrSkywalker/LLaMA-Adapter/blob/41c3546fe1997ab8a65809dc8d8f9252b19d9faf/llama/model.py#L141
+
+        Args:
+            kwargs: See the original LlamaAttention module.
+        """
+        if kwargs.get("output_attention", False):
+            raise NotImplementedError("output_attention is not currently supported.")
+
+        output, _, past_key_value = self.model(**kwargs)
+        bsz = output.shape[0]
+        q_len = output.shape[1]
+        embed_dim = output.shape[2]
+        k_proj_layer = TRANSFORMERS_MODEL_CONFIG[self.model_type].k_proj_layer
+        v_proj_layer = TRANSFORMERS_MODEL_CONFIG[self.model_type].v_proj_layer
+        o_proj_layer = TRANSFORMERS_MODEL_CONFIG[self.model_type].o_proj_layer
+
+        if k_proj_layer == v_proj_layer:
+            _, key, value = getattr(self.model, k_proj_layer)(self.adaption_prompt).split(embed_dim, dim=2)
+        else:
+            key = getattr(self.model, k_proj_layer)(self.adaption_prompt)
+            value = getattr(self.model, v_proj_layer)(self.adaption_prompt)
+        # (bsz, num_heads, adapter_len, head_dim)
+        adapter_k = (
+            key.view(1, self.adapter_len, self.model.num_heads, self.model.head_dim)
+            .repeat(bsz, 1, 1, 1)
+            .transpose(1, 2)
+        )
+        # (bsz, num_heads, adapter_len, head_dim)
+        adapter_v = (
+            value.view(1, self.adapter_len, self.model.num_heads, self.model.head_dim)
+            .repeat(bsz, 1, 1, 1)
+            .transpose(1, 2)
+        )
+
+        # Recompute query states.
+        compute_query_states = TRANSFORMERS_MODEL_CONFIG[self.model_type].compute_query_states
+        # (bsz, num_heads, q_len, head_dim)
+        query_states = compute_query_states(model=self.model, **kwargs)
+
+        previous_dtype = query_states.dtype
+        # (bsz, num_heads, q_len, adapter_len)
+        scores = torch.matmul(query_states, adapter_k.transpose(2, 3).to(previous_dtype)) / math.sqrt(
+            self.model.head_dim
+        )
+        # Upcast attention to fp32
+        # (bsz, num_heads, q_len, adapter_len)
+        scores = self.adaption_gate * F.softmax(scores, dim=-1, dtype=torch.float32).to(previous_dtype)
+        # (bsz, q_len, num_heads * head_dim)
+        adapter_output = torch.matmul(scores, adapter_v).transpose(1, 2).reshape(bsz, q_len, -1)
+        # (bsz, q_len, hidden_size)
+        if o_proj_layer is not None:
+            adapter_output = getattr(self.model, o_proj_layer)(adapter_output)
+
+        # Add adaption prompt output to original output.
+        output = output + adapter_output
+
+        # Restore original dtype.
+        output = output.to(previous_dtype)
+        return output, None, past_key_value

model/peft/tuners/debug_utils.py

@@ -0,0 +1,64 @@
+import os
+import torch
+import matplotlib.pyplot as plt
+import numpy as np
+from typing import Optional
+
+class DebugUtils:
+    def __init__(self, pic_dir: str):
+        self.pic_dir = pic_dir
+        if not os.path.exists(self.pic_dir):
+            os.makedirs(self.pic_dir)
+
+    def check_orthogonal(self, tensor: torch.Tensor) -> float:
+        if tensor.dim() > 2:
+            raise ValueError("Tensor dimension must be 1 or 2.")
+
+        if tensor.dim() == 1:
+            return 0.0  # 1D tensor is not applicable for orthogonality
+
+        if tensor.dim() == 2:
+            num_rows, num_dims = tensor.shape
+            dot_products = torch.mm(tensor, tensor.t())
+            norms_squared = torch.diag(dot_products)
+            sum_abs_dot_products = torch.sum(torch.abs(dot_products - torch.diag(norms_squared)))
+            max_sum = (num_rows * (num_rows - 1)) / 2
+            orthogonality_score = 1.0 - (sum_abs_dot_products / max_sum)
+            return orthogonality_score.item()
+
+    def plot_tensor(self, tensor: torch.Tensor, pic_name: str, x_axis: Optional[str], y_axis: Optional[str], plot_number: bool = False) -> None:
+        if tensor.dim() > 2:
+            raise ValueError("Tensor dimensions must be 2 or lower for plotting.")
+        
+        fig, ax = plt.subplots()
+
+        if tensor.dim() == 1:
+            tensor_shape = tensor.shape[0]
+            ax.set_yticks([])
+            ax.set_xticks(np.arange(tensor_shape))
+            color_matrix = np.ones((1, tensor_shape, 3)) * [0.7, 0.7, 1.0]
+            ax.imshow(color_matrix, aspect='equal')
+            for i in range(1, tensor_shape):
+                ax.axvline(x=i - 0.5, color='black', linewidth=1)
+            if x_axis:
+                ax.set_xlabel(x_axis)
+        else:
+            heatmap = ax.imshow(tensor, cmap='YlOrRd', aspect='auto')
+
+            if plot_number:
+                for i in range(tensor.shape[0]):
+                    for j in range(tensor.shape[1]):
+                        ax.text(j, i, f'{tensor[i, j]:.3f}', ha='center', va='center', color='black')
+
+            if y_axis:
+                ax.set_ylabel(y_axis)
+            if x_axis:
+                ax.set_xlabel(x_axis)
+
+            plt.colorbar(heatmap, ax=ax, orientation='vertical')
+
+        plt.savefig(os.path.join(self.pic_dir, pic_name))
+        plt.close()
+
+
+debuge_utils = DebugUtils("./picture")

model/peft/tuners/gating.py

@@ -0,0 +1,124 @@
+import torch
+from torch import nn
+import torch.nn.functional as F
+
+class Dense(nn.Module):
+    def __init__(self, dim: int, num_moe: int) -> None:
+        super().__init__()
+        self.dim = 64
+        self.num_moe = num_moe
+        self.linear_layer = nn.Linear(self.dim, num_moe, bias=False)
+        self.softmax = nn.Softmax(dim=-1)
+
+    def forward(self, x):
+        logits = self.linear_layer(x)
+        probs = self.softmax(logits)
+        return probs
+
+class topK(nn.Module):
+    def __init__(self, dim: int, num_moe: int) -> None:
+        super().__init__()
+        self.dim = 64
+        self.num_moe = num_moe
+        self.linear_layer = nn.Linear(self.dim, num_moe, bias=False)
+        self.softmax = nn.Softmax(dim=-1)
+
+    def forward(self, x, topk=1):
+        logits = self.linear_layer(x)
+        probs = self.softmax(logits)
+        # 使用topk来选择最高的k个概率
+        topk_values, topk_indices = torch.topk(probs, k=topk, dim=-1)
+        # 创建一个初始值全为负无穷的张量，形状与probs相同
+        topk_probs = torch.full_like(probs, float('-inf'))
+        # 使用scatter填充topk的概率值
+        topk_probs = topk_probs.scatter_(-1, topk_indices, topk_values)
+        # 应用softmax确保top k值的和为1
+        topk_probs = self.softmax(topk_probs)
+        return topk_probs
+
+class MLP(nn.Module):
+    def __init__(self, dim: int, num_moe: int, hidden_dim: int = 128) -> None:
+        super().__init__()
+        self.dim = 64
+        self.num_moe = num_moe
+        # 添加多层感知机结构
+        self.linear_layer1 = nn.Linear(self.dim, hidden_dim)
+        self.activation = nn.GELU()  # 使用GELU激活函数
+        self.linear_layer2 = nn.Linear(hidden_dim, self.num_moe)
+        self.softmax = nn.Softmax(dim=-1)
+
+    def forward(self, x):
+        x = self.linear_layer1(x)
+        x = self.activation(x)
+        logits = self.linear_layer2(x)
+        probs = self.softmax(logits)
+        return probs
+    
+    
+class Noise(nn.Module):
+    def __init__(self, dim: int, num_moe: int, noise_std: float = 0.1) -> None:
+        super().__init__()
+        self.dim = 64
+        self.num_moe = num_moe
+        self.noise_std = noise_std
+        self.linear_layer = nn.Linear(self.dim, num_moe, bias=False)
+        self.softmax = nn.Softmax(dim=-1)
+
+    def forward(self, x):
+        logits = self.linear_layer(x)
+        
+        # 添加噪声
+        noise = torch.randn_like(logits) * self.noise_std
+        logits = logits + noise
+        
+        probs = self.softmax(logits)
+        return probs
+    
+class MLP_noise(nn.Module):
+    def __init__(self, dim: int, num_moe: int, hidden_dim: int = 128, noise_std: float = 0.1) -> None:
+        super().__init__()
+        self.dim = 64
+        self.num_moe = num_moe
+        self.noise_std = noise_std
+        self.linear1 = nn.Linear(self.dim, hidden_dim, bias=False)
+        self.relu = nn.ReLU()
+        self.linear2 = nn.Linear(hidden_dim, num_moe, bias=False)
+        self.softmax = nn.Softmax(dim=-1)
+
+    def forward(self, x):
+        hidden = self.linear1(x)
+        hidden = self.relu(hidden)
+        logits = self.linear2(hidden)
+        
+        # 添加噪声
+        noise = torch.randn_like(logits) * self.noise_std
+        logits = logits + noise
+        
+        probs = self.softmax(logits)
+        return probs
+
+    
+class Drop(nn.Module):
+    def __init__(self, dim: int, num_moe: int, dropout_rate: float = 0.1) -> None:
+        super().__init__()
+        self.dim = 64
+        self.num_moe = num_moe
+        self.linear_layer = nn.Linear(self.dim, num_moe, bias=False)
+        self.dropout = nn.Dropout(dropout_rate)
+        self.softmax = nn.Softmax(dim=-1)
+
+    def forward(self, x):
+        logits = self.linear_layer(x)
+        # 添加Dropout
+        logits = self.dropout(logits)
+        probs = self.softmax(logits)
+        return probs
+    
+GATING_TO_MODEL_MAPPING = {
+    "Dense": Dense,
+    "topK": topK,
+    "MLP": MLP,
+    "Drop": Drop,
+    "MLP_noise": MLP_noise,
+    "Noise": Noise,
+}

model/peft/tuners/lora.py

@@ -0,0 +1,1033 @@
+# coding=utf-8
+# Copyright 2023-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import math
+import re
+import warnings
+from dataclasses import asdict, dataclass, field
+from enum import Enum
+from typing import List, Optional, Tuple, Union
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from transformers.pytorch_utils import Conv1D
+
+from ..import_utils import is_bnb_4bit_available, is_bnb_available
+from ..utils import (
+    COMMON_LAYERS_PATTERN,
+    TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPING,
+    ModulesToSaveWrapper,
+    PeftConfig,
+    PeftType,
+    _freeze_adapter,
+    _get_submodules,
+    transpose,
+)
+
+
+if is_bnb_available():
+    import bitsandbytes as bnb
+
+
+@dataclass
+class LoraConfig(PeftConfig):
+    """
+    This is the configuration class to store the configuration of a [`LoraModel`].
+
+    Args:
+        r (`int`): Lora attention dimension.
+        target_modules (`Union[List[str],str]`): The names of the modules to apply Lora to.
+        lora_alpha (`int`): The alpha parameter for Lora scaling.
+        lora_dropout (`float`): The dropout probability for Lora layers.
+        fan_in_fan_out (`bool`): Set this to True if the layer to replace stores weight like (fan_in, fan_out).
+        For example, gpt-2 uses `Conv1D` which stores weights like (fan_in, fan_out) and hence this should be set to `True`.:
+        bias (`str`): Bias type for Lora. Can be 'none', 'all' or 'lora_only'
+        modules_to_save (`List[str]`):List of modules apart from LoRA layers to be set as trainable
+            and saved in the final checkpoint.
+        layers_to_transform (`Union[List[int],int]`):
+            The layer indexes to transform, if this argument is specified, it will apply the LoRA transformations on
+            the layer indexes that are specified in this list. If a single integer is passed, it will apply the LoRA
+            transformations on the layer at this index.
+        layers_pattern (`str`):
+            The layer pattern name, used only if `layers_to_transform` is different from `None` and if the layer
+            pattern is not in the common layers pattern.
+    """
+
+    r: int = field(default=8, metadata={"help": "Lora attention dimension"})
+    target_modules: Optional[Union[List[str], str]] = field(
+        default=None,
+        metadata={
+            "help": "List of module names or regex expression of the module names to replace with Lora."
+            "For example, ['q', 'v'] or '.*decoder.*(SelfAttention|EncDecAttention).*(q|v)$' "
+        },
+    )
+    lora_alpha: int = field(default=8, metadata={"help": "Lora alpha"})
+    lora_dropout: float = field(default=0.0, metadata={"help": "Lora dropout"})
+    fan_in_fan_out: bool = field(
+        default=False,
+        metadata={"help": "Set this to True if the layer to replace stores weight like (fan_in, fan_out)"},
+    )
+    bias: str = field(default="none", metadata={"help": "Bias type for Lora. Can be 'none', 'all' or 'lora_only'"})
+    modules_to_save: Optional[List[str]] = field(
+        default=None,
+        metadata={
+            "help": "List of modules apart from LoRA layers to be set as trainable and saved in the final checkpoint. "
+            "For example, in Sequence Classification or Token Classification tasks, "
+            "the final layer `classifier/score` are randomly initialized and as such need to be trainable and saved."
+        },
+    )
+    init_lora_weights: bool = field(
+        default=True,
+        metadata={"help": "Whether to initialize the weights of the Lora layers."},
+    )
+    layers_to_transform: Optional[Union[List, int]] = field(
+        default=None,
+        metadata={
+            "help": "The layer indexes to transform, is this argument is specified, PEFT will transform only the layers indexes that are specified inside this list. If a single integer is passed, PEFT will transform only the layer at this index."
+        },
+    )
+    layers_pattern: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": "The layer pattern name, used only if `layers_to_transform` is different to None and if the layer pattern is not in the common layers pattern."
+        },
+    )
+
+    def __post_init__(self):
+        self.peft_type = PeftType.LORA
+
+
+class LoraModel(torch.nn.Module):
+    """
+    Creates Low Rank Adapter (Lora) model from a pretrained transformers model.
+
+    Args:
+        model ([`~transformers.PreTrainedModel`]): The model to be adapted.
+        config ([`LoraConfig`]): The configuration of the Lora model.
+
+    Returns:
+        `torch.nn.Module`: The Lora model.
+
+    Example:
+
+        ```py
+        >>> from transformers import AutoModelForSeq2SeqLM
+        >>> from peft import LoraModel, LoraConfig
+
+        >>> config = LoraConfig(
+        ...     peft_type="LORA",
+        ...     task_type="SEQ_2_SEQ_LM",
+        ...     r=8,
+        ...     lora_alpha=32,
+        ...     target_modules=["q", "v"],
+        ...     lora_dropout=0.01,
+        ... )
+
+        >>> model = AutoModelForSeq2SeqLM.from_pretrained("t5-base")
+        >>> lora_model = LoraModel(config, model)
+        ```
+
+        ```py
+        >>> import transformers
+        >>> from peft import LoraConfig, PeftModel, get_peft_model, prepare_model_for_int8_training
+
+        >>> target_modules = ["q_proj", "k_proj", "v_proj", "out_proj", "fc_in", "fc_out", "wte"]
+        >>> config = LoraConfig(
+        ...     r=4, lora_alpha=16, target_modules=target_modules, lora_dropout=0.1, bias="none", task_type="CAUSAL_LM"
+        ... )
+
+        >>> model = transformers.GPTJForCausalLM.from_pretrained(
+        ...     "kakaobrain/kogpt",
+        ...     revision="KoGPT6B-ryan1.5b-float16",  # or float32 version: revision=KoGPT6B-ryan1.5b
+        ...     pad_token_id=tokenizer.eos_token_id,
+        ...     use_cache=False,
+        ...     device_map={"": rank},
+        ...     torch_dtype=torch.float16,
+        ...     load_in_8bit=True,
+        ... )
+        >>> model = prepare_model_for_int8_training(model)
+        >>> lora_model = get_peft_model(model, config)
+        ```
+
+    **Attributes**:
+        - **model** ([`~transformers.PreTrainedModel`]) -- The model to be adapted.
+        - **peft_config** ([`LoraConfig`]): The configuration of the Lora model.
+    """
+
+    def __init__(self, model, config, adapter_name):
+        super().__init__()
+        self.model = model
+        self.forward = self.model.forward
+        self.peft_config = config
+        self.add_adapter(adapter_name, self.peft_config[adapter_name])
+
+    def add_adapter(self, adapter_name, config=None):
+        if config is not None:
+            model_config = self.model.config.to_dict() if hasattr(self.model.config, "to_dict") else self.model.config
+            config = self._prepare_lora_config(config, model_config)
+            self.peft_config[adapter_name] = config
+        self._find_and_replace(adapter_name)
+        if len(self.peft_config) > 1 and self.peft_config[adapter_name].bias != "none":
+            raise ValueError(
+                "LoraModel supports only 1 adapter with bias. When using multiple adapters, set bias to 'none' for all adapters."
+            )
+        mark_only_lora_as_trainable(self.model, self.peft_config[adapter_name].bias)
+        if self.peft_config[adapter_name].inference_mode:
+            _freeze_adapter(self.model, adapter_name)
+
+    def _check_quantization_dependency(self):
+        loaded_in_4bit = getattr(self.model, "is_loaded_in_4bit", False)
+        loaded_in_8bit = getattr(self.model, "is_loaded_in_8bit", False)
+        if (loaded_in_4bit or loaded_in_8bit) and not is_bnb_available():
+            raise ImportError(
+                "To use Lora with 8-bit or 4-bit quantization, please install the `bitsandbytes` package. "
+                "You can install it with `pip install bitsandbytes`."
+            )
+
+    def _check_target_module_exists(self, lora_config, key):
+        if isinstance(lora_config.target_modules, str):
+            target_module_found = re.fullmatch(lora_config.target_modules, key)
+        else:
+            target_module_found = any(key.endswith(target_key) for target_key in lora_config.target_modules)
+            is_using_layer_indexes = getattr(lora_config, "layers_to_transform", None) is not None
+            layer_indexing_pattern = getattr(lora_config, "layers_pattern", None)
+
+            if is_using_layer_indexes and target_module_found:
+                layers_pattern = COMMON_LAYERS_PATTERN if layer_indexing_pattern is None else layer_indexing_pattern
+                layers_pattern = [layers_pattern] if isinstance(layers_pattern, str) else layers_pattern
+
+                for pattern in layers_pattern:
+                    layer_index = re.match(f".*.{pattern}\.(\d+)\.*", key)
+                    if layer_index is not None:
+                        layer_index = int(layer_index.group(1))
+                        if isinstance(lora_config.layers_to_transform, int):
+                            target_module_found = layer_index == lora_config.layers_to_transform
+                        else:
+                            target_module_found = layer_index in lora_config.layers_to_transform
+
+                        break
+                    else:
+                        target_module_found = False
+        return target_module_found
+
+    def _create_new_module(self, lora_config, adapter_name, target):
+        bias = hasattr(target, "bias") and target.bias is not None
+        kwargs = {
+            "r": lora_config.r,
+            "lora_alpha": lora_config.lora_alpha,
+            "lora_dropout": lora_config.lora_dropout,
+            "fan_in_fan_out": lora_config.fan_in_fan_out,
+            "init_lora_weights": lora_config.init_lora_weights,
+        }
+        loaded_in_4bit = getattr(self.model, "is_loaded_in_4bit", False)
+        loaded_in_8bit = getattr(self.model, "is_loaded_in_8bit", False)
+
+        if loaded_in_8bit and isinstance(target, bnb.nn.Linear8bitLt):
+            eightbit_kwargs = kwargs.copy()
+            eightbit_kwargs.update(
+                {
+                    "has_fp16_weights": target.state.has_fp16_weights,
+                    "memory_efficient_backward": target.state.memory_efficient_backward,
+                    "threshold": target.state.threshold,
+                    "index": target.index,
+                }
+            )
+            new_module = Linear8bitLt(
+                adapter_name, target.in_features, target.out_features, bias=bias, **eightbit_kwargs
+            )
+        elif loaded_in_4bit and is_bnb_4bit_available() and isinstance(target, bnb.nn.Linear4bit):
+            fourbit_kwargs = kwargs.copy()
+            fourbit_kwargs.update(
+                {
+                    "compute_dtype": target.compute_dtype,
+                    "compress_statistics": target.weight.compress_statistics,
+                    "quant_type": target.weight.quant_type,
+                }
+            )
+            new_module = Linear4bit(adapter_name, target.in_features, target.out_features, bias=bias, **fourbit_kwargs)
+        elif isinstance(target, torch.nn.Embedding):
+            embedding_kwargs = kwargs.copy()
+            embedding_kwargs.pop("fan_in_fan_out", None)
+            in_features, out_features = target.num_embeddings, target.embedding_dim
+            new_module = Embedding(adapter_name, in_features, out_features, **embedding_kwargs)
+        elif isinstance(target, torch.nn.Conv2d):
+            out_channels, in_channels = target.weight.size()[:2]
+            kernel_size = target.weight.size()[2:]
+            stride = target.stride
+            padding = target.padding
+            new_module = Conv2d(adapter_name, in_channels, out_channels, kernel_size, stride, padding, **kwargs)
+        else:
+            if isinstance(target, torch.nn.Linear):
+                in_features, out_features = target.in_features, target.out_features
+                if kwargs["fan_in_fan_out"]:
+                    warnings.warn(
+                        "fan_in_fan_out is set to True but the target module is `torch.nn.Linear`. "
+                        "Setting fan_in_fan_out to False."
+                    )
+                    kwargs["fan_in_fan_out"] = lora_config.fan_in_fan_out = False
+            elif isinstance(target, Conv1D):
+                in_features, out_features = (
+                    target.weight.ds_shape if hasattr(target.weight, "ds_shape") else target.weight.shape
+                )
+                if not kwargs["fan_in_fan_out"]:
+                    warnings.warn(
+                        "fan_in_fan_out is set to False but the target module is `Conv1D`. "
+                        "Setting fan_in_fan_out to True."
+                    )
+                    kwargs["fan_in_fan_out"] = lora_config.fan_in_fan_out = True
+            else:
+                raise ValueError(
+                    f"Target module {target} is not supported. "
+                    f"Currently, only `torch.nn.Linear` and `Conv1D` are supported."
+                )
+            new_module = Linear(adapter_name, in_features, out_features, bias=bias, **kwargs)
+
+        return new_module
+
+    def _find_and_replace(self, adapter_name):
+        lora_config = self.peft_config[adapter_name]
+        self._check_quantization_dependency()
+        is_target_modules_in_base_model = False
+        key_list = [key for key, _ in self.model.named_modules()]
+
+        for key in key_list:
+            if not self._check_target_module_exists(lora_config, key):
+                continue
+
+            is_target_modules_in_base_model = True
+            parent, target, target_name = _get_submodules(self.model, key)
+
+            if isinstance(target, LoraLayer) and isinstance(target, torch.nn.Conv2d):
+                target.update_layer_conv2d(
+                    adapter_name,
+                    lora_config.r,
+                    lora_config.lora_alpha,
+                    lora_config.lora_dropout,
+                    lora_config.init_lora_weights,
+                )
+            elif isinstance(target, LoraLayer):
+                target.update_layer(
+                    adapter_name,
+                    lora_config.r,
+                    lora_config.lora_alpha,
+                    lora_config.lora_dropout,
+                    lora_config.init_lora_weights,
+                )
+            else:
+                new_module = self._create_new_module(lora_config, adapter_name, target)
+                self._replace_module(parent, target_name, new_module, target)
+
+        if not is_target_modules_in_base_model:
+            raise ValueError(
+                f"Target modules {lora_config.target_modules} not found in the base model. "
+                f"Please check the target modules and try again."
+            )
+
+    def _replace_module(self, parent_module, child_name, new_module, old_module):
+        setattr(parent_module, child_name, new_module)
+        new_module.weight = old_module.weight
+        if hasattr(old_module, "bias"):
+            if old_module.bias is not None:
+                new_module.bias = old_module.bias
+
+        if getattr(old_module, "state", None) is not None:
+            new_module.state = old_module.state
+            new_module.to(old_module.weight.device)
+
+        # dispatch to correct device
+        for name, module in new_module.named_modules():
+            if "lora_" in name:
+                module.to(old_module.weight.device)
+            if "ranknum" in name:
+                module.to(old_module.weight.device)
+
+    def __getattr__(self, name: str):
+        """Forward missing attributes to the wrapped module."""
+        try:
+            return super().__getattr__(name)  # defer to nn.Module's logic
+        except AttributeError:
+            return getattr(self.model, name)
+
+    def get_peft_config_as_dict(self, inference: bool = False):
+        config_dict = {}
+        for key, value in self.peft_config.items():
+            config = {k: v.value if isinstance(v, Enum) else v for k, v in asdict(value).items()}
+            if inference:
+                config["inference_mode"] = True
+        config_dict[key] = config
+        return config
+
+    def _set_adapter_layers(self, enabled=True):
+        for module in self.model.modules():
+            if isinstance(module, LoraLayer):
+                module.disable_adapters = False if enabled else True
+
+    def enable_adapter_layers(self):
+        self._set_adapter_layers(enabled=True)
+
+    def disable_adapter_layers(self):
+        self._set_adapter_layers(enabled=False)
+
+    def set_adapter(self, adapter_name):
+        for module in self.model.modules():
+            if isinstance(module, LoraLayer):
+                if module.merged:
+                    warnings.warn("Adapter cannot be set when the model is merged. Unmerging the model first.")
+                    module.unmerge()
+                module.active_adapter = adapter_name
+
+    def merge_adapter(self):
+        for module in self.model.modules():
+            if isinstance(module, LoraLayer):
+                module.merge()
+
+    def unmerge_adapter(self):
+        for module in self.model.modules():
+            if isinstance(module, LoraLayer):
+                module.unmerge()
+
+    @staticmethod
+    def _prepare_lora_config(peft_config, model_config):
+        if peft_config.target_modules is None:
+            if model_config["model_type"] not in TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPING:
+                raise ValueError("Please specify `target_modules` in `peft_config`")
+            peft_config.target_modules = TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPING[model_config["model_type"]]
+        return peft_config
+
+    def merge_and_unload(self):
+        r"""
+        This method merges the LoRa layers into the base model. This is needed if someone wants to use the base model
+        as a standalone model.
+        """
+        if getattr(self.config, "model_type", None) == "gpt2":
+            raise ValueError("GPT2 models are not supported for merging LORA layers")
+
+        if getattr(self.model, "is_loaded_in_8bit", False) or getattr(self.model, "is_loaded_in_4bit", False):
+            raise ValueError("Cannot merge LORA layers when the model is loaded in 8-bit mode")
+
+        key_list = [key for key, _ in self.model.named_modules() if "lora" not in key]
+        for key in key_list:
+            try:
+                parent, target, target_name = _get_submodules(self.model, key)
+            except AttributeError:
+                continue
+            if isinstance(target, LoraLayer):
+                if isinstance(target, nn.Embedding):
+                    new_module = torch.nn.Embedding(target.in_features, target.out_features)
+                elif isinstance(target, nn.Conv2d):
+                    new_module = torch.nn.Conv2d(
+                        target.in_channels,
+                        target.out_channels,
+                        kernel_size=target.kernel_size,
+                        stride=target.stride,
+                        padding=target.padding,
+                        dilation=target.dilation,
+                    )
+                else:
+                    bias = target.bias is not None
+                    new_module = torch.nn.Linear(target.in_features, target.out_features, bias=bias)
+                target.merge()
+                self._replace_module(parent, target_name, new_module, target)
+
+            # save any additional trainable modules part of `modules_to_save`
+            if isinstance(target, ModulesToSaveWrapper):
+                setattr(parent, target_name, target.modules_to_save[target.active_adapter])
+
+        return self.model
+
+    def add_weighted_adapter(self, adapters, weights, adapter_name):
+        if len({self.peft_config[adapter].r for adapter in adapters}) != 1:
+            raise ValueError("All adapters must have the same r value")
+        self.peft_config[adapter_name] = self.peft_config[adapters[0]]
+        self.peft_config[adapter_name].lora_alpha = self.peft_config[adapters[0]].r
+        self._find_and_replace(adapter_name)
+        mark_only_lora_as_trainable(self.model, self.peft_config[adapter_name].bias)
+        _freeze_adapter(self.model, adapter_name)
+        key_list = [key for key, _ in self.model.named_modules() if "lora" not in key]
+        for key in key_list:
+            _, target, _ = _get_submodules(self.model, key)
+            if isinstance(target, LoraLayer):
+                if adapter_name in target.lora_A:
+                    target.lora_A[adapter_name].weight.data = target.lora_A[adapter_name].weight.data * 0.0
+                    target.lora_B[adapter_name].weight.data = target.lora_B[adapter_name].weight.data * 0.0
+                    for adapter, weight in zip(adapters, weights):
+                        if adapter not in target.lora_A:
+                            continue
+                        target.lora_A[adapter_name].weight.data += (
+                            target.lora_A[adapter].weight.data * weight * target.scaling[adapter]
+                        )
+                        target.lora_B[adapter_name].weight.data += target.lora_B[adapter].weight.data * weight
+
+                elif adapter_name in target.lora_embedding_A:
+                    target.lora_embedding_A[adapter_name].data = target.lora_embedding_A[adapter_name].data * 0.0
+                    target.lora_embedding_B[adapter_name].data = target.lora_embedding_B[adapter_name].data * 0.0
+                    for adapter, weight in zip(adapters, weights):
+                        if adapter not in target.lora_embedding_A:
+                            continue
+                        target.lora_embedding_A[adapter_name].data += (
+                            target.lora_embedding_A[adapter].data * weight * target.scaling[adapter]
+                        )
+                        target.lora_embedding_B[adapter_name].data += target.lora_embedding_B[adapter].data * weight
+
+
+# Below code is based on https://github.com/microsoft/LoRA/blob/main/loralib/layers.py
+# and modified to work with PyTorch FSDP
+
+
+#  ------------------------------------------------------------------------------------------
+#  Copyright (c) Microsoft Corporation. All rights reserved.
+#  Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+#  ------------------------------------------------------------------------------------------
+
+
+# had to adapt it for `lora_only` to work
+def mark_only_lora_as_trainable(model: nn.Module, bias: str = "none") -> None:
+    for n, p in model.named_parameters():
+        if "lora_" not in n:
+            p.requires_grad = False
+    if bias == "none":
+        return
+    elif bias == "all":
+        for n, p in model.named_parameters():
+            if "bias" in n:
+                p.requires_grad = True
+    elif bias == "lora_only":
+        for m in model.modules():
+            if isinstance(m, LoraLayer) and hasattr(m, "bias") and m.bias is not None:
+                m.bias.requires_grad = True
+    else:
+        raise NotImplementedError
+
+
+class LoraLayer:
+    def __init__(self, in_features: int, out_features: int, **kwargs):
+        self.r = {}
+        self.lora_alpha = {}
+        self.scaling = {}
+        self.lora_dropout = nn.ModuleDict({})
+        self.lora_A = nn.ModuleDict({})
+        self.lora_B = nn.ModuleDict({})
+        # For Embedding layer
+        self.lora_embedding_A = nn.ParameterDict({})
+        self.lora_embedding_B = nn.ParameterDict({})
+        # Mark the weight as unmerged
+        self.merged = False
+        self.disable_adapters = False
+        self.in_features = in_features
+        self.out_features = out_features
+        self.kwargs = kwargs
+
+    def update_layer(self, adapter_name, r, lora_alpha, lora_dropout, init_lora_weights):
+        self.r[adapter_name] = r
+        self.lora_alpha[adapter_name] = lora_alpha
+        if lora_dropout > 0.0:
+            lora_dropout_layer = nn.Dropout(p=lora_dropout)
+        else:
+            lora_dropout_layer = nn.Identity()
+
+        self.lora_dropout.update(nn.ModuleDict({adapter_name: lora_dropout_layer}))
+        # Actual trainable parameters
+        if r > 0:
+            self.lora_A.update(nn.ModuleDict({adapter_name: nn.Linear(self.in_features, r, bias=False)}))
+            self.lora_B.update(nn.ModuleDict({adapter_name: nn.Linear(r, self.out_features, bias=False)}))
+            self.scaling[adapter_name] = lora_alpha / r
+        if init_lora_weights:
+            self.reset_lora_parameters(adapter_name)
+        self.to(self.weight.device)
+
+    def update_layer_conv2d(self, adapter_name, r, lora_alpha, lora_dropout, init_lora_weights):
+        self.r[adapter_name] = r
+        self.lora_alpha[adapter_name] = lora_alpha
+        if lora_dropout > 0.0:
+            lora_dropout_layer = nn.Dropout(p=lora_dropout)
+        else:
+            lora_dropout_layer = nn.Identity()
+
+        self.lora_dropout.update(nn.ModuleDict({adapter_name: lora_dropout_layer}))
+        # Actual trainable parameters
+        if r > 0:
+            kernel_size = self.kwargs["kernel_size"]
+            stride = self.kwargs["stride"]
+            padding = self.kwargs["padding"]
+            self.lora_A.update(
+                nn.ModuleDict({adapter_name: nn.Conv2d(self.in_features, r, kernel_size, stride, padding, bias=False)})
+            )
+            self.lora_B.update(
+                nn.ModuleDict({adapter_name: nn.Conv2d(r, self.out_features, (1, 1), (1, 1), bias=False)})
+            )
+            self.scaling[adapter_name] = lora_alpha / r
+        if init_lora_weights:
+            self.reset_lora_parameters(adapter_name)
+        self.to(self.weight.device)
+
+    def update_layer_embedding(self, adapter_name, r, lora_alpha, lora_dropout, init_lora_weights):
+        self.r[adapter_name] = r
+        self.lora_alpha[adapter_name] = lora_alpha
+        if lora_dropout > 0.0:
+            lora_dropout_layer = nn.Dropout(p=lora_dropout)
+        else:
+            lora_dropout_layer = nn.Identity()
+
+        self.lora_dropout.update(nn.ModuleDict({adapter_name: lora_dropout_layer}))
+        # Actual trainable parameters
+        if r > 0:
+            self.lora_embedding_A.update(
+                nn.ParameterDict({adapter_name: nn.Parameter(self.weight.new_zeros((r, self.in_features)))})
+            )
+            self.lora_embedding_B.update(
+                nn.ParameterDict({adapter_name: nn.Parameter(self.weight.new_zeros((self.out_features, r)))})
+            )
+            self.scaling[adapter_name] = lora_alpha / r
+        if init_lora_weights:
+            self.reset_lora_parameters(adapter_name)
+        self.to(self.weight.device)
+
+    def reset_lora_parameters(self, adapter_name):
+        if adapter_name in self.lora_A.keys():
+            # initialize A the same way as the default for nn.Linear and B to zero
+            nn.init.kaiming_uniform_(self.lora_A[adapter_name].weight, a=math.sqrt(5))
+            nn.init.zeros_(self.lora_B[adapter_name].weight)
+        if adapter_name in self.lora_embedding_A.keys():
+            # initialize a the same way as the default for nn.linear and b to zero
+            nn.init.zeros_(self.lora_embedding_A[adapter_name])
+            nn.init.normal_(self.lora_embedding_B[adapter_name])
+
+
+class Linear(nn.Linear, LoraLayer):
+    # Lora implemented in a dense layer
+    def __init__(
+        self,
+        adapter_name: str,
+        in_features: int,
+        out_features: int,
+        r: int = 0,
+        lora_alpha: int = 1,
+        lora_dropout: float = 0.0,
+        fan_in_fan_out: bool = False,  # Set this to True if the layer to replace stores weight like (fan_in, fan_out)
+        **kwargs,
+    ):
+        init_lora_weights = kwargs.pop("init_lora_weights", True)
+
+        nn.Linear.__init__(self, in_features, out_features, **kwargs)
+        LoraLayer.__init__(self, in_features=in_features, out_features=out_features)
+        # Freezing the pre-trained weight matrix
+        self.weight.requires_grad = False
+
+        self.fan_in_fan_out = fan_in_fan_out
+        if fan_in_fan_out:
+            self.weight.data = self.weight.data.T
+
+        nn.Linear.reset_parameters(self)
+        self.update_layer(adapter_name, r, lora_alpha, lora_dropout, init_lora_weights)
+        self.active_adapter = adapter_name
+
+    def merge(self):
+        if self.active_adapter not in self.lora_A.keys():
+            return
+        if self.merged:
+            warnings.warn("Already merged. Nothing to do.")
+            return
+        if self.r[self.active_adapter] > 0:
+            self.weight.data += (
+                transpose(
+                    self.lora_B[self.active_adapter].weight @ self.lora_A[self.active_adapter].weight,
+                    self.fan_in_fan_out,
+                )
+                * self.scaling[self.active_adapter]
+            )
+            self.merged = True
+
+    def unmerge(self):
+        if self.active_adapter not in self.lora_A.keys():
+            return
+        if not self.merged:
+            warnings.warn("Already unmerged. Nothing to do.")
+            return
+        if self.r[self.active_adapter] > 0:
+            self.weight.data -= (
+                transpose(
+                    self.lora_B[self.active_adapter].weight @ self.lora_A[self.active_adapter].weight,
+                    self.fan_in_fan_out,
+                )
+                * self.scaling[self.active_adapter]
+            )
+            self.merged = False
+
+    def forward(self, x: torch.Tensor):
+        previous_dtype = x.dtype
+        if self.active_adapter not in self.lora_A.keys():
+            return F.linear(x, transpose(self.weight, self.fan_in_fan_out), bias=self.bias)
+        if self.disable_adapters:
+            if self.r[self.active_adapter] > 0 and self.merged:
+                self.unmerge()
+            result = F.linear(x, transpose(self.weight, self.fan_in_fan_out), bias=self.bias)
+        elif self.r[self.active_adapter] > 0 and not self.merged:
+            result = F.linear(x, transpose(self.weight, self.fan_in_fan_out), bias=self.bias)
+
+            x = x.to(self.lora_A[self.active_adapter].weight.dtype)
+
+            result += (
+                self.lora_B[self.active_adapter](
+                    self.lora_A[self.active_adapter](self.lora_dropout[self.active_adapter](x))
+                )
+                * self.scaling[self.active_adapter]
+            )
+        else:
+            result = F.linear(x, transpose(self.weight, self.fan_in_fan_out), bias=self.bias)
+
+        result = result.to(previous_dtype)
+
+        return result
+
+
+class Embedding(nn.Embedding, LoraLayer):
+    # LoRA implemented in a Embedding layer
+    def __init__(
+        self,
+        adapter_name: str,
+        num_embeddings: int,
+        embedding_dim: int,
+        r: int = 0,
+        lora_alpha: int = 1,
+        lora_dropout: float = 0.0,
+        **kwargs,
+    ):
+        init_lora_weights = kwargs.pop("init_lora_weights", True)
+
+        nn.Embedding.__init__(self, num_embeddings, embedding_dim, **kwargs)
+        LoraLayer.__init__(self, in_features=num_embeddings, out_features=embedding_dim)
+
+        self.weight.requires_grad = False
+
+        nn.Embedding.reset_parameters(self)
+        self.update_layer_embedding(adapter_name, r, lora_alpha, lora_dropout, init_lora_weights)
+        self.active_adapter = adapter_name
+
+    def unmerge(self, mode: bool = True):
+        if not self.merged:
+            warnings.warn("Already unmerged. Nothing to do.")
+            return
+        if self.r[self.active_adapter] > 0:
+            self.weight.data -= (
+                transpose(
+                    self.lora_embedding_B[self.active_adapter] @ self.lora_embedding_A[self.active_adapter], True
+                )
+                * self.scaling[self.active_adapter]
+            )
+            self.merged = False
+
+    def merge(self):
+        if self.merged:
+            warnings.warn("Already merged. Nothing to do.")
+            return
+        if self.r[self.active_adapter] > 0:
+            self.weight.data += (
+                transpose(
+                    self.lora_embedding_B[self.active_adapter] @ self.lora_embedding_A[self.active_adapter], True
+                )
+                * self.scaling[self.active_adapter]
+            )
+            self.merged = True
+
+    def forward(self, x: torch.Tensor):
+        if self.disable_adapters:
+            if self.r[self.active.adapter] > 0 and self.merged:
+                self.weight.data -= (
+                    transpose(
+                        self.lora_embedding_B[self.active_adapter].weight
+                        @ self.lora_embedding_A[self.active_adapter].weight,
+                        True,
+                    )
+                    * self.scaling[self.active_adapter]
+                )
+                self.merged = False
+            return nn.Embedding.forward(self, x)
+
+        elif self.r[self.active_adapter] > 0 and not self.merged:
+            result = nn.Embedding.forward(self, x)
+            if self.r[self.active_adapter] > 0:
+                after_A = F.embedding(
+                    x,
+                    self.lora_embedding_A[self.active_adapter].T,
+                    self.padding_idx,
+                    self.max_norm,
+                    self.norm_type,
+                    self.scale_grad_by_freq,
+                    self.sparse,
+                )
+                result += (after_A @ self.lora_embedding_B[self.active_adapter].T) * self.scaling[self.active_adapter]
+            return result
+        else:
+            return nn.Embedding.forward(self, x)
+
+
+class Conv2d(nn.Conv2d, LoraLayer):
+    # Lora implemented in a conv2d layer
+    def __init__(
+        self,
+        adapter_name: str,
+        in_channels: int,
+        out_channels: int,
+        kernel_size: Union[int, Tuple[int]],
+        stride: Union[int, Tuple[int]] = 1,
+        padding: Union[int, Tuple[int]] = 0,
+        r: int = 0,
+        lora_alpha: int = 1,
+        lora_dropout: float = 0.0,
+        **kwargs,
+    ):
+        init_lora_weights = kwargs.pop("init_lora_weights", True)
+
+        nn.Conv2d.__init__(self, in_channels, out_channels, kernel_size, stride, padding)
+        LoraLayer.__init__(
+            self,
+            in_features=in_channels,
+            out_features=out_channels,
+            kernel_size=kernel_size,
+            stride=stride,
+            padding=padding,
+        )
+        # Freezing the pre-trained weight matrix
+        self.weight.requires_grad = False
+
+        nn.Conv2d.reset_parameters(self)
+        self.update_layer_conv2d(adapter_name, r, lora_alpha, lora_dropout, init_lora_weights)
+        self.active_adapter = adapter_name
+
+    def merge(self):
+        if self.active_adapter not in self.lora_A.keys():
+            return
+        if self.merged:
+            warnings.warn("Already merged. Nothing to do.")
+            return
+        if self.r[self.active_adapter] > 0:
+            # https://github.com/bmaltais/kohya_ss/blob/feb6728762a8f463d15ba936d189d4c3abfaa1ab/networks/lora.py#L117
+            if self.weight.size()[2:4] == (1, 1):
+                # conv2d 1x1
+                self.weight.data += (
+                    self.lora_B[self.active_adapter].weight.squeeze(3).squeeze(2)
+                    @ self.lora_A[self.active_adapter].weight.squeeze(3).squeeze(2)
+                ).unsqueeze(2).unsqueeze(3) * self.scaling[self.active_adapter]
+            else:
+                # conv2d 3x3
+                self.weight.data += (
+                    F.conv2d(
+                        self.lora_A[self.active_adapter].weight.permute(1, 0, 2, 3),
+                        self.lora_B[self.active_adapter].weight,
+                    ).permute(1, 0, 2, 3)
+                    * self.scaling[self.active_adapter]
+                )
+            self.merged = True
+
+    def unmerge(self):
+        if self.active_adapter not in self.lora_A.keys():
+            return
+        if not self.merged:
+            warnings.warn("Already unmerged. Nothing to do.")
+            return
+        if self.r[self.active_adapter] > 0:
+            if self.weight.size()[2:4] == (1, 1):
+                # conv2d 1x1
+                self.weight.data -= (
+                    self.lora_B[self.active_adapter].weight.squeeze(3).squeeze(2)
+                    @ self.lora_A[self.active_adapter].weight.squeeze(3).squeeze(2)
+                ).unsqueeze(2).unsqueeze(3) * self.scaling[self.active_adapter]
+            else:
+                # conv2d 3x3
+                self.weight.data += (
+                    F.conv2d(
+                        self.lora_A[self.active_adapter].weight.permute(1, 0, 2, 3),
+                        self.lora_B[self.active_adapter].weight,
+                    ).permute(1, 0, 2, 3)
+                    * self.scaling[self.active_adapter]
+                )
+            self.merged = False
+
+    def forward(self, x: torch.Tensor):
+        previous_dtype = x.dtype
+
+        if self.active_adapter not in self.lora_A.keys():
+            return F.conv2d(
+                x,
+                self.weight,
+                bias=self.bias,
+                stride=self.stride,
+                padding=self.padding,
+                dilation=self.dilation,
+                groups=self.groups,
+            )
+        if self.disable_adapters:
+            if self.r[self.active_adapter] > 0 and self.merged:
+                self.unmerge()
+            result = F.conv2d(
+                x,
+                self.weight,
+                bias=self.bias,
+                stride=self.stride,
+                padding=self.padding,
+                dilation=self.dilation,
+                groups=self.groups,
+            )
+        elif self.r[self.active_adapter] > 0 and not self.merged:
+            result = F.conv2d(
+                x,
+                self.weight,
+                bias=self.bias,
+                stride=self.stride,
+                padding=self.padding,
+                dilation=self.dilation,
+                groups=self.groups,
+            )
+
+            x = x.to(self.lora_A[self.active_adapter].weight.dtype)
+
+            result += (
+                self.lora_B[self.active_adapter](
+                    self.lora_A[self.active_adapter](self.lora_dropout[self.active_adapter](x))
+                )
+                * self.scaling[self.active_adapter]
+            )
+        else:
+            result = F.conv2d(
+                x,
+                self.weight,
+                bias=self.bias,
+                stride=self.stride,
+                padding=self.padding,
+                dilation=self.dilation,
+                groups=self.groups,
+            )
+
+        result = result.to(previous_dtype)
+
+        return result
+
+
+if is_bnb_available():
+
+    class Linear8bitLt(bnb.nn.Linear8bitLt, LoraLayer):
+        # Lora implemented in a dense layer
+        def __init__(
+            self,
+            adapter_name,
+            in_features,
+            out_features,
+            r: int = 0,
+            lora_alpha: int = 1,
+            lora_dropout: float = 0.0,
+            **kwargs,
+        ):
+            bnb.nn.Linear8bitLt.__init__(
+                self,
+                in_features,
+                out_features,
+                bias=kwargs.get("bias", True),
+                has_fp16_weights=kwargs.get("has_fp16_weights", True),
+                memory_efficient_backward=kwargs.get("memory_efficient_backward", False),
+                threshold=kwargs.get("threshold", 0.0),
+                index=kwargs.get("index", None),
+            )
+            LoraLayer.__init__(self, in_features=in_features, out_features=out_features)
+
+            # Freezing the pre-trained weight matrix
+            self.weight.requires_grad = False
+            init_lora_weights = kwargs.pop("init_lora_weights", True)
+            self.update_layer(adapter_name, r, lora_alpha, lora_dropout, init_lora_weights)
+            self.active_adapter = adapter_name
+
+        def forward(self, x: torch.Tensor):
+            result = super().forward(x)
+
+            if self.disable_adapters or self.active_adapter not in self.lora_A.keys():
+                return result
+            elif self.r[self.active_adapter] > 0:
+                if not torch.is_autocast_enabled():
+                    expected_dtype = result.dtype
+
+                    if x.dtype != torch.float32:
+                        x = x.float()
+                    output = (
+                        self.lora_B[self.active_adapter](
+                            self.lora_A[self.active_adapter](self.lora_dropout[self.active_adapter](x))
+                        ).to(expected_dtype)
+                        * self.scaling[self.active_adapter]
+                    )
+                else:
+                    output = (
+                        self.lora_B[self.active_adapter](
+                            self.lora_A[self.active_adapter](self.lora_dropout[self.active_adapter](x))
+                        )
+                        * self.scaling[self.active_adapter]
+                    )
+                result += output
+            return result
+
+    if is_bnb_4bit_available():
+
+        class Linear4bit(bnb.nn.Linear4bit, LoraLayer):
+            # Lora implemented in a dense layer
+            def __init__(
+                self,
+                adapter_name,
+                in_features,
+                out_features,
+                r: int = 0,
+                lora_alpha: int = 1,
+                lora_dropout: float = 0.0,
+                **kwargs,
+            ):
+                bnb.nn.Linear4bit.__init__(
+                    self,
+                    in_features,
+                    out_features,
+                    bias=kwargs.get("bias", True),
+                    compute_dtype=kwargs.get("compute_dtype", torch.float32),
+                    compress_statistics=kwargs.get("compress_statistics", True),
+                    quant_type=kwargs.get("quant_type", "nf4"),
+                )
+                LoraLayer.__init__(self, in_features=in_features, out_features=out_features)
+
+                # Freezing the pre-trained weight matrix
+                self.weight.requires_grad = False
+
+                init_lora_weights = kwargs.pop("init_lora_weights", True)
+                self.update_layer(adapter_name, r, lora_alpha, lora_dropout, init_lora_weights)
+                self.active_adapter = adapter_name
+
+            def forward(self, x: torch.Tensor):
+                result = super().forward(x)
+
+                if self.disable_adapters or self.active_adapter not in self.lora_A.keys():
+                    return result
+                elif self.r[self.active_adapter] > 0:
+                    result = result.clone()
+                    if not torch.is_autocast_enabled():
+                        expected_dtype = result.dtype
+                        x = x.to(self.lora_A[self.active_adapter].weight.dtype)
+                        output = (
+                            self.lora_B[self.active_adapter](
+                                self.lora_A[self.active_adapter](self.lora_dropout[self.active_adapter](x))
+                            ).to(expected_dtype)
+                            * self.scaling[self.active_adapter]
+                        )
+                    else:
+                        output = (
+                            self.lora_B[self.active_adapter](
+                                self.lora_A[self.active_adapter](self.lora_dropout[self.active_adapter](x))
+                            )
+                            * self.scaling[self.active_adapter]
+                        )
+                    result += output
+                return result

model/peft/tuners/moelora.py

@@ -0,0 +1,1164 @@
+# coding=utf-8
+# Copyright 2023-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import traceback
+import pickle
+import math
+import re
+import warnings
+from dataclasses import asdict, dataclass, field
+from enum import Enum
+from typing import List, Optional, Tuple, Union
+import itertools
+import copy
+import numpy as np
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from transformers.pytorch_utils import Conv1D
+from .gating import GATING_TO_MODEL_MAPPING
+
+from ..import_utils import is_bnb_4bit_available, is_bnb_available
+from ..utils import (
+    COMMON_LAYERS_PATTERN,
+    TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPING,
+    ModulesToSaveWrapper,
+    PeftConfig,
+    PeftType,
+    _freeze_adapter,
+    _get_submodules,
+    transpose,
+)
+
+if is_bnb_available():
+    import bitsandbytes as bnb
+
+
+@dataclass
+class MoeLoraConfig(PeftConfig):
+    """
+    This is the configuration class to store the configuration of a [`MoeLoraModel`].
+
+    Args:
+        r (`int`): Lora attention dimension.
+        target_modules (`Union[List[str],str]`): The names of the modules to apply Lora to.
+        lora_alpha (`int`): The alpha parameter for Lora scaling.
+        lora_dropout (`float`): The dropout probability for Lora layers.
+        fan_in_fan_out (`bool`): Set this to True if the layer to replace stores weight like (fan_in, fan_out).
+        For example, gpt-2 uses `Conv1D` which stores weights like (fan_in, fan_out) and hence this should be set to `True`.:
+        bias (`str`): Bias type for Lora. Can be 'none', 'all' or 'lora_only'
+        modules_to_save (`List[str]`):List of modules apart from LoRA layers to be set as trainable
+            and saved in the final checkpoint.
+        layers_to_transform (`Union[List[int],int]`):
+            The layer indexes to transform, if this argument is specified, it will apply the LoRA transformations on
+            the layer indexes that are specified in this list. If a single integer is passed, it will apply the LoRA
+            transformations on the layer at this index.
+        layers_pattern (`str`):
+            The layer pattern name, used only if `layers_to_transform` is different from `None` and if the layer
+            pattern is not in the common layers pattern.
+    """
+
+    r: int = field(default=8, metadata={"help": "Lora attention dimension"})
+    num_moe: int = field(default=8, metadata={"help": "Num experts of MoeLora"})
+    gating: str = field(default="Standard", metadata={"help": "Select the gating network for each MoeLora"})
+    target_modules: Optional[Union[List[str], str]] = field(
+        default=None,
+        metadata={
+            "help": "List of module names or regex expression of the module names to replace with Lora."
+                    "For example, ['q', 'v'] or '.*decoder.*(SelfAttention|EncDecAttention).*(q|v)$' "
+        },
+    )
+    lora_alpha: int = field(default=8, metadata={"help": "Lora alpha"})
+    lora_dropout: float = field(default=0.0, metadata={"help": "Lora dropout"})
+    fan_in_fan_out: bool = field(
+        default=False,
+        metadata={"help": "Set this to True if the layer to replace stores weight like (fan_in, fan_out)"},
+    )
+    bias: str = field(default="none", metadata={"help": "Bias type for Lora. Can be 'none', 'all' or 'lora_only'"})
+    modules_to_save: Optional[List[str]] = field(
+        default=None,
+        metadata={
+            "help": "List of modules apart from LoRA layers to be set as trainable and saved in the final checkpoint. "
+                    "For example, in Sequence Classification or Token Classification tasks, "
+                    "the final layer `classifier/score` are randomly initialized and as such need to be trainable and saved."
+        },
+    )
+    init_lora_weights: bool = field(
+        default=True,
+        metadata={"help": "Whether to initialize the weights of the Lora layers."},
+    )
+    layers_to_transform: Optional[Union[List, int]] = field(
+        default=None,
+        metadata={
+            "help": "The layer indexes to transform, is this argument is specified, PEFT will transform only the layers indexes that are specified inside this list. If a single integer is passed, PEFT will transform only the layer at this index."
+        },
+    )
+    layers_pattern: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": "The layer pattern name, used only if `layers_to_transform` is different to None and if the layer pattern is not in the common layers pattern."
+        },
+    )
+
+    def __post_init__(self):
+        self.peft_type = PeftType.MOELORA
+
+
+class MoeLoraModel(torch.nn.Module):
+    """
+    Creates Low Rank Adapter (Lora) model from a pretrained transformers model.
+
+    Args:
+        model ([`~transformers.PreTrainedModel`]): The model to be adapted.
+        config ([`LoraConfig`]): The configuration of the Lora model.
+
+    Returns:
+        `torch.nn.Module`: The Lora model.
+
+    Example:
+
+        ```py
+        >>> from transformers import AutoModelForSeq2SeqLM
+        >>> from moelora import MoeLoraModel, MoeLoraConfig
+
+        >>> config = LoraConfig(
+        ...     peft_type="LORA",
+        ...     task_type="SEQ_2_SEQ_LM",
+        ...     r=8,
+        ...     lora_alpha=32,
+        ...     target_modules=["q", "v"],
+        ...     lora_dropout=0.01,
+        ... )
+
+        >>> model = AutoModelForSeq2SeqLM.from_pretrained("t5-base")
+        >>> lora_model = MoeLoraModel(config, model)
+        ```
+
+        ```py
+        >>> import transformers
+        >>> from peft import LoraConfig, PeftModel, get_peft_model, prepare_model_for_int8_training
+
+        >>> target_modules = ["q_proj", "k_proj", "v_proj", "out_proj", "fc_in", "fc_out", "wte"]
+        >>> config = LoraConfig(
+        ...     r=4, lora_alpha=16, target_modules=target_modules, lora_dropout=0.1, bias="none", task_type="CAUSAL_LM"
+        ... )
+
+        >>> model = transformers.GPTJForCausalLM.from_pretrained(
+        ...     "kakaobrain/kogpt",
+        ...     revision="KoGPT6B-ryan1.5b-float16",  # or float32 version: revision=KoGPT6B-ryan1.5b
+        ...     pad_token_id=tokenizer.eos_token_id,
+        ...     use_cache=False,
+        ...     device_map={"": rank},
+        ...     torch_dtype=torch.float16,
+        ...     load_in_8bit=True,
+        ... )
+        >>> model = prepare_model_for_int8_training(model)
+        >>> lora_model = get_peft_model(model, config)
+        ```
+
+    **Attributes**:
+        - **model** ([`~transformers.PreTrainedModel`]) -- The model to be adapted.
+        - **peft_config** ([`LoraConfig`]): The configuration of the Lora model.
+    """
+
+    def __init__(self, model, config, adapter_name):
+        super().__init__()
+        self.model = model
+
+        # self.forward = self.model.forward
+        
+        
+        self.self_model_generate = self.model.generate
+        self.model.generate = self.generate
+    
+        """self.self_model_forward = self.model.forward
+        self.model.forward = self.forward"""
+        
+        self.peft_config = config
+        self.global_user_embeds = []
+        self.gate_weights = []
+        self.add_adapter(adapter_name, self.peft_config[adapter_name])
+    
+    def forward(self, **kwargs):        
+        self.global_user_embeds.clear()
+        user_embeds = kwargs['user_embeds']
+        self.global_user_embeds.extend([user_embeds])
+        
+        self.gate_weights.clear()
+        gate_weights = kwargs['gate_weights']
+        self.gate_weights.extend([gate_weights])
+        
+        return self.model(**kwargs)
+        # return self.self_model_forward(**kwargs)
+    
+    def generate(self, **kwargs):
+        self.global_user_embeds.clear()
+        user_embeds = kwargs['user_embeds']
+        self.global_user_embeds.extend([user_embeds])
+        
+        self.gate_weights.clear()
+        gate_weights = kwargs['gate_weights']
+        # print("generate, gate_weights:", gate_weights)
+        self.gate_weights.extend([gate_weights])
+        
+        return self.self_model_generate(**kwargs)
+    
+    def add_adapter(self, adapter_name, config=None):
+        if config is not None:
+            model_config = self.model.config.to_dict() if hasattr(self.model.config, "to_dict") else self.model.config
+            config = self._prepare_moelora_config(config, model_config)
+            self.peft_config[adapter_name] = config
+        self._find_and_replace(adapter_name)
+        if len(self.peft_config) > 1 and self.peft_config[adapter_name].bias != "none":
+            raise ValueError(
+                "MoeLoraModel supports only 1 adapter with bias. When using multiple adapters, set bias to 'none' for all adapters."
+            )
+        mark_only_lora_as_trainable(self.model, self.peft_config[adapter_name].bias)
+        if self.peft_config[adapter_name].inference_mode:
+            _freeze_adapter(self.model, adapter_name)
+
+    def _check_quantization_dependency(self):
+        loaded_in_4bit = getattr(self.model, "is_loaded_in_4bit", False)
+        loaded_in_8bit = getattr(self.model, "is_loaded_in_8bit", False)
+        if (loaded_in_4bit or loaded_in_8bit) and not is_bnb_available():
+            raise ImportError(
+                "To use Lora with 8-bit or 4-bit quantization, please install the `bitsandbytes` package. "
+                "You can install it with `pip install bitsandbytes`."
+            )
+
+    def _check_target_module_exists(self, moelora_config, key):
+        if isinstance(moelora_config.target_modules, str):
+            target_module_found = re.fullmatch(moelora_config.target_modules, key)
+        else:
+            target_module_found = any(key.endswith(target_key) for target_key in moelora_config.target_modules)
+            is_using_layer_indexes = getattr(moelora_config, "layers_to_transform", None) is not None
+            layer_indexing_pattern = getattr(moelora_config, "layers_pattern", None)
+
+            if is_using_layer_indexes and target_module_found:
+                layers_pattern = COMMON_LAYERS_PATTERN if layer_indexing_pattern is None else layer_indexing_pattern
+                layers_pattern = [layers_pattern] if isinstance(layers_pattern, str) else layers_pattern
+
+                for pattern in layers_pattern:
+                    layer_index = re.match(f".*.{pattern}\.(\d+)\.*", key)
+                    if layer_index is not None:
+                        layer_index = int(layer_index.group(1))
+                        if isinstance(moelora_config.layers_to_transform, int):
+                            target_module_found = layer_index == moelora_config.layers_to_transform
+                        else:
+                            target_module_found = layer_index in moelora_config.layers_to_transform
+
+                        break
+                    else:
+                        target_module_found = False
+        return target_module_found
+
+    def _create_new_module(self, moelora_config, adapter_name, target, **kwargs):
+        bias = hasattr(target, "bias") and target.bias is not None
+        kwargs = {
+            "r": moelora_config.r,
+
+            "num_moe": moelora_config.num_moe,
+            "gating": moelora_config.gating,
+            "global_user_embeds": self.global_user_embeds,
+            
+            "gate_weights": self.gate_weights,
+            
+            "lora_alpha": moelora_config.lora_alpha,
+            "lora_dropout": moelora_config.lora_dropout,
+            "fan_in_fan_out": moelora_config.fan_in_fan_out,
+            "init_lora_weights": moelora_config.init_lora_weights,
+        }
+        loaded_in_4bit = getattr(self.model, "is_loaded_in_4bit", False)
+        loaded_in_8bit = getattr(self.model, "is_loaded_in_8bit", False)
+
+        if loaded_in_8bit and isinstance(target, bnb.nn.Linear8bitLt):
+            eightbit_kwargs = kwargs.copy()
+            eightbit_kwargs.update(
+                {
+                    "has_fp16_weights": target.state.has_fp16_weights,
+                    "memory_efficient_backward": target.state.memory_efficient_backward,
+                    "threshold": target.state.threshold,
+                    "index": target.index,
+                }
+            )
+            eightbit_kwargs.pop('fan_in_fan_out', None)
+            eightbit_kwargs.pop('init_lora_weights', None)
+            new_module = Linear8bitLt(
+                adapter_name, target.in_features, target.out_features, bias=bias, **eightbit_kwargs
+            )
+        elif loaded_in_4bit and is_bnb_4bit_available() and isinstance(target, bnb.nn.Linear4bit):
+            fourbit_kwargs = kwargs.copy()
+            fourbit_kwargs.update(
+                {
+                    "compute_dtype": target.compute_dtype,
+                    "compress_statistics": target.weight.compress_statistics,
+                    "quant_type": target.weight.quant_type,
+                }
+            )
+            new_module = Linear4bit(adapter_name, target.in_features, target.out_features, bias=bias, **fourbit_kwargs)
+        elif isinstance(target, torch.nn.Embedding):
+            embedding_kwargs = kwargs.copy()
+            embedding_kwargs.pop("fan_in_fan_out", None)
+            in_features, out_features = target.num_embeddings, target.embedding_dim
+            new_module = Embedding(adapter_name, in_features, out_features, **embedding_kwargs)
+        elif isinstance(target, torch.nn.Conv2d):
+            out_channels, in_channels = target.weight.size()[:2]
+            kernel_size = target.weight.size()[2:]
+            stride = target.stride
+            padding = target.padding
+            new_module = Conv2d(adapter_name, in_channels, out_channels, kernel_size, stride, padding, **kwargs)
+        else:
+            if isinstance(target, torch.nn.Linear):
+                in_features, out_features = target.in_features, target.out_features
+                if kwargs["fan_in_fan_out"]:
+                    warnings.warn(
+                        "fan_in_fan_out is set to True but the target module is `torch.nn.Linear`. "
+                        "Setting fan_in_fan_out to False."
+                    )
+                    kwargs["fan_in_fan_out"] = moelora_config.fan_in_fan_out = False
+            elif isinstance(target, Conv1D):
+                in_features, out_features = (
+                    target.weight.ds_shape if hasattr(target.weight, "ds_shape") else target.weight.shape
+                )
+                if not kwargs["fan_in_fan_out"]:
+                    warnings.warn(
+                        "fan_in_fan_out is set to False but the target module is `Conv1D`. "
+                        "Setting fan_in_fan_out to True."
+                    )
+                    kwargs["fan_in_fan_out"] = moelora_config.fan_in_fan_out = True
+            else:
+                raise ValueError(
+                    f"Target module {target} is not supported. "
+                    f"Currently, only `torch.nn.Linear` and `Conv1D` are supported."
+                )
+            new_module = Linear(adapter_name, in_features, out_features, bias=bias, **kwargs)
+
+        return new_module
+
+    def _find_and_replace(self, adapter_name):
+        moelora_config = self.peft_config[adapter_name]
+        self._check_quantization_dependency()
+        is_target_modules_in_base_model = False
+        key_list = [key for key, _ in self.model.named_modules()]
+
+        for key in key_list:
+            if not self._check_target_module_exists(moelora_config, key):
+                continue
+
+            is_target_modules_in_base_model = True
+            parent, target, target_name = _get_submodules(self.model, key)
+
+            if isinstance(target, MoeLoraLayer) and isinstance(target, torch.nn.Conv2d):
+                target.update_layer_conv2d(
+                    adapter_name,
+                    moelora_config.r,
+                    moelora_config.lora_alpha,
+                    moelora_config.lora_dropout,
+                    moelora_config.init_lora_weights,
+                )
+            elif isinstance(target, MoeLoraLayer):
+                target.update_layer(
+                    adapter_name,
+                    moelora_config.r,
+                    moelora_config.num_moe,
+                    moelora_config.gating,
+                    moelora_config.lora_alpha,
+                    moelora_config.lora_dropout,
+                    moelora_config.init_lora_weights,
+                )
+            else:
+                new_module = self._create_new_module(moelora_config, adapter_name, target)
+                self._replace_module(parent, target_name, new_module, target)
+
+        if not is_target_modules_in_base_model:
+            raise ValueError(
+                f"Target modules {moelora_config.target_modules} not found in the base model. "
+                f"Please check the target modules and try again."
+            )
+
+    def _replace_module(self, parent_module, child_name, new_module, old_module):
+        setattr(parent_module, child_name, new_module)
+        new_module.weight = old_module.weight
+        if hasattr(old_module, "bias"):
+            if old_module.bias is not None:
+                new_module.bias = old_module.bias
+
+        if getattr(old_module, "state", None) is not None:
+            new_module.state = old_module.state
+            new_module.to(old_module.weight.device)
+
+        # dispatch to correct device
+        for name, module in new_module.named_modules():
+            if "lora_" in name:
+                module.to(old_module.weight.device)
+            
+            if "gating" in name:
+                module.to(old_module.weight.device)
+
+            if "ranknum" in name:
+                module.to(old_module.weight.device)
+
+    def __getattr__(self, name: str):
+        """Forward missing attributes to the wrapped module."""
+        try:
+            return super().__getattr__(name)  # defer to nn.Module's logic
+        except AttributeError:
+            return getattr(self.model, name)
+
+    def get_peft_config_as_dict(self, inference: bool = False):
+        config_dict = {}
+        for key, value in self.peft_config.items():
+            config = {k: v.value if isinstance(v, Enum) else v for k, v in asdict(value).items()}
+            if inference:
+                config["inference_mode"] = True
+        config_dict[key] = config
+        return config
+
+    def _set_adapter_layers(self, enabled=True):
+        for module in self.model.modules():
+            if isinstance(module, MoeLoraLayer):
+                module.disable_adapters = False if enabled else True
+
+    def enable_adapter_layers(self):
+        self._set_adapter_layers(enabled=True)
+
+    def disable_adapter_layers(self):
+        self._set_adapter_layers(enabled=False)
+
+    def set_adapter(self, adapter_name):
+        for module in self.model.modules():
+            if isinstance(module, MoeLoraLayer):
+                if module.merged:
+                    warnings.warn("Adapter cannot be set when the model is merged. Unmerging the model first.")
+                    module.unmerge()
+                module.active_adapter = adapter_name
+
+    def merge_adapter(self):
+        for module in self.model.modules():
+            if isinstance(module, MoeLoraLayer):
+                module.merge()
+
+    def unmerge_adapter(self):
+        for module in self.model.modules():
+            if isinstance(module, MoeLoraLayer):
+                module.unmerge()
+
+    @staticmethod
+    def _prepare_moelora_config(peft_config, model_config):
+        if peft_config.target_modules is None:
+            if model_config["model_type"] not in TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPING:
+                raise ValueError("Please specify `target_modules` in `peft_config`")
+            peft_config.target_modules = TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPING[model_config["model_type"]]
+        return peft_config
+
+    def merge_and_unload(self):
+        r"""
+        This method merges the LoRa layers into the base model. This is needed if someone wants to use the base model
+        as a standalone model.
+        """
+        if getattr(self.config, "model_type", None) == "gpt2":
+            raise ValueError("GPT2 models are not supported for merging LORA layers")
+
+        if getattr(self.model, "is_loaded_in_8bit", False) or getattr(self.model, "is_loaded_in_4bit", False):
+            raise ValueError("Cannot merge LORA layers when the model is loaded in 8-bit mode")
+
+        key_list = [key for key, _ in self.model.named_modules() if "lora" not in key]
+        for key in key_list:
+            try:
+                parent, target, target_name = _get_submodules(self.model, key)
+            except AttributeError:
+                continue
+            if isinstance(target, MoeLoraLayer):
+                if isinstance(target, nn.Embedding):
+                    new_module = torch.nn.Embedding(target.in_features, target.out_features)
+                elif isinstance(target, nn.Conv2d):
+                    new_module = torch.nn.Conv2d(
+                        target.in_channels,
+                        target.out_channels,
+                        kernel_size=target.kernel_size,
+                        stride=target.stride,
+                        padding=target.padding,
+                        dilation=target.dilation,
+                    )
+                else:
+                    bias = target.bias is not None
+                    new_module = torch.nn.Linear(target.in_features, target.out_features, bias=bias)
+                target.merge()
+                self._replace_module(parent, target_name, new_module, target)
+
+            # save any additional trainable modules part of `modules_to_save`
+            if isinstance(target, ModulesToSaveWrapper):
+                setattr(parent, target_name, target.modules_to_save[target.active_adapter])
+
+        return self.model
+
+    def add_weighted_adapter(self, adapters, weights, adapter_name):
+        if len({self.peft_config[adapter].r for adapter in adapters}) != 1:
+            raise ValueError("All adapters must have the same r value")
+        self.peft_config[adapter_name] = self.peft_config[adapters[0]]
+        self.peft_config[adapter_name].lora_alpha = self.peft_config[adapters[0]].r
+        self._find_and_replace(adapter_name)
+        mark_only_lora_as_trainable(self.model, self.peft_config[adapter_name].bias)
+        _freeze_adapter(self.model, adapter_name)
+        key_list = [key for key, _ in self.model.named_modules() if "lora" not in key]
+        for key in key_list:
+            _, target, _ = _get_submodules(self.model, key)
+            if isinstance(target, MoeLoraLayer):
+                if adapter_name in target.lora_A:
+                    target.lora_A[adapter_name].weight.data = target.lora_A[adapter_name].weight.data * 0.0
+                    target.lora_B[adapter_name].weight.data = target.lora_B[adapter_name].weight.data * 0.0
+                    for adapter, weight in zip(adapters, weights):
+                        if adapter not in target.lora_A:
+                            continue
+                        target.lora_A[adapter_name].weight.data += (
+                                target.lora_A[adapter].weight.data * weight * target.scaling[adapter]
+                        )
+                        target.lora_B[adapter_name].weight.data += target.lora_B[adapter].weight.data * weight
+
+                elif adapter_name in target.lora_embedding_A:
+                    target.lora_embedding_A[adapter_name].data = target.lora_embedding_A[adapter_name].data * 0.0
+                    target.lora_embedding_B[adapter_name].data = target.lora_embedding_B[adapter_name].data * 0.0
+                    for adapter, weight in zip(adapters, weights):
+                        if adapter not in target.lora_embedding_A:
+                            continue
+                        target.lora_embedding_A[adapter_name].data += (
+                                target.lora_embedding_A[adapter].data * weight * target.scaling[adapter]
+                        )
+                        target.lora_embedding_B[adapter_name].data += target.lora_embedding_B[adapter].data * weight
+
+
+# Below code is based on https://github.com/microsoft/LoRA/blob/main/loralib/layers.py
+# and modified to work with PyTorch FSDP
+
+
+#  ------------------------------------------------------------------------------------------
+#  Copyright (c) Microsoft Corporation. All rights reserved.
+#  Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+#  ------------------------------------------------------------------------------------------
+
+
+# had to adapt it for `lora_only` to work
+def mark_only_lora_as_trainable(model: nn.Module, bias: str = "none") -> None:
+    for n, p in model.named_parameters():
+        if "lora_" not in n and "gating" not in n:
+            p.requires_grad = False
+    if bias == "none":
+        return
+    elif bias == "all":
+        for n, p in model.named_parameters():
+            if "bias" in n:
+                p.requires_grad = True
+    elif bias == "lora_only":
+        for m in model.modules():
+            if isinstance(m, MoeLoraLayer) and hasattr(m, "bias") and m.bias is not None:
+                m.bias.requires_grad = True
+    else:
+        raise NotImplementedError
+
+
+class MoeLoraLayer:
+    def __init__(self, in_features: int, out_features: int, **kwargs):
+        self.r = {}
+        
+        self.num_moe = {}
+        
+        self.lora_alpha = {}
+        self.scaling = {}
+        self.lora_dropout = nn.ModuleDict({})
+        self.lora_A = nn.ModuleDict({})
+        self.lora_B = nn.ModuleDict({})
+        
+        self.gating = nn.ModuleDict({})
+        
+        # For Embedding layer
+        self.lora_embedding_A = nn.ParameterDict({})
+        self.lora_embedding_B = nn.ParameterDict({})
+        # Mark the weight as unmerged
+        self.merged = False
+        self.disable_adapters = False
+        self.in_features = in_features
+        self.out_features = out_features
+        self.kwargs = kwargs
+
+    def update_layer(self, adapter_name, r, num_moe, gating, lora_alpha, lora_dropout,
+                     init_lora_weights):
+        self.r[adapter_name] = r
+        
+        self.num_moe[adapter_name] = num_moe
+        
+        self.lora_alpha[adapter_name] = lora_alpha
+        
+        self.gating.update(
+            nn.ModuleDict({adapter_name: GATING_TO_MODEL_MAPPING[gating](num_moe=num_moe, dim=self.in_features)}))
+        self.gating[adapter_name].to(self.weight.device)
+        
+        if lora_dropout > 0.0:
+            lora_dropout_layer = nn.Dropout(p=lora_dropout)
+        else:
+            lora_dropout_layer = nn.Identity()
+
+        self.lora_dropout.update(nn.ModuleDict({adapter_name: lora_dropout_layer}))
+        # Actual trainable parameters
+        if r > 0:
+            self.lora_A.update(nn.ModuleDict({adapter_name: nn.Linear(self.in_features, r, bias=False)}))  # TODO
+            self.lora_B.update(nn.ModuleDict({adapter_name: nn.Linear(r, self.out_features, bias=False)}))
+            self.scaling[adapter_name] = lora_alpha / (r // num_moe)
+        if init_lora_weights:
+            self.reset_lora_parameters(adapter_name)
+        self.to(self.weight.device)
+
+    def update_layer_conv2d(self, adapter_name, r, lora_alpha, lora_dropout, init_lora_weights):
+        self.r[adapter_name] = r
+        self.lora_alpha[adapter_name] = lora_alpha
+        if lora_dropout > 0.0:
+            lora_dropout_layer = nn.Dropout(p=lora_dropout)
+        else:
+            lora_dropout_layer = nn.Identity()
+
+        self.lora_dropout.update(nn.ModuleDict({adapter_name: lora_dropout_layer}))
+        # Actual trainable parameters
+        if r > 0:
+            kernel_size = self.kwargs["kernel_size"]
+            stride = self.kwargs["stride"]
+            padding = self.kwargs["padding"]
+            self.lora_A.update(
+                nn.ModuleDict({adapter_name: nn.Conv2d(self.in_features, r, kernel_size, stride, padding, bias=False)})
+            )
+            self.lora_B.update(
+                nn.ModuleDict({adapter_name: nn.Conv2d(r, self.out_features, (1, 1), (1, 1), bias=False)})
+            )
+            self.scaling[adapter_name] = lora_alpha / r
+        if init_lora_weights:
+            self.reset_lora_parameters(adapter_name)
+        self.to(self.weight.device)
+
+    def update_layer_embedding(self, adapter_name, r, lora_alpha, lora_dropout, init_lora_weights):
+        self.r[adapter_name] = r
+        self.lora_alpha[adapter_name] = lora_alpha
+        if lora_dropout > 0.0:
+            lora_dropout_layer = nn.Dropout(p=lora_dropout)
+        else:
+            lora_dropout_layer = nn.Identity()
+
+        self.lora_dropout.update(nn.ModuleDict({adapter_name: lora_dropout_layer}))
+        # Actual trainable parameters
+        if r > 0:
+            self.lora_embedding_A.update(
+                nn.ParameterDict({adapter_name: nn.Parameter(self.weight.new_zeros((r, self.in_features)))})
+            )
+            self.lora_embedding_B.update(
+                nn.ParameterDict({adapter_name: nn.Parameter(self.weight.new_zeros((self.out_features, r)))})
+            )
+            self.scaling[adapter_name] = lora_alpha / r
+        if init_lora_weights:
+            self.reset_lora_parameters(adapter_name)
+        self.to(self.weight.device)
+
+    def reset_lora_parameters(self, adapter_name):
+        if adapter_name in self.lora_A.keys():
+            # initialize A the same way as the default for nn.Linear and B to zero
+            nn.init.kaiming_uniform_(self.lora_A[adapter_name].weight, a=math.sqrt(5))
+            nn.init.zeros_(self.lora_B[adapter_name].weight)
+        if adapter_name in self.lora_embedding_A.keys():
+            # initialize a the same way as the default for nn.linear and b to zero
+            nn.init.zeros_(self.lora_embedding_A[adapter_name])
+            nn.init.normal_(self.lora_embedding_B[adapter_name])
+
+
+class Linear(nn.Linear, MoeLoraLayer):
+    # Lora implemented in a dense layer
+    def __init__(
+            self,
+            adapter_name: str,
+            in_features: int,
+            out_features: int,
+            r: int = 0,
+            
+            # moe额外参数
+            num_moe: int = 0,
+            gating: str = "",
+            global_user_embeds: List = [],
+            gate_weights: List = [],
+            
+            lora_alpha: int = 1,
+            lora_dropout: float = 0.0,
+            fan_in_fan_out: bool = False,
+            # Set this to True if the layer to replace stores weight like (fan_in, fan_out)
+            **kwargs,
+    ):
+        init_lora_weights = kwargs.pop("init_lora_weights", True)
+        
+        nn.Linear.__init__(self, in_features, out_features, **kwargs)
+        MoeLoraLayer.__init__(self, in_features=in_features, out_features=out_features)
+        # Freezing the pre-trained weight matrix冻结预训练参数
+        self.weight.requires_grad = False
+        self.gating.requires_grad_ = True
+        self.fan_in_fan_out = fan_in_fan_out
+        if fan_in_fan_out:
+            self.weight.data = self.weight.data.T
+        nn.Linear.reset_parameters(self)
+        self.update_layer(adapter_name, r, num_moe, gating, lora_alpha, lora_dropout, init_lora_weights)
+        self.active_adapter = adapter_name
+        self.global_user_embeds = global_user_embeds
+        self.gate_weights = gate_weights
+
+    def merge(self):
+        if self.active_adapter not in self.lora_A.keys():
+            return
+        if self.merged:
+            warnings.warn("Already merged. Nothing to do.")
+            return
+        if self.r[self.active_adapter] > 0:
+            self.weight.data += (
+                    transpose(
+                        self.lora_B[self.active_adapter].weight @ self.lora_A[self.active_adapter].weight,
+                        self.fan_in_fan_out,
+                    )
+                    * self.scaling[self.active_adapter]
+            )
+            self.merged = True
+
+    def unmerge(self):
+        if self.active_adapter not in self.lora_A.keys():
+            return
+        if not self.merged:
+            warnings.warn("Already unmerged. Nothing to do.")
+            return
+        if self.r[self.active_adapter] > 0:
+            self.weight.data -= (
+                    transpose(
+                        self.lora_B[self.active_adapter].weight @ self.lora_A[self.active_adapter].weight,
+                        self.fan_in_fan_out,
+                    )
+                    * self.scaling[self.active_adapter]
+            )
+            self.merged = False
+    
+    def calculate_B(self, A_out):
+        batch_size, seq_len, n, r = A_out.size()
+        weight = self.lora_B[self.active_adapter].weight.t().reshape(n, r, -1)
+        return torch.einsum('ijkl, klm->ijkm', A_out, weight)
+
+
+    def forward(self, x: torch.Tensor, **kwargs):
+        flag = False
+        # user_embeds = self.global_user_embeds[0]
+        gate_weights = self.gate_weights[0]
+        previous_dtype = x.dtype
+        batch_size, seq_len, _ = x.size()
+        
+        try:
+            if self.active_adapter not in self.lora_A.keys():
+                return F.linear(x, transpose(self.weight, self.fan_in_fan_out), bias=self.bias)
+            if self.disable_adapters:
+                if self.r[self.active_adapter] > 0 and self.merged:
+                    self.unmerge()
+                result = F.linear(x, transpose(self.weight, self.fan_in_fan_out), bias=self.bias)
+            elif self.r[self.active_adapter] > 0 and not self.merged:
+                result = F.linear(x, transpose(self.weight, self.fan_in_fan_out), bias=self.bias)
+                x = x.to(self.lora_A[self.active_adapter].weight.dtype)
+                    
+                A_out = self.lora_A[self.active_adapter](
+                            self.lora_dropout[self.active_adapter](x)
+                            ).reshape(batch_size, seq_len, self.num_moe[self.active_adapter], -1)
+                B_out = self.calculate_B(A_out)
+                
+                # Gate = self.gating[self.active_adapter](user_embeds).unsqueeze(-1)
+                Gate = gate_weights.unsqueeze(-1)
+                
+                
+                result += ((B_out * Gate).sum(dim=-2) * self.scaling[self.active_adapter])
+                
+                
+                gate_weights_squeezed = Gate.squeeze(-1)  
+                gate_weights_split = gate_weights_squeezed.split(1, dim=0) 
+                gate_weight_vectors = [x.squeeze() for x in gate_weights_split] 
+                        
+            else:
+                result = F.linear(x, transpose(self.weight, self.fan_in_fan_out), bias=self.bias)
+
+            
+            result = result.to(previous_dtype)
+
+            return result
+        except Exception as e:
+            print(f"An error occurred: {e}")
+            return torch.zeros_like(x)  # 返回一个与输入张量相同形状的全零张量
+
+
+class Embedding(nn.Embedding, MoeLoraLayer):
+    # LoRA implemented in a Embedding layer
+    def __init__(
+            self,
+            adapter_name: str,
+            num_embeddings: int,
+            embedding_dim: int,
+            r: int = 0,
+            lora_alpha: int = 1,
+            lora_dropout: float = 0.0,
+            **kwargs,
+    ):
+        init_lora_weights = kwargs.pop("init_lora_weights", True)
+
+        nn.Embedding.__init__(self, num_embeddings, embedding_dim, **kwargs)
+        MoeLoraLayer.__init__(self, in_features=num_embeddings, out_features=embedding_dim)
+
+        self.weight.requires_grad = False
+
+        nn.Embedding.reset_parameters(self)
+        self.update_layer_embedding(adapter_name, r, lora_alpha, lora_dropout, init_lora_weights)
+        self.active_adapter = adapter_name
+
+    def unmerge(self, mode: bool = True):
+        if not self.merged:
+            warnings.warn("Already unmerged. Nothing to do.")
+            return
+        if self.r[self.active_adapter] > 0:
+            self.weight.data -= (
+                    transpose(
+                        self.lora_embedding_B[self.active_adapter] @ self.lora_embedding_A[self.active_adapter], True
+                    )
+                    * self.scaling[self.active_adapter]
+            )
+            self.merged = False
+
+    def merge(self):
+        if self.merged:
+            warnings.warn("Already merged. Nothing to do.")
+            return
+        if self.r[self.active_adapter] > 0:
+            self.weight.data += (
+                    transpose(
+                        self.lora_embedding_B[self.active_adapter] @ self.lora_embedding_A[self.active_adapter], True
+                    )
+                    * self.scaling[self.active_adapter]
+            )
+            self.merged = True
+
+    def forward(self, x: torch.Tensor):
+        if self.disable_adapters:
+            if self.r[self.active.adapter] > 0 and self.merged:
+                self.weight.data -= (
+                        transpose(
+                            self.lora_embedding_B[self.active_adapter].weight
+                            @ self.lora_embedding_A[self.active_adapter].weight,
+                            True,
+                        )
+                        * self.scaling[self.active_adapter]
+                )
+                self.merged = False
+            return nn.Embedding.forward(self, x)
+
+        elif self.r[self.active_adapter] > 0 and not self.merged:
+            result = nn.Embedding.forward(self, x)
+            if self.r[self.active_adapter] > 0:
+                after_A = F.embedding(
+                    x,
+                    self.lora_embedding_A[self.active_adapter].T,
+                    self.padding_idx,
+                    self.max_norm,
+                    self.norm_type,
+                    self.scale_grad_by_freq,
+                    self.sparse,
+                )
+                result += (after_A @ self.lora_embedding_B[self.active_adapter].T) * self.scaling[self.active_adapter]
+            return result
+        else:
+            return nn.Embedding.forward(self, x)
+
+
+class Conv2d(nn.Conv2d, MoeLoraLayer):
+    # Lora implemented in a conv2d layer
+    def __init__(
+            self,
+            adapter_name: str,
+            in_channels: int,
+            out_channels: int,
+            kernel_size: Union[int, Tuple[int]],
+            stride: Union[int, Tuple[int]] = 1,
+            padding: Union[int, Tuple[int]] = 0,
+            r: int = 0,
+            lora_alpha: int = 1,
+            lora_dropout: float = 0.0,
+            **kwargs,
+    ):
+        init_lora_weights = kwargs.pop("init_lora_weights", True)
+
+        nn.Conv2d.__init__(self, in_channels, out_channels, kernel_size, stride, padding)
+        MoeLoraLayer.__init__(
+            self,
+            in_features=in_channels,
+            out_features=out_channels,
+            kernel_size=kernel_size,
+            stride=stride,
+            padding=padding,
+        )
+        # Freezing the pre-trained weight matrix
+        self.weight.requires_grad = False
+
+        nn.Conv2d.reset_parameters(self)
+        self.update_layer_conv2d(adapter_name, r, lora_alpha, lora_dropout, init_lora_weights)
+        self.active_adapter = adapter_name
+
+    def merge(self):
+        if self.active_adapter not in self.lora_A.keys():
+            return
+        if self.merged:
+            warnings.warn("Already merged. Nothing to do.")
+            return
+        if self.r[self.active_adapter] > 0:
+            # https://github.com/bmaltais/kohya_ss/blob/feb6728762a8f463d15ba936d189d4c3abfaa1ab/networks/lora.py#L117
+            if self.weight.size()[2:4] == (1, 1):
+                # conv2d 1x1
+                self.weight.data += (
+                                            self.lora_B[self.active_adapter].weight.squeeze(3).squeeze(2)
+                                            @ self.lora_A[self.active_adapter].weight.squeeze(3).squeeze(2)
+                                    ).unsqueeze(2).unsqueeze(3) * self.scaling[self.active_adapter]
+            else:
+                # conv2d 3x3
+                self.weight.data += (
+                        F.conv2d(
+                            self.lora_A[self.active_adapter].weight.permute(1, 0, 2, 3),
+                            self.lora_B[self.active_adapter].weight,
+                        ).permute(1, 0, 2, 3)
+                        * self.scaling[self.active_adapter]
+                )
+            self.merged = True
+
+    def unmerge(self):
+        if self.active_adapter not in self.lora_A.keys():
+            return
+        if not self.merged:
+            warnings.warn("Already unmerged. Nothing to do.")
+            return
+        if self.r[self.active_adapter] > 0:
+            if self.weight.size()[2:4] == (1, 1):
+                # conv2d 1x1
+                self.weight.data -= (
+                                            self.lora_B[self.active_adapter].weight.squeeze(3).squeeze(2)
+                                            @ self.lora_A[self.active_adapter].weight.squeeze(3).squeeze(2)
+                                    ).unsqueeze(2).unsqueeze(3) * self.scaling[self.active_adapter]
+            else:
+                # conv2d 3x3
+                self.weight.data += (
+                        F.conv2d(
+                            self.lora_A[self.active_adapter].weight.permute(1, 0, 2, 3),
+                            self.lora_B[self.active_adapter].weight,
+                        ).permute(1, 0, 2, 3)
+                        * self.scaling[self.active_adapter]
+                )
+            self.merged = False
+
+    def forward(self, x: torch.Tensor):
+        previous_dtype = x.dtype
+
+        if self.active_adapter not in self.lora_A.keys():
+            return F.conv2d(
+                x,
+                self.weight,
+                bias=self.bias,
+                stride=self.stride,
+                padding=self.padding,
+                dilation=self.dilation,
+                groups=self.groups,
+            )
+        if self.disable_adapters:
+            if self.r[self.active_adapter] > 0 and self.merged:
+                self.unmerge()
+            result = F.conv2d(
+                x,
+                self.weight,
+                bias=self.bias,
+                stride=self.stride,
+                padding=self.padding,
+                dilation=self.dilation,
+                groups=self.groups,
+            )
+        elif self.r[self.active_adapter] > 0 and not self.merged:
+            result = F.conv2d(
+                x,
+                self.weight,
+                bias=self.bias,
+                stride=self.stride,
+                padding=self.padding,
+                dilation=self.dilation,
+                groups=self.groups,
+            )
+
+            x = x.to(self.lora_A[self.active_adapter].weight.dtype)
+
+            result += (
+                    self.lora_B[self.active_adapter](
+                        self.lora_A[self.active_adapter](self.lora_dropout[self.active_adapter](x))
+                    )
+                    * self.scaling[self.active_adapter]
+            )
+        else:
+            result = F.conv2d(
+                x,
+                self.weight,
+                bias=self.bias,
+                stride=self.stride,
+                padding=self.padding,
+                dilation=self.dilation,
+                groups=self.groups,
+            )
+
+        result = result.to(previous_dtype)
+
+        return result
+
+
+if is_bnb_available():
+
+    class Linear8bitLt(bnb.nn.Linear8bitLt, MoeLoraLayer):
+        # Lora implemented in a dense layer
+        def __init__(
+                self,
+                adapter_name: str,
+                in_features: int,
+                out_features: int,
+                r: int = 0,
+                
+                # moe额外参数
+                num_moe: int = 0,
+                gating: str = "",
+                global_user_embeds: List = [],
+                loss_fn: str = "",
+                
+                lora_alpha: int = 1,
+                lora_dropout: float = 0.0,
+                **kwargs,
+        ):
+            kwargs.setdefault('bias', True)
+            bnb.nn.Linear8bitLt.__init__(
+                self,
+                in_features,
+                out_features,
+                # **kwargs
+            )
+            MoeLoraLayer.__init__(self, in_features=in_features, out_features=out_features)
+
+            # Freezing the pre-trained weight matrix
+            self.weight.requires_grad = False
+            init_lora_weights = kwargs.pop("init_lora_weights", True)
+            # self.update_layer(adapter_name, r, num_moe, gating, loss_fn, lora_alpha, lora_dropout, init_lora_weights)
+            # bnb.nn.Linear8bitLt.reset_parameters(self)
+            
+            self.update_layer(adapter_name, r, num_moe, gating, lora_alpha, lora_dropout, init_lora_weights)
+            self.active_adapter = adapter_name
+            self.global_user_embeds = global_user_embeds
+        
+        def calculate_B(self, A_out):
+            # batch_size, seq_len, self.num_moe[self.active_adapter], -1
+            batch_size, seq_len, n, r = A_out.size()
+            weight = self.lora_B[self.active_adapter].weight.t().reshape(n, r, -1)
+            return torch.einsum('ijkl, klm->ijkm', A_out, weight)
+
+        def forward(self, x: torch.Tensor):
+            user_embeds = self.global_user_embeds[0]
+            result = super().forward(x)
+            
+            batch_size, seq_len, _ = x.size()
+            
+            if self.disable_adapters or self.active_adapter not in self.lora_A.keys():
+                return result
+            elif self.r[self.active_adapter] > 0:
+                if not torch.is_autocast_enabled():
+                    expected_dtype = result.dtype
+
+                    if x.dtype != torch.float32:
+                        x = x.float()
+                    A_out = self.lora_A[self.active_adapter](
+                                self.lora_dropout[self.active_adapter](x)
+                                ).reshape(batch_size, seq_len, self.num_moe[self.active_adapter], -1)
+                    B_out = self.calculate_B(A_out)
+                    Gate = self.gating[self.active_adapter](user_embeds).unsqueeze(-1)
+                    output = ((B_out * Gate).sum(dim=-2).to(expected_dtype) * self.scaling[self.active_adapter])
+                        
+                else:
+                    A_out = self.lora_A[self.active_adapter](
+                        self.lora_dropout[self.active_adapter](x)
+                        ).reshape(batch_size, seq_len, self.num_moe[self.active_adapter], -1)
+                    B_out = self.calculate_B(A_out)
+                    Gate = self.gating[self.active_adapter](user_embeds).unsqueeze(-1)
+                    output = ((B_out * Gate).sum(dim=-2) * self.scaling[self.active_adapter])
+                    
+                result += output
+            return result
+
+
+    if is_bnb_4bit_available():
+
+        class Linear4bit(bnb.nn.Linear4bit, MoeLoraLayer):
+            # Lora implemented in a dense layer
+            def __init__(
+                    self,
+                    adapter_name,
+                    in_features,
+                    out_features,
+                    r: int = 0,
+                    lora_alpha: int = 1,
+                    lora_dropout: float = 0.0,
+                    **kwargs,
+            ):
+                bnb.nn.Linear4bit.__init__(
+                    self,
+                    in_features,
+                    out_features,
+                    bias=kwargs.get("bias", True),
+                    compute_dtype=kwargs.get("compute_dtype", torch.float32),
+                    compress_statistics=kwargs.get("compress_statistics", True),
+                    quant_type=kwargs.get("quant_type", "nf4"),
+                )
+                MoeLoraLayer.__init__(self, in_features=in_features, out_features=out_features)
+
+                # Freezing the pre-trained weight matrix
+                self.weight.requires_grad = False
+
+                init_lora_weights = kwargs.pop("init_lora_weights", True)
+                self.update_layer(adapter_name, r, lora_alpha, lora_dropout, init_lora_weights)
+                self.active_adapter = adapter_name
+
+            def forward(self, x: torch.Tensor):
+                result = super().forward(x)
+                user_embeds = self.global_user_embeds[0]
+
+                if self.disable_adapters or self.active_adapter not in self.lora_A.keys():
+                    return result
+                elif self.r[self.active_adapter] > 0:
+                    result = result.clone()
+                    if not torch.is_autocast_enabled():
+                        expected_dtype = result.dtype
+                        x = x.to(self.lora_A[self.active_adapter].weight.dtype)
+                        batch_size, seq_len, _ = x.size()
+                        A_out = self.lora_A[self.active_adapter](
+                                self.lora_dropout[self.active_adapter](x)
+                                ).reshape(batch_size, seq_len, self.num_moe[self.active_adapter], -1)
+                        B_out = self.calculate_B(A_out)
+                        Gate = self.gating[self.active_adapter](user_embeds).unsqueeze(-1)
+                        output = ((B_out * Gate).sum(dim=-2).to(expected_dtype) * self.scaling[self.active_adapter])
+                    else:
+                        x = x.to(self.lora_A[self.active_adapter].weight.dtype)
+                        batch_size, seq_len, _ = x.size()
+                        A_out = self.lora_A[self.active_adapter](
+                                self.lora_dropout[self.active_adapter](x)
+                                ).reshape(batch_size, seq_len, self.num_moe[self.active_adapter], -1)
+                        B_out = self.calculate_B(A_out)
+                        Gate = self.gating[self.active_adapter](user_embeds).unsqueeze(-1)
+                        output = ((B_out * Gate).sum(dim=-2) * self.scaling[self.active_adapter])
+                    result += output
+                return result

model/peft/tuners/p_tuning.py

@@ -0,0 +1,170 @@
+# coding=utf-8
+# Copyright 2023-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import enum
+import warnings
+from dataclasses import dataclass, field
+from typing import Union
+
+import torch
+
+from ..utils import PeftType, PromptLearningConfig
+
+
+class PromptEncoderReparameterizationType(str, enum.Enum):
+    MLP = "MLP"
+    LSTM = "LSTM"
+
+
+@dataclass
+class PromptEncoderConfig(PromptLearningConfig):
+    """
+    This is the configuration class to store the configuration of a [`PromptEncoder`].
+
+    Args:
+        encoder_reparameterization_type (Union[[`PromptEncoderReparameterizationType`], `str`]):
+            The type of reparameterization to use.
+        encoder_hidden_size (`int`): The hidden size of the prompt encoder.
+        encoder_num_layers (`int`): The number of layers of the prompt encoder.
+        encoder_dropout (`float`): The dropout probability of the prompt encoder.
+    """
+
+    encoder_reparameterization_type: Union[str, PromptEncoderReparameterizationType] = field(
+        default=PromptEncoderReparameterizationType.MLP,
+        metadata={"help": "How to reparameterize the prompt encoder"},
+    )
+    encoder_hidden_size: int = field(
+        default=None,
+        metadata={"help": "The hidden size of the prompt encoder"},
+    )
+    encoder_num_layers: int = field(
+        default=2,
+        metadata={"help": "The number of layers of the prompt encoder"},
+    )
+    encoder_dropout: float = field(
+        default=0.0,
+        metadata={"help": "The dropout of the prompt encoder"},
+    )
+
+    def __post_init__(self):
+        self.peft_type = PeftType.P_TUNING
+
+
+# Based on https://github.com/NVIDIA/NeMo/blob/main/nemo/collections/nlp/modules/common/prompt_encoder.py
+# with some refactor
+class PromptEncoder(torch.nn.Module):
+    """
+    The prompt encoder network that is used to generate the virtual token embeddings for p-tuning.
+
+    Args:
+        config ([`PromptEncoderConfig`]): The configuration of the prompt encoder.
+
+    Example:
+
+    ```py
+    >>> from peft import PromptEncoder, PromptEncoderConfig
+
+    >>> config = PromptEncoderConfig(
+    ...     peft_type="P_TUNING",
+    ...     task_type="SEQ_2_SEQ_LM",
+    ...     num_virtual_tokens=20,
+    ...     token_dim=768,
+    ...     num_transformer_submodules=1,
+    ...     num_attention_heads=12,
+    ...     num_layers=12,
+    ...     encoder_reparameterization_type="MLP",
+    ...     encoder_hidden_size=768,
+    ... )
+
+    >>> prompt_encoder = PromptEncoder(config)
+    ```
+
+    **Attributes**:
+        - **embedding** (`torch.nn.Embedding`) -- The embedding layer of the prompt encoder.
+        - **mlp_head** (`torch.nn.Sequential`) -- The MLP head of the prompt encoder if `inference_mode=False`.
+        - **lstm_head** (`torch.nn.LSTM`) -- The LSTM head of the prompt encoder if `inference_mode=False` and
+        `encoder_reparameterization_type="LSTM"`.
+        - **token_dim** (`int`) -- The hidden embedding dimension of the base transformer model.
+        - **input_size** (`int`) -- The input size of the prompt encoder.
+        - **output_size** (`int`) -- The output size of the prompt encoder.
+        - **hidden_size** (`int`) -- The hidden size of the prompt encoder.
+        - **total_virtual_tokens** (`int`): The total number of virtual tokens of the
+        prompt encoder.
+        - **encoder_type** (Union[[`PromptEncoderReparameterizationType`], `str`]): The encoder type of the prompt
+          encoder.
+
+
+    Input shape: (`batch_size`, `total_virtual_tokens`)
+
+    Output shape: (`batch_size`, `total_virtual_tokens`, `token_dim`)
+    """
+
+    def __init__(self, config):
+        super().__init__()
+        self.token_dim = config.token_dim
+        self.input_size = self.token_dim
+        self.output_size = self.token_dim
+        self.hidden_size = config.encoder_hidden_size
+        self.total_virtual_tokens = config.num_virtual_tokens * config.num_transformer_submodules
+        self.encoder_type = config.encoder_reparameterization_type
+
+        # embedding
+        self.embedding = torch.nn.Embedding(self.total_virtual_tokens, self.token_dim)
+        if not config.inference_mode:
+            if self.encoder_type == PromptEncoderReparameterizationType.LSTM:
+                lstm_dropout = config.encoder_dropout
+                num_layers = config.encoder_num_layers
+                # LSTM
+                self.lstm_head = torch.nn.LSTM(
+                    input_size=self.input_size,
+                    hidden_size=self.hidden_size,
+                    num_layers=num_layers,
+                    dropout=lstm_dropout,
+                    bidirectional=True,
+                    batch_first=True,
+                )
+
+                self.mlp_head = torch.nn.Sequential(
+                    torch.nn.Linear(self.hidden_size * 2, self.hidden_size * 2),
+                    torch.nn.ReLU(),
+                    torch.nn.Linear(self.hidden_size * 2, self.output_size),
+                )
+
+            elif self.encoder_type == PromptEncoderReparameterizationType.MLP:
+                warnings.warn(
+                    f"for {self.encoder_type}, the `encoder_num_layers` is ignored. Exactly 2 MLP layers are used."
+                )
+                layers = [
+                    torch.nn.Linear(self.input_size, self.hidden_size),
+                    torch.nn.ReLU(),
+                    torch.nn.Linear(self.hidden_size, self.hidden_size),
+                    torch.nn.ReLU(),
+                    torch.nn.Linear(self.hidden_size, self.output_size),
+                ]
+                self.mlp_head = torch.nn.Sequential(*layers)
+
+            else:
+                raise ValueError("Prompt encoder type not recognized. Please use one of MLP (recommended) or LSTM.")
+
+    def forward(self, indices):
+        input_embeds = self.embedding(indices)
+        if self.encoder_type == PromptEncoderReparameterizationType.LSTM:
+            output_embeds = self.mlp_head(self.lstm_head(input_embeds)[0])
+        elif self.encoder_type == PromptEncoderReparameterizationType.MLP:
+            output_embeds = self.mlp_head(input_embeds)
+        else:
+            raise ValueError("Prompt encoder type not recognized. Please use one of MLP (recommended) or LSTM.")
+
+        return output_embeds

model/peft/tuners/prefix_tuning.py

@@ -0,0 +1,109 @@
+# coding=utf-8
+# Copyright 2023-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from dataclasses import dataclass, field
+
+import torch
+
+from ..utils import PeftType, PromptLearningConfig
+
+
+@dataclass
+class PrefixTuningConfig(PromptLearningConfig):
+    """
+    This is the configuration class to store the configuration of a [`PrefixEncoder`].
+
+    Args:
+        encoder_hidden_size (`int`): The hidden size of the prompt encoder.
+        prefix_projection (`bool`): Whether to project the prefix embeddings.
+    """
+
+    encoder_hidden_size: int = field(
+        default=None,
+        metadata={"help": "The hidden size of the encoder"},
+    )
+    prefix_projection: bool = field(
+        default=False,
+        metadata={"help": "Whether to project the prefix tokens"},
+    )
+
+    def __post_init__(self):
+        self.peft_type = PeftType.PREFIX_TUNING
+
+
+# Based on https://github.com/THUDM/P-tuning-v2/blob/main/model/prefix_encoder.py
+# with some refactor
+class PrefixEncoder(torch.nn.Module):
+    r"""
+    The `torch.nn` model to encode the prefix.
+
+    Args:
+        config ([`PrefixTuningConfig`]): The configuration of the prefix encoder.
+
+    Example:
+
+    ```py
+    >>> from peft import PrefixEncoder, PrefixTuningConfig
+
+    >>> config = PrefixTuningConfig(
+    ...     peft_type="PREFIX_TUNING",
+    ...     task_type="SEQ_2_SEQ_LM",
+    ...     num_virtual_tokens=20,
+    ...     token_dim=768,
+    ...     num_transformer_submodules=1,
+    ...     num_attention_heads=12,
+    ...     num_layers=12,
+    ...     encoder_hidden_size=768,
+    ... )
+    >>> prefix_encoder = PrefixEncoder(config)
+    ```
+
+    **Attributes**:
+        - **embedding** (`torch.nn.Embedding`) -- The embedding layer of the prefix encoder.
+        - **transform** (`torch.nn.Sequential`) -- The two-layer MLP to transform the prefix embeddings if
+          `prefix_projection` is `True`.
+        - **prefix_projection** (`bool`) -- Whether to project the prefix embeddings.
+
+    Input shape: (`batch_size`, `num_virtual_tokens`)
+
+    Output shape: (`batch_size`, `num_virtual_tokens`, `2*layers*hidden`)
+    """
+
+    def __init__(self, config):
+        super().__init__()
+        self.prefix_projection = config.prefix_projection
+        token_dim = config.token_dim
+        num_layers = config.num_layers
+        encoder_hidden_size = config.encoder_hidden_size
+        num_virtual_tokens = config.num_virtual_tokens
+        if self.prefix_projection and not config.inference_mode:
+            # Use a two-layer MLP to encode the prefix
+            self.embedding = torch.nn.Embedding(num_virtual_tokens, token_dim)
+            self.transform = torch.nn.Sequential(
+                torch.nn.Linear(token_dim, encoder_hidden_size),
+                torch.nn.Tanh(),
+                torch.nn.Linear(encoder_hidden_size, num_layers * 2 * token_dim),
+            )
+        else:
+            self.embedding = torch.nn.Embedding(num_virtual_tokens, num_layers * 2 * token_dim)
+
+    def forward(self, prefix: torch.Tensor):
+        if self.prefix_projection:
+            prefix_tokens = self.embedding(prefix)
+            past_key_values = self.transform(prefix_tokens)
+        else:
+            past_key_values = self.embedding(prefix)
+        return past_key_values

model/peft/tuners/prompt_tuning.py

@@ -0,0 +1,130 @@
+# coding=utf-8
+# Copyright 2023-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import enum
+import math
+from dataclasses import dataclass, field
+from typing import Optional, Union
+
+import torch
+
+from ..utils import PeftType, PromptLearningConfig
+
+
+class PromptTuningInit(str, enum.Enum):
+    TEXT = "TEXT"
+    RANDOM = "RANDOM"
+
+
+@dataclass
+class PromptTuningConfig(PromptLearningConfig):
+    """
+    This is the configuration class to store the configuration of a [`PromptEmbedding`].
+
+    Args:
+        prompt_tuning_init (Union[[`PromptTuningInit`], `str`]): The initialization of the prompt embedding.
+        prompt_tuning_init_text (`str`, *optional*):
+            The text to initialize the prompt embedding. Only used if `prompt_tuning_init` is `TEXT`.
+        tokenizer_name_or_path (`str`, *optional*):
+            The name or path of the tokenizer. Only used if `prompt_tuning_init` is `TEXT`.
+    """
+
+    prompt_tuning_init: Union[PromptTuningInit, str] = field(
+        default=PromptTuningInit.RANDOM,
+        metadata={"help": "How to initialize the prompt tuning parameters"},
+    )
+    prompt_tuning_init_text: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": "The text to use for prompt tuning initialization. Only used if prompt_tuning_init is `TEXT`"
+        },
+    )
+    tokenizer_name_or_path: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": "The tokenizer to use for prompt tuning initialization. Only used if prompt_tuning_init is `TEXT`"
+        },
+    )
+
+    def __post_init__(self):
+        self.peft_type = PeftType.PROMPT_TUNING
+
+
+class PromptEmbedding(torch.nn.Module):
+    """
+    The model to encode virtual tokens into prompt embeddings.
+
+    Args:
+        config ([`PromptTuningConfig`]): The configuration of the prompt embedding.
+        word_embeddings (`torch.nn.Module`): The word embeddings of the base transformer model.
+
+    **Attributes**:
+        - **embedding** (`torch.nn.Embedding`) -- The embedding layer of the prompt embedding.
+
+    Example:
+
+    ```py
+    >>> from peft import PromptEmbedding, PromptTuningConfig
+
+    >>> config = PromptTuningConfig(
+    ...     peft_type="PROMPT_TUNING",
+    ...     task_type="SEQ_2_SEQ_LM",
+    ...     num_virtual_tokens=20,
+    ...     token_dim=768,
+    ...     num_transformer_submodules=1,
+    ...     num_attention_heads=12,
+    ...     num_layers=12,
+    ...     prompt_tuning_init="TEXT",
+    ...     prompt_tuning_init_text="Predict if sentiment of this review is positive, negative or neutral",
+    ...     tokenizer_name_or_path="t5-base",
+    ... )
+
+    >>> # t5_model.shared is the word embeddings of the base model
+    >>> prompt_embedding = PromptEmbedding(config, t5_model.shared)
+    ```
+
+    Input Shape: (`batch_size`, `total_virtual_tokens`)
+
+    Output Shape: (`batch_size`, `total_virtual_tokens`, `token_dim`)
+    """
+
+    def __init__(self, config, word_embeddings):
+        super().__init__()
+
+        total_virtual_tokens = config.num_virtual_tokens * config.num_transformer_submodules
+        self.embedding = torch.nn.Embedding(total_virtual_tokens, config.token_dim)
+        if config.prompt_tuning_init == PromptTuningInit.TEXT:
+            from transformers import AutoTokenizer
+
+            tokenizer = AutoTokenizer.from_pretrained(config.tokenizer_name_or_path)
+            init_text = config.prompt_tuning_init_text
+            init_token_ids = tokenizer(init_text)["input_ids"]
+            # Trim or iterate until num_text_tokens matches total_virtual_tokens
+            num_text_tokens = len(init_token_ids)
+            if num_text_tokens > total_virtual_tokens:
+                init_token_ids = init_token_ids[:total_virtual_tokens]
+            elif num_text_tokens < total_virtual_tokens:
+                num_reps = math.ceil(total_virtual_tokens / num_text_tokens)
+                init_token_ids = init_token_ids * num_reps
+            init_token_ids = init_token_ids[:total_virtual_tokens]
+
+            word_embedding_weights = word_embeddings(torch.LongTensor(init_token_ids)).detach().clone()
+            word_embedding_weights = word_embedding_weights.to(torch.float32)
+            self.embedding.weight = torch.nn.Parameter(word_embedding_weights)
+
+    def forward(self, indices):
+        # Just get embeddings
+        prompt_embeddings = self.embedding(indices)
+        return prompt_embeddings

model/peft/tuners/test_moelora.py

@@ -0,0 +1,97 @@
+import torch
+import unittest
+import math
+import re
+import warnings
+from dataclasses import asdict, dataclass, field
+from enum import Enum
+from typing import List, Optional, Tuple, Union
+import itertools
+import copy
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from transformers.pytorch_utils import Conv1D
+from .gating import GATING_TO_MODEL_MAPPING
+
+from ..import_utils import is_bnb_4bit_available, is_bnb_available
+from ..utils import (
+    COMMON_LAYERS_PATTERN,
+    TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPING,
+    ModulesToSaveWrapper,
+    PeftConfig,
+    PeftType,
+    _freeze_adapter,
+    _get_submodules,
+    transpose,
+)
+
+if is_bnb_available():
+    import bitsandbytes as bnb
+from moelora import *
+
+class TestMoELoRA(unittest.TestCase):
+    def setUp(self):
+        self.model = MoELoRA()  # Instantiate your MoELoRA model here
+
+    def test_forward_no_adapters(self):
+        x = torch.randn(10, 20, 30)  # Create a random input tensor
+        output = self.model.forward(x)
+        self.assertEqual(output.shape, (10, 20, 30))  # Assert the output shape is correct
+
+    def test_forward_with_adapters(self):
+        x = torch.randn(10, 20, 30)  # Create a random input tensor
+        self.model.active_adapter = 'adapter1'  # Set the active adapter
+        output = self.model.forward(x)
+        self.assertEqual(output.shape, (10, 20, 30))  # Assert the output shape is correct
+
+    def test_forward_with_global_user_embeds(self):
+        x = torch.randn(10, 20, 30)  # Create a random input tensor
+        self.model.active_adapter = 'adapter1'  # Set the active adapter
+        self.model.global_user_embeds = [torch.randn(10, 30)]  # Set the global_user_embeds
+        output = self.model.forward(x)
+        self.assertEqual(output.shape, (10, 20, 30))  # Assert the output shape is correct
+
+if __name__ == '__main__':
+    unittest.main()import torch
+import unittest
+
+class TestMoELoRA(unittest.TestCase):
+    def setUp(self):
+        self.model = MoELoRA()  # Instantiate your MoELoRA model here
+
+    def test_forward_no_adapters(self):
+        x = torch.randn(10, 20, 30)  # Create a random input tensor
+        output = self.model.forward(x)
+        self.assertEqual(output.shape, (10, 20, 30))  # Assert the output shape is correct
+
+    def test_forward_with_adapters(self):
+        x = torch.randn(10, 20, 30)  # Create a random input tensor
+        self.model.active_adapter = 'adapter1'  # Set the active adapter
+        output = self.model.forward(x)
+        self.assertEqual(output.shape, (10, 20, 30))  # Assert the output shape is correct
+
+    def test_forward_with_global_user_embeds(self):
+        x = torch.randn(10, 20, 30)  # Create a random input tensor
+        self.model.active_adapter = 'adapter1'  # Set the active adapter
+        self.model.global_user_embeds = [torch.randn(10, 30)]  # Set the global_user_embeds
+        output = self.model.forward(x)
+        self.assertEqual(output.shape, (10, 20, 30))  # Assert the output shape is correct
+
+    def test_forward_with_global_user_embeds_exception(self):
+        x = torch.randn(10, 20, 30)  # Create a random input tensor
+        self.model.active_adapter = 'adapter1'  # Set the active adapter
+        self.model.global_user_embeds = [torch.randn(5, 30)]  # Set the global_user_embeds with incompatible shape
+        output = self.model.forward(x)
+        self.assertEqual(output.shape, (10, 20, 30))  # Assert the output shape is correct
+
+    def test_forward_no_global_user_embeds(self):
+        x = torch.randn(10, 20, 30)  # Create a random input tensor
+        self.model.active_adapter = 'adapter1'  # Set the active adapter
+        self.model.global_user_embeds = []  # Set an empty global_user_embeds
+        output = self.model.forward(x)
+        self.assertEqual(output.shape, (10, 20, 30))  # Assert the output shape is correct
+
+if __name__ == '__main__':
+    unittest.main()

model/peft/utils/__init__.py

@@ -0,0 +1,42 @@
+# flake8: noqa
+# There's no way to ignore "F401 '...' imported but unused" warnings in this
+# module, but to preserve other warnings. So, don't check this module at all
+
+# coding=utf-8
+# Copyright 2023-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .config import PeftConfig, PeftType, PromptLearningConfig, TaskType
+from .other import (
+    TRANSFORMERS_MODELS_TO_PREFIX_TUNING_POSTPROCESS_MAPPING,
+    TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPING,
+    TRANSFORMERS_MODELS_TO_ADALORA_TARGET_MODULES_MAPPING,
+    COMMON_LAYERS_PATTERN,
+    CONFIG_NAME,
+    WEIGHTS_NAME,
+    SAFETENSORS_WEIGHTS_NAME,
+    _set_trainable,
+    add_library_to_model_card,
+    bloom_model_postprocess_past_key_value,
+    prepare_model_for_int8_training,
+    prepare_model_for_kbit_training,
+    shift_tokens_right,
+    transpose,
+    _get_submodules,
+    _set_adapter,
+    _freeze_adapter,
+    ModulesToSaveWrapper,
+)
+from .hub_utils import hub_file_exists
+from .save_and_load import get_peft_model_state_dict, set_peft_model_state_dict

model/peft/utils/config.py

@@ -0,0 +1,222 @@
+# coding=utf-8
+# Copyright 2023-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import enum
+import inspect
+import json
+import os
+from dataclasses import asdict, dataclass, field
+from typing import Optional, Union
+
+from huggingface_hub import hf_hub_download
+from transformers.utils import PushToHubMixin
+
+from .other import CONFIG_NAME
+
+
+class PeftType(str, enum.Enum):
+    PROMPT_TUNING = "PROMPT_TUNING"
+    P_TUNING = "P_TUNING"
+    PREFIX_TUNING = "PREFIX_TUNING"
+    LORA = "LORA"
+    ADALORA = "ADALORA"
+    ADAPTION_PROMPT = "ADAPTION_PROMPT"
+    MOELORA = "MOELORA"
+
+
+class TaskType(str, enum.Enum):
+    SEQ_CLS = "SEQ_CLS"
+    SEQ_2_SEQ_LM = "SEQ_2_SEQ_LM"
+    CAUSAL_LM = "CAUSAL_LM"
+    TOKEN_CLS = "TOKEN_CLS"
+    QUESTION_ANS = "QUESTION_ANS"
+
+
+@dataclass
+class PeftConfigMixin(PushToHubMixin):
+    r"""
+    This is the base configuration class for PEFT adapter models. It contains all the methods that are common to all
+    PEFT adapter models. This class inherits from [`~transformers.utils.PushToHubMixin`] which contains the methods to
+    push your model to the Hub. The method `save_pretrained` will save the configuration of your adapter model in a
+    directory. The method `from_pretrained` will load the configuration of your adapter model from a directory.
+
+    Args:
+        peft_type (Union[[`~peft.utils.config.PeftType`], `str`]): The type of Peft method to use.
+    """
+    peft_type: Optional[PeftType] = field(default=None, metadata={"help": "The type of PEFT model."})
+
+    @property
+    def __dict__(self):
+        return asdict(self)
+
+    def to_dict(self):
+        return self.__dict__
+
+    def save_pretrained(self, save_directory, **kwargs):
+        r"""
+        This method saves the configuration of your adapter model in a directory.
+
+        Args:
+            save_directory (`str`):
+                The directory where the configuration will be saved.
+            kwargs (additional keyword arguments, *optional*):
+                Additional keyword arguments passed along to the [`~transformers.utils.PushToHubMixin.push_to_hub`]
+                method.
+        """
+        if os.path.isfile(save_directory):
+            raise AssertionError(f"Provided path ({save_directory}) should be a directory, not a file")
+
+        os.makedirs(save_directory, exist_ok=True)
+
+        output_dict = self.__dict__
+        output_path = os.path.join(save_directory, CONFIG_NAME)
+
+        # save it
+        with open(output_path, "w") as writer:
+            writer.write(json.dumps(output_dict, indent=2, sort_keys=True))
+
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path, subfolder=None, **kwargs):
+        r"""
+        This method loads the configuration of your adapter model from a directory.
+
+        Args:
+            pretrained_model_name_or_path (`str`):
+                The directory or the Hub repository id where the configuration is saved.
+            kwargs (additional keyword arguments, *optional*):
+                Additional keyword arguments passed along to the child class initialization.
+        """
+        path = (
+            os.path.join(pretrained_model_name_or_path, subfolder)
+            if subfolder is not None
+            else pretrained_model_name_or_path
+        )
+
+        hf_hub_download_kwargs, class_kwargs, other_kwargs = cls._split_kwargs(kwargs)
+
+        if os.path.isfile(os.path.join(path, CONFIG_NAME)):
+            config_file = os.path.join(path, CONFIG_NAME)
+        else:
+            try:
+                config_file = hf_hub_download(
+                    pretrained_model_name_or_path, CONFIG_NAME, subfolder=subfolder, **hf_hub_download_kwargs
+                )
+            except Exception:
+                raise ValueError(f"Can't find '{CONFIG_NAME}' at '{pretrained_model_name_or_path}'")
+
+        loaded_attributes = cls.from_json_file(config_file)
+
+        config = cls(**class_kwargs)
+
+        for key, value in loaded_attributes.items():
+            if hasattr(config, key):
+                setattr(config, key, value)
+
+        return config
+
+    @classmethod
+    def from_json_file(cls, path_json_file, **kwargs):
+        r"""
+        Loads a configuration file from a json file.
+
+        Args:
+            path_json_file (`str`):
+                The path to the json file.
+        """
+        with open(path_json_file, "r") as file:
+            json_object = json.load(file)
+
+        return json_object
+
+    @classmethod
+    def _split_kwargs(cls, kwargs):
+        hf_hub_download_kwargs = {}
+        class_kwargs = {}
+        other_kwargs = {}
+
+        for key, value in kwargs.items():
+            if key in inspect.signature(hf_hub_download).parameters:
+                hf_hub_download_kwargs[key] = value
+            elif key in list(cls.__annotations__):
+                class_kwargs[key] = value
+            else:
+                other_kwargs[key] = value
+
+        return hf_hub_download_kwargs, class_kwargs, other_kwargs
+
+    @classmethod
+    def _get_peft_type(
+        cls,
+        model_id,
+        subfolder: Optional[str] = None,
+        revision: Optional[str] = None,
+        cache_dir: Optional[str] = None,
+    ):
+        path = os.path.join(model_id, subfolder) if subfolder is not None else model_id
+
+        if os.path.isfile(os.path.join(path, CONFIG_NAME)):
+            config_file = os.path.join(path, CONFIG_NAME)
+        else:
+            try:
+                config_file = hf_hub_download(
+                    model_id, CONFIG_NAME, subfolder=subfolder, revision=revision, cache_dir=cache_dir
+                )
+            except Exception:
+                raise ValueError(f"Can't find '{CONFIG_NAME}' at '{model_id}'")
+
+        loaded_attributes = cls.from_json_file(config_file)
+        return loaded_attributes["peft_type"]
+
+
+@dataclass
+class PeftConfig(PeftConfigMixin):
+    """
+    This is the base configuration class to store the configuration of a [`PeftModel`].
+
+    Args:
+        peft_type (Union[[`~peft.utils.config.PeftType`], `str`]): The type of Peft method to use.
+        task_type (Union[[`~peft.utils.config.TaskType`], `str`]): The type of task to perform.
+        inference_mode (`bool`, defaults to `False`): Whether to use the Peft model in inference mode.
+    """
+
+    base_model_name_or_path: str = field(default=None, metadata={"help": "The name of the base model to use."})
+    revision: str = field(default=None, metadata={"help": "The specific model version to use."})
+    peft_type: Union[str, PeftType] = field(default=None, metadata={"help": "Peft type"})
+    task_type: Union[str, TaskType] = field(default=None, metadata={"help": "Task type"})
+    inference_mode: bool = field(default=False, metadata={"help": "Whether to use inference mode"})
+
+
+@dataclass
+class PromptLearningConfig(PeftConfig):
+    """
+    This is the base configuration class to store the configuration of [`PrefixTuning`], [`PromptEncoder`], or
+    [`PromptTuning`].
+
+    Args:
+        num_virtual_tokens (`int`): The number of virtual tokens to use.
+        token_dim (`int`): The hidden embedding dimension of the base transformer model.
+        num_transformer_submodules (`int`): The number of transformer submodules in the base transformer model.
+        num_attention_heads (`int`): The number of attention heads in the base transformer model.
+        num_layers (`int`): The number of layers in the base transformer model.
+    """
+
+    num_virtual_tokens: int = field(default=None, metadata={"help": "Number of virtual tokens"})
+    token_dim: int = field(
+        default=None, metadata={"help": "The hidden embedding dimension of the base transformer model"}
+    )
+    num_transformer_submodules: Optional[int] = field(
+        default=None, metadata={"help": "Number of transformer submodules"}
+    )
+    num_attention_heads: Optional[int] = field(default=None, metadata={"help": "Number of attention heads"})
+    num_layers: Optional[int] = field(default=None, metadata={"help": "Number of transformer layers"})

model/peft/utils/hub_utils.py

@@ -0,0 +1,29 @@
+# coding=utf-8
+# Copyright 2023-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from huggingface_hub import get_hf_file_metadata, hf_hub_url
+from huggingface_hub.utils import EntryNotFoundError
+
+
+def hub_file_exists(repo_id: str, filename: str, revision: str = None, repo_type: str = None) -> bool:
+    r"""
+    Checks if a file exists in a remote Hub repository.
+    """
+    url = hf_hub_url(repo_id=repo_id, filename=filename, repo_type=repo_type, revision=revision)
+    try:
+        get_hf_file_metadata(url)
+        return True
+    except EntryNotFoundError:
+        return False

model/peft/utils/other.py

@@ -0,0 +1,271 @@
+# coding=utf-8
+# Copyright 2023-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import copy
+import os
+import warnings
+
+import torch
+
+
+# Add or edit model card to have `library_name: peft`
+def add_library_to_model_card(output_dir):
+    if os.path.exists(os.path.join(output_dir, "README.md")):
+        with open(os.path.join(output_dir, "README.md"), "r") as f:
+            lines = f.readlines()
+        # check if the first line is `---`
+        if len(lines) > 0 and lines[0].startswith("---"):
+            for i, line in enumerate(lines[1:]):
+                # check if line starts with `library_name`, if yes, update it
+                if line.startswith("library_name"):
+                    lines[i + 1] = "library_name: peft\n"
+                    break
+                elif line.startswith("---"):
+                    # insert `library_name: peft` before the last `---`
+                    lines.insert(i + 1, "library_name: peft\n")
+                    break
+        else:
+            lines = ["---\n", "library_name: peft\n", "---\n"] + lines
+    else:
+        lines = ["---\n", "library_name: peft\n", "---\n"]
+    # write the lines back to README.md
+    with open(os.path.join(output_dir, "README.md"), "w") as f:
+        f.writelines(lines)
+
+
+# needed for prefix-tuning of bloom model
+def bloom_model_postprocess_past_key_value(past_key_values):
+    past_key_values = torch.cat(past_key_values)
+    total_layers, batch_size, num_attention_heads, num_virtual_tokens, head_dim = past_key_values.shape
+    keys = past_key_values[: total_layers // 2]
+    keys = keys.transpose(2, 3).reshape(
+        total_layers // 2, batch_size * num_attention_heads, head_dim, num_virtual_tokens
+    )
+    values = past_key_values[total_layers // 2 :]
+    values = values.reshape(total_layers // 2, batch_size * num_attention_heads, num_virtual_tokens, head_dim)
+
+    return tuple(zip(keys, values))
+
+
+def prepare_model_for_kbit_training(model, use_gradient_checkpointing=True):
+    r"""
+    This method wraps the entire protocol for preparing a model before running a training. This includes:
+        1- Cast the layernorm in fp32 2- making output embedding layer require grads 3- Add the upcasting of the lm
+        head to fp32
+
+    Args:
+        model, (`transformers.PreTrainedModel`):
+            The loaded model from `transformers`
+    """
+    loaded_in_kbit = getattr(model, "is_loaded_in_8bit", False) or getattr(model, "is_loaded_in_4bit", False)
+
+    for name, param in model.named_parameters():
+        # freeze base model's layers
+        param.requires_grad = False
+
+    # cast all non INT8 parameters to fp32
+    for param in model.parameters():
+        if (param.dtype == torch.float16) or (param.dtype == torch.bfloat16):
+            param.data = param.data.to(torch.float32)
+
+    if loaded_in_kbit and use_gradient_checkpointing:
+        # For backward compatibility
+        if hasattr(model, "enable_input_require_grads"):
+            model.enable_input_require_grads()
+        else:
+
+            def make_inputs_require_grad(module, input, output):
+                output.requires_grad_(True)
+
+            model.get_input_embeddings().register_forward_hook(make_inputs_require_grad)
+
+        # enable gradient checkpointing for memory efficiency
+        model.gradient_checkpointing_enable()
+
+    return model
+
+
+# For backward compatibility
+def prepare_model_for_int8_training(*args, **kwargs):
+    warnings.warn(
+        "prepare_model_for_int8_training is deprecated and will be removed in a future version. Use prepare_model_for_kbit_training instead.",
+        FutureWarning,
+    )
+    return prepare_model_for_kbit_training(*args, **kwargs)
+
+
+# copied from transformers.models.bart.modeling_bart
+def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start_token_id: int):
+    """
+    Shift input ids one token to the right.
+
+    Args:
+        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): input ids
+        pad_token_id (`int`): The id of the `padding` token.
+        decoder_start_token_id (`int`): The id of the `start` token.
+    """
+    shifted_input_ids = input_ids.new_zeros(input_ids.shape)
+    shifted_input_ids[:, 1:] = input_ids[:, :-1].clone()
+    shifted_input_ids[:, 0] = decoder_start_token_id
+
+    if pad_token_id is None:
+        raise ValueError("self.model.config.pad_token_id has to be defined.")
+    # replace possible -100 values in labels by `pad_token_id`
+    shifted_input_ids.masked_fill_(shifted_input_ids == -100, pad_token_id)
+
+    return shifted_input_ids
+
+
+class ModulesToSaveWrapper(torch.nn.Module):
+    def __init__(self, module_to_save, adapter_name):
+        super().__init__()
+        self.original_module = module_to_save
+        self.modules_to_save = torch.nn.ModuleDict({})
+        self.update(adapter_name)
+        self.active_adapter = adapter_name
+
+    def update(self, adapter_name):
+        self.modules_to_save.update(torch.nn.ModuleDict({adapter_name: copy.deepcopy(self.original_module)}))
+
+    def forward(self, *args, **kwargs):
+        if self.active_adapter not in self.modules_to_save:
+            return self.original_module(*args, **kwargs)
+        return self.modules_to_save[self.active_adapter](*args, **kwargs)
+
+
+def _get_submodules(model, key):
+    parent = model.get_submodule(".".join(key.split(".")[:-1]))
+    target_name = key.split(".")[-1]
+    target = model.get_submodule(key)
+    return parent, target, target_name
+
+
+def _freeze_adapter(model, adapter_name):
+    for n, p in model.named_parameters():
+        if adapter_name in n:
+            p.requires_grad = False
+
+
+def _set_trainable(model, adapter_name):
+    key_list = [key for key, _ in model.named_modules()]
+    for key in key_list:
+        target_module_found = any(key.endswith(target_key) for target_key in model.modules_to_save)
+        if target_module_found:
+            parent, target, target_name = _get_submodules(model, key)
+            if isinstance(target, ModulesToSaveWrapper):
+                target.update(adapter_name)
+            else:
+                for param in target.parameters():
+                    param.requires_grad = True
+                setattr(parent, target_name, ModulesToSaveWrapper(target, adapter_name))
+
+
+def _set_adapter(model, adapter_name):
+    for module in model.modules():
+        if isinstance(module, ModulesToSaveWrapper):
+            module.active_adapter = adapter_name
+
+
+def fsdp_auto_wrap_policy(model):
+    import functools
+    import os
+
+    from accelerate import FullyShardedDataParallelPlugin
+    from torch.distributed.fsdp.wrap import _or_policy, lambda_auto_wrap_policy, transformer_auto_wrap_policy
+
+    from ..tuners import PrefixEncoder, PromptEmbedding, PromptEncoder
+
+    def lambda_policy_fn(module):
+        if (
+            len(list(module.named_children())) == 0
+            and getattr(module, "weight", None) is not None
+            and module.weight.requires_grad
+        ):
+            return True
+        return False
+
+    lambda_policy = functools.partial(lambda_auto_wrap_policy, lambda_fn=lambda_policy_fn)
+    transformer_wrap_policy = functools.partial(
+        transformer_auto_wrap_policy,
+        transformer_layer_cls=(
+            PrefixEncoder,
+            PromptEncoder,
+            PromptEmbedding,
+            FullyShardedDataParallelPlugin.get_module_class_from_name(
+                model, os.environ.get("FSDP_TRANSFORMER_CLS_TO_WRAP", "")
+            ),
+        ),
+    )
+
+    auto_wrap_policy = functools.partial(_or_policy, policies=[lambda_policy, transformer_wrap_policy])
+    return auto_wrap_policy
+
+
+def transpose(weight, fan_in_fan_out):
+    return weight.T if fan_in_fan_out else weight
+
+
+TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPING = {
+    "t5": ["q", "v"],
+    "mt5": ["q", "v"],
+    "bart": ["q_proj", "v_proj"],
+    "gpt2": ["c_attn"],
+    "bloom": ["query_key_value"],
+    "blip-2": ["q", "v", "q_proj", "v_proj"],
+    "opt": ["q_proj", "v_proj"],
+    "gptj": ["q_proj", "v_proj"],
+    "gpt_neox": ["query_key_value"],
+    "gpt_neo": ["q_proj", "v_proj"],
+    "bert": ["query", "value"],
+    "roberta": ["query", "value"],
+    "xlm-roberta": ["query", "value"],
+    "electra": ["query", "value"],
+    "deberta-v2": ["query_proj", "value_proj"],
+    "deberta": ["in_proj"],
+    "layoutlm": ["query", "value"],
+    "llama": ["q_proj", "v_proj"],
+    "chatglm": ["query_key_value"],
+    "gpt_bigcode": ["c_attn"],
+    "mpt": ["Wqkv"],
+}
+
+COMMON_LAYERS_PATTERN = ["layers", "h", "block", "blocks"]
+
+TRANSFORMERS_MODELS_TO_ADALORA_TARGET_MODULES_MAPPING = {
+    "t5": ["q", "k", "v", "o", "wi", "wo"],
+    "mt5": ["q", "k", "v", "o", "wi_0", "wi_1", "wo"],
+    "bart": ["q_proj", "k_proj", "v_proj", "out_proj", "fc1", "fc2"],
+    # "gpt2": ["c_attn"],
+    # "bloom": ["query_key_value"],
+    "opt": ["q_proj", "k_proj", "v_proj", "out_proj", "fc1", "fc2"],
+    # "gptj": ["q_proj", "v_proj"],
+    # "gpt_neox": ["query_key_value"],
+    # "gpt_neo": ["q_proj", "v_proj"],
+    # "bert": ["query", "value"],
+    "roberta": ["query", "key", "value", "dense"],
+    # "xlm-roberta": ["query", "value"],
+    # "electra": ["query", "value"],
+    "deberta-v2": ["query_proj", "key_proj", "value_proj", "dense"],
+    # "deberta": ["in_proj"],
+    # "layoutlm": ["query", "value"],
+}
+
+TRANSFORMERS_MODELS_TO_PREFIX_TUNING_POSTPROCESS_MAPPING = {
+    "bloom": bloom_model_postprocess_past_key_value,
+}
+
+WEIGHTS_NAME = "adapter_model.bin"
+SAFETENSORS_WEIGHTS_NAME = "adapter_model.safetensors"
+CONFIG_NAME = "adapter_config.json"

model/peft/utils/save_and_load.py

@@ -0,0 +1,137 @@
+# coding=utf-8
+# Copyright 2023-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .config import PeftType, PromptLearningConfig
+
+
+def get_peft_model_state_dict(model, state_dict=None, adapter_name="default"):
+    """
+    Get the state dict of the Peft model.
+
+    Args:
+        model ([`PeftModel`]): The Peft model. When using torch.nn.DistributedDataParallel, DeepSpeed or FSDP,
+        the model should be the underlying model/unwrapped model (i.e. model.module).
+        state_dict (`dict`, *optional*, defaults to `None`):
+            The state dict of the model. If not provided, the state dict of the model
+        will be used.
+    """
+    config = model.peft_config[adapter_name]
+    if state_dict is None:
+        state_dict = model.state_dict()
+    if config.peft_type in (PeftType.LORA, PeftType.ADALORA, PeftType.MOELORA):
+        # to_return = lora_state_dict(model, bias=model.peft_config.bias)
+        # adapted from `https://github.com/microsoft/LoRA/blob/main/loralib/utils.py`
+        # to be used directly with the state dict which is necessary when using DeepSpeed or FSDP
+        bias = config.bias
+        if bias == "none":
+            to_return = {k: state_dict[k] for k in state_dict if "lora_" in k or "gating" in k}
+        elif bias == "all":
+            to_return = {k: state_dict[k] for k in state_dict if "lora_" in k or "bias" in k or "gating" in k}
+        elif bias == "lora_only":
+            to_return = {}
+            for k in state_dict:
+                if "lora_" in k:
+                    to_return[k] = state_dict[k]
+                    bias_name = k.split("lora_")[0] + "bias"
+                    if bias_name in state_dict:
+                        to_return[bias_name] = state_dict[bias_name]
+                if "gating" in k:
+                    to_return[k] = state_dict[k]
+        else:
+            raise NotImplementedError
+        to_return = {k: v for k, v in to_return.items() if (("lora_" in k and adapter_name in k) or ("bias" in k) or ("gating" in k))}
+        if config.peft_type == PeftType.ADALORA:
+            rank_pattern = config.rank_pattern
+            if rank_pattern is not None:
+                rank_pattern = {k.replace(f".{adapter_name}", ""): v for k, v in rank_pattern.items()}
+                config.rank_pattern = rank_pattern
+                to_return = model.resize_state_dict_by_rank_pattern(rank_pattern, to_return, adapter_name)
+
+    elif config.peft_type == PeftType.ADAPTION_PROMPT:
+        to_return = {k: state_dict[k] for k in state_dict if k.split(".")[-1].startswith("adaption_")}
+    elif isinstance(config, PromptLearningConfig):
+        to_return = {}
+        if config.inference_mode:
+            prompt_embeddings = model.prompt_encoder[adapter_name].embedding.weight
+        else:
+            prompt_embeddings = model.get_prompt_embedding_to_save(adapter_name)
+        to_return["prompt_embeddings"] = prompt_embeddings
+    else:
+        raise NotImplementedError
+    if model.modules_to_save is not None:
+        for key, value in state_dict.items():
+            if any(f"{module_name}.modules_to_save.{adapter_name}" in key for module_name in model.modules_to_save):
+                to_return[key.replace("modules_to_save.", "")] = value
+
+    to_return = {k.replace(f".{adapter_name}", ""): v for k, v in to_return.items()}
+    return to_return
+
+
+def set_peft_model_state_dict(model, peft_model_state_dict, adapter_name="default"):
+    """
+    Set the state dict of the Peft model.
+
+    Args:
+        model ([`PeftModel`]): The Peft model.
+        peft_model_state_dict (`dict`): The state dict of the Peft model.
+    """
+    config = model.peft_config[adapter_name]
+    state_dict = {}
+    if model.modules_to_save is not None:
+        for key, value in peft_model_state_dict.items():
+            if any(module_name in key for module_name in model.modules_to_save):
+                for module_name in model.modules_to_save:
+                    if module_name in key:
+                        key = key.replace(module_name, f"{module_name}.modules_to_save.{adapter_name}")
+                        break
+            state_dict[key] = value
+    else:
+        state_dict = peft_model_state_dict
+
+    if config.peft_type in (PeftType.LORA, PeftType.ADALORA, PeftType.MOELORA):
+        peft_model_state_dict = {}
+        for k, v in state_dict.items():
+            if "lora_A" in k:
+                k = k.replace("lora_A", f"lora_A.{adapter_name}")
+                peft_model_state_dict[k] = v
+                # suffix = k.split("lora_")[1]
+                # if "." in suffix:
+                #     suffix_to_replace = ".".join(suffix.split(".")[1:])
+                #     k = k.replace(suffix_to_replace, f"{adapter_name}.{suffix_to_replace}")
+                # else:
+                #     k = f"{k}.{adapter_name}"
+                # peft_model_state_dict[k] = v
+            elif "lora_B" in k:
+                k = k.replace("lora_B", f"lora_B.{adapter_name}")
+                peft_model_state_dict[k] = v
+            elif "gating" in k:
+                k = k.replace("gating", f"gating.{adapter_name}")
+                peft_model_state_dict[k] = v
+            else:
+                peft_model_state_dict[k] = v
+        if config.peft_type == PeftType.ADALORA:
+            rank_pattern = config.rank_pattern
+            if rank_pattern is not None:
+                model.resize_modules_by_rank_pattern(rank_pattern, adapter_name)
+    elif isinstance(config, PromptLearningConfig) or config.peft_type == PeftType.ADAPTION_PROMPT:
+        peft_model_state_dict = state_dict
+    else:
+        raise NotImplementedError
+    load_result = model.load_state_dict(peft_model_state_dict, strict=False)
+    if isinstance(config, PromptLearningConfig):
+        model.prompt_encoder[adapter_name].embedding.load_state_dict(
+            {"weight": peft_model_state_dict["prompt_embeddings"]}, strict=True
+        )
+    return load_result

model/router/cvr.py

@@ -0,0 +1,73 @@
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+class LambdaLayer(nn.Module):
+    def __init__(self, lambd):
+        super(LambdaLayer, self).__init__()
+        self.lambd = lambd
+
+    def forward(self, x):
+        return self.lambd(x)
+
+
+class Block(nn.Module):
+    expansion = 1
+
+    def __init__(self, in_planes, planes, conv_layer, stride=1):
+        super(Block, self).__init__()
+        self.conv1 = conv_layer(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)
+        self.bn1 = nn.BatchNorm2d(planes)
+        self.conv2 = conv_layer(planes, planes, kernel_size=3, stride=1, padding=1, bias=False)
+        self.bn2 = nn.BatchNorm2d(planes)
+
+        self.shortcut = nn.Sequential()
+        if stride != 1 or in_planes != planes:
+            diff = planes - in_planes
+            self.shortcut = LambdaLayer(
+                lambda x: F.pad(x[:, :, ::2, ::2], (0, 0, 0, 0, int(diff * 0.5), int((diff + 1) * 0.5)), "constant", 0))
+    def forward(self, x):
+        out = F.relu(self.bn1(self.conv1(x)))
+        out = self.bn2(self.conv2(out))
+        out += self.shortcut(x)
+        out = F.relu(out)
+        return out
+
+
+class Router(nn.Module):
+    def __init__(self, block, num_blocks, num_experts=2):
+        super(Router, self).__init__()
+        self.in_planes = 16
+        self.conv_layer = nn.Conv2d
+
+        self.conv1 = nn.Conv2d(3, self.in_planes, kernel_size=3, stride=1, padding=1, bias=False)
+        self.bn1 = nn.BatchNorm2d(self.in_planes)
+        self.layer1 = self._make_layer(block, 16, num_blocks[0], stride=1)
+        self.layer2 = self._make_layer(block, 32, num_blocks[1], stride=2)
+        self.layer3 = self._make_layer(block, 64, num_blocks[2], stride=2)
+        self.fc = nn.Linear(64, num_experts)
+        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))
+
+    def _make_layer(self, block, planes, num_blocks, stride):
+        planes = planes
+        strides = [stride] + [1] * (num_blocks - 1)
+        layers = []
+        for stride in strides:
+            layers.append(block(self.in_planes, planes, self.conv_layer, stride))
+            self.in_planes = planes * block.expansion
+
+        return nn.Sequential(*layers)
+
+    def forward(self, x):
+        out = F.relu(self.bn1(self.conv1(x)))
+        out = self.layer1(out)
+        out = self.layer2(out)
+        out = self.layer3(out)
+        out = self.avgpool(out)
+        out = out.view(out.size(0), -1)
+        out = self.fc(out)
+        return out
+
+
+def build_router(**kwargs):
+    return Router(Block, [3, 3, 3], **kwargs)

model/router/nlpr.py

@@ -0,0 +1,84 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+class LambdaLayer(nn.Module):
+    def __init__(self, lambd):
+        super(LambdaLayer, self).__init__()
+        self.lambd = lambd
+
+    def forward(self, x):
+        return self.lambd(x)
+
+
+class ResidualBlock(nn.Module):
+    expansion = 1
+    
+    def __init__(self, in_planes, planes, conv_layer, stride=1):
+        super(ResidualBlock, self).__init__()
+        self.conv1 = conv_layer(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)
+        self.bn1 = nn.BatchNorm1d(planes)
+        self.conv2 = conv_layer(planes, planes, kernel_size=3, stride=1, padding=1, bias=False)
+        self.bn2 = nn.BatchNorm1d(planes)
+
+        self.shortcut = nn.Sequential()
+        if stride != 1 or in_planes != planes:
+            diff = planes - in_planes
+            self.shortcut = LambdaLayer(
+                lambda x: F.pad(x[:, :, ::2], (0, 0, int(diff * 0.5), int((diff + 1) * 0.5)), "constant", 0))
+
+    def forward(self, x):
+        out = F.relu(self.bn1(self.conv1(x)))
+        out = self.bn2(self.conv2(out))
+        out += self.shortcut(x)
+        out = F.relu(out)
+        return out
+
+
+class GateFunction(nn.Module):
+    def __init__(self, input_size, output_size):
+        super(GateFunction, self).__init__()
+        self.fc = nn.Linear(input_size, output_size)
+
+    def forward(self, x):
+        return F.softmax(self.fc(x), dim=-1)
+
+
+class NLPRecommendationRouter(nn.Module):
+    def __init__(self, block, num_blocks, input_size=64, num_experts=4):
+        super(NLPRecommendationRouter, self).__init__()
+        self.in_planes = 16
+        self.conv_layer = nn.Conv1d
+
+        self.conv1 = nn.Conv1d(1, self.in_planes, kernel_size=3, stride=1, padding=1, bias=False)
+        self.bn1 = nn.BatchNorm1d(self.in_planes)
+        self.layer1 = self._make_layer(block, 16, num_blocks[0], stride=1)
+        self.layer2 = self._make_layer(block, 32, num_blocks[1], stride=2)
+        self.layer3 = self._make_layer(block, 64, num_blocks[2], stride=2)
+        self.avgpool = nn.AdaptiveAvgPool1d(1)
+
+        # Gate function
+        self.gate = GateFunction(input_size, num_experts)
+
+    def _make_layer(self, block, planes, num_blocks, stride):
+        strides = [stride] + [1] * (num_blocks - 1)
+        layers = []
+        for stride in strides:
+            layers.append(block(self.in_planes, planes, self.conv_layer, stride))
+            self.in_planes = planes * block.expansion
+        return nn.Sequential(*layers)
+
+    def forward(self, x):
+        out = F.relu(self.bn1(self.conv1(x)))
+        out = self.layer1(out)
+        out = self.layer2(out)
+        out = self.layer3(out)
+        out = self.avgpool(out)
+        out = out.view(out.size(0), -1)
+        out = self.gate(out)
+        return out.unsqueeze(1)
+
+
+def build_router(**kwargs):
+    return NLPRecommendationRouter(ResidualBlock, [3, 3, 3], input_size=64, num_experts=4)

optims.py

@@ -0,0 +1,43 @@
+import math
+
+class LinearWarmupCosineLRScheduler:
+    def __init__(
+        self,
+        optimizer,
+        min_lr_list,
+        init_lr_list,
+        warmup_steps=0,
+        warmup_start_lr_list=None,
+        **kwargs
+    ):
+        self.optimizer = optimizer
+
+        self.min_lr_list = min_lr_list
+        self.init_lr_list = init_lr_list
+        self.warmup_steps = warmup_steps
+        self.warmup_start_lr_list = warmup_start_lr_list if warmup_start_lr_list is not None else init_lr_list
+
+    def step(self, cur_step, cur_epoch, max_step):
+        for i, param_group in enumerate(self.optimizer.param_groups):
+            if cur_epoch == 0 and cur_step < self.warmup_steps:
+                lr = self.warmup_lr_schedule(cur_step, self.warmup_start_lr_list[i], self.init_lr_list[i])
+            else:
+                lr = self.cosine_lr_schedule(cur_step - self.warmup_steps, max_step - self.warmup_steps, self.init_lr_list[i], self.min_lr_list[i])
+            param_group["lr"] = lr
+            
+    def cosine_lr_schedule(self, step, max_step, init_lr, min_lr):
+        """Decay the learning rate using cosine schedule"""
+        lr = (init_lr - min_lr) * 0.5 * (1 + math.cos(math.pi * step / max_step)) + min_lr
+        return lr
+
+    def warmup_lr_schedule(self, step, init_lr, max_lr):
+        """Warmup the learning rate"""
+        lr = min(max_lr, init_lr + (max_lr - init_lr) * step / max(self.warmup_steps, 1))
+        return lr
+
+    def state_dict(self):
+        return {key: value for key, value in self.__dict__.items() if key != 'optimizer'}
+
+    def load_state_dict(self, state_dict):
+        self.__dict__.update(state_dict)
+