Upload 4 files

model/frequential.py

@@ -0,0 +1,45 @@
+import torch
+import torch.nn as nn
+import numpy as np
+
+class FreqNetwork(nn.Module):
+    def __init__(self,
+                 tfidf_input_dim,
+                 tfidf_output_dim,
+                 tfidf_hidden_dim,
+                 n_layers,
+                 skip_in=(4,),
+                 weight_norm=True):
+        super(FreqNetwork, self).__init__()
+        dims = [tfidf_input_dim] + [tfidf_hidden_dim for _ in range(n_layers)] + [tfidf_output_dim]
+        self.num_layers = n_layers
+        self.skip_in = skip_in
+        for l in range(0, self.num_layers+1):
+            if l+1 in self.skip_in:
+                out_dim = dims[l + 1] + dims[0]
+                dims[l + 1] = out_dim
+            else:
+                out_dim = dims[l + 1]
+            lin = nn.Linear(dims[l], out_dim)
+            if weight_norm:
+                lin = nn.utils.weight_norm(lin)
+            setattr(self, "lin" + str(l), lin)
+        self.activation = nn.ReLU
+
+    def forward(self, inputs):
+        x = inputs
+        for l in range(0, self.num_layers+1):
+            lin = getattr(self, "lin" + str(l))
+
+            if l in self.skip_in:
+                x = torch.cat([x, inputs], 1) / np.sqrt(2)
+
+            x = lin(x)
+
+            if l < self.num_layers:
+                x = self.activation()(x)
+
+            # x = torch.dropout(x, p=0.2, train=self.training)
+
+
+        return x

model/network.py

@@ -0,0 +1,50 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from sklearn.linear_model import LogisticRegression
+import numpy as np
+
+
+
+class Classifier(nn.Module):
+    def __init__(self,
+                 combined_input,
+                 combined_dim,
+                 num_classes,
+                 n_layers,
+                 skip_in=(4,),
+                 weight_norm=True):
+        super(Classifier, self).__init__()
+        self.num_layers = n_layers
+        self.skip_in = skip_in
+        self.model = LogisticRegression()
+        # Combined classification layers
+        dims = [combined_input] + [combined_dim for _ in range(n_layers)] + [num_classes]
+        for l in range(0, self.num_layers + 1):
+            if l+1 in self.skip_in:
+                out_dim = dims[l + 1] + dims[0]
+                dims[l + 1] = out_dim
+            else:
+                out_dim = dims[l + 1]
+            lin = nn.Linear(dims[l], out_dim)
+            if weight_norm:
+                lin = nn.utils.weight_norm(lin)
+            setattr(self, "lin" + str(l), lin)
+        self.activation = nn.ReLU
+
+    def forward(self, inputs):
+
+        x = inputs
+        for l in range(0, self.num_layers + 1):
+            lin = getattr(self, "lin" + str(l))
+
+            if l+1 in self.skip_in:
+                x = torch.cat([x, inputs], 1) / np.sqrt(2)
+            x = lin(x)
+
+            if l < self.num_layers:
+                x = self.activation()(x)
+
+        # x = torch.dropout(x, p=0.2, train=self.training)
+        # Output layer
+        return x

model/positional.py

@@ -0,0 +1,41 @@
+import torch
+import torch.nn as nn
+import numpy as np
+from model.siren import SIREN
+
+class PosNetwork(nn.Module):
+    def __init__(self,
+                 input_dim,
+                 output_dim,
+                 hidden_dim,
+                 n_layers,
+                 skip_in=(4,),
+                 weight_norm=True):
+        super(PosNetwork, self).__init__()
+        dims = [input_dim] + [hidden_dim for _ in range(n_layers)] + [output_dim]
+        self.num_layers = n_layers
+        self.skip_in = skip_in
+        self.siren_layers = nn.ModuleList()
+        for l in range(0, self.num_layers + 1):
+            if l + 1 in self.skip_in:
+                out_dim = dims[l + 1] + dims[0]
+                dims[l + 1] = out_dim
+            else:
+                out_dim = dims[l + 1]
+            lin = nn.Linear(dims[l], out_dim)
+            if weight_norm:
+                lin = nn.utils.weight_norm(lin)
+            setattr(self, "lin" + str(l), lin)
+        self.activation = nn.ReLU
+
+    def forward(self, inputs):
+        x = inputs
+        for l in range(0, self.num_layers + 1):
+            lin = getattr(self, "lin" + str(l))
+            if l + 1 in self.skip_in:
+                x = torch.cat([x, inputs], 1) / np.sqrt(2)
+            x = lin(x)
+            if l < self.num_layers:
+                x = self.activation()(x)
+        x = x.mean(dim=1)  # Pooling to match Shape: (32, 128)
+        return x

model/sequential.py

@@ -0,0 +1,68 @@
+import torch
+import torch.nn as nn
+from transformers import BertModel, BertTokenizer
+import numpy as np
+
+class SeqNetwork(nn.Module):
+    def __init__(self,
+                 input_dim,
+                 output_dim,
+                 hidden_dim,
+                 lstm_in,
+                 n_layers,
+                 skip_in=(4,),
+                 weight_norm=True,
+                 freeze = False,
+                 use_LSTM = False):
+        super(SeqNetwork, self).__init__()
+
+        self.freeze = freeze
+        self.skip_in = skip_in
+        self.num_layers = n_layers
+        self.weight_norm = weight_norm
+        self.use_LSTM = use_LSTM
+        # BERT model
+        self.bert = BertModel.from_pretrained('bert-base-uncased')
+        if self.freeze:
+            for param in self.bert.parameters():
+                param.requires_grad = False
+
+        # Sequential model for BERT embeddings
+        self.lstm = nn.LSTM(input_size=lstm_in, hidden_size=input_dim, batch_first=True)
+        dims = [input_dim] + [hidden_dim for _ in range(n_layers)] + [output_dim]
+        for l in range(0, self.num_layers + 1):
+            if l+1 in self.skip_in:
+                out_dim = dims[l + 1] + dims[0]
+                dims[l + 1] = out_dim
+            else:
+                out_dim = dims[l + 1]
+            lin = nn.Linear(dims[l], out_dim)
+            if weight_norm:
+                lin = nn.utils.weight_norm(lin)
+            setattr(self, "lin" + str(l), lin)
+        self.activation = nn.ReLU
+
+    def forward(self, input_ids, attention_mask):
+        # BERT embeddings
+        bert_outputs = self.bert(input_ids=input_ids, attention_mask=attention_mask)
+        bert_sequence_output = bert_outputs.pooler_output  # Shape: (batch_size, feature_size)
+        # print(bert_sequence_output.shape)
+        # LSTM over BERT embeddings
+        if self.use_LSTM:
+            lstm_out, (h_n, c_n) = self.lstm(bert_sequence_output)
+            inputs = h_n[-1]  # Use the last hidden state
+        else:
+            inputs = bert_sequence_output
+        x = inputs
+        for l in range(0, self.num_layers + 1):
+            lin = getattr(self, "lin" + str(l))
+
+            if l in self.skip_in:
+                x = torch.cat([x, inputs], 1) / np.sqrt(2)
+
+            x = lin(x)
+
+            if l < self.num_layers:
+                x = self.activation()(x)
+        bert_feature = x # Shape: (batch_size, feature_size)
+        return bert_feature