improved Janus

app.py

@@ -27,7 +27,8 @@ clip_utils.init_Clip()
 model_utils, vl_gpt, tokenizer = None, None, None
 model_name = "Clip"
 language_model_max_layer = 24
-language_model_best_layer = 8
+language_model_best_layer_min = 8
+language_model_best_layer_max = 8
 vision_model_best_layer = 24
 
 def clean():
@@ -215,7 +216,7 @@ def multimodal_understanding(model_type,
 # Gradio interface
 
 def model_slider_change(model_type):
-    global model_utils, vl_gpt, tokenizer, clip_utils, model_name, language_model_max_layer, language_model_best_layer, vision_model_best_layer
+    global model_utils, vl_gpt, tokenizer, clip_utils, model_name, language_model_max_layer, language_model_best_layer_min, language_model_best_layer_max, vision_model_best_layer
     model_name = model_type
 
 
@@ -226,13 +227,6 @@ def model_slider_change(model_type):
         gr.Dropdown(choices=["CLS", "max", "avg"], value="CLS", label="visual pooling method")
     ]
 
-    visual_res = [
-        gr.Dropdown(choices=["Visualization only", "answer + visualization"], value="Visualization only", label="response_type"),
-        gr.Dropdown(choices=["Visual Encoder"], value="Visual Encoder", label="focus"),
-        gr.Dropdown(choices=["GradCAM"], value="GradCAM", label="activation map type"),
-        gr.Dropdown(choices=["softmax", "sigmoid"], value="softmax", label="activation function")
-    ]
-
     language_res = [
         gr.Dropdown(choices=["Visualization only", "answer + visualization"], value="answer + visualization", label="response_type"),
         gr.Dropdown(choices=["Language Model"], value="Language Model", label="focus"),
@@ -253,7 +247,7 @@ def model_slider_change(model_type):
         return tuple(encoder_only_res + sliders)
     
     elif model_type.split('-')[0] == "Janus":
-        
+        # best seed: 70
         clean()
         set_seed()
         model_utils = Janus_Utils()
@@ -262,13 +256,14 @@ def model_slider_change(model_type):
             layer.self_attn = ModifiedLlamaAttention(layer.self_attn)
         
         language_model_max_layer = 24
-        language_model_best_layer = 8
+        language_model_best_layer_min = 8
+        language_model_best_layer_max = 10
         
         sliders = [
-            gr.Slider(minimum=1, maximum=24, value=24, step=1, label="visualization layers min"),
-            gr.Slider(minimum=1, maximum=24, value=24, step=1, label="visualization layers max"),
+            gr.Slider(minimum=1, maximum=24, value=language_model_best_layer_min, step=1, label="visualization layers min"),
+            gr.Slider(minimum=1, maximum=24, value=language_model_best_layer_max, step=1, label="visualization layers max"),
         ]
-        return tuple(visual_res + sliders)
+        return tuple(language_res + sliders)
     
     elif model_type.split('-')[0] == "LLaVA":
         
@@ -278,11 +273,12 @@ def model_slider_change(model_type):
         version = model_type.split('-')[1]
         vl_gpt, tokenizer = model_utils.init_LLaVA(version=version)
         language_model_max_layer = 32 if version == "1.5" else 28
-        language_model_best_layer = 10
+        language_model_best_layer_min = 10
+        language_model_best_layer_max = 10
 
         sliders = [
-            gr.Slider(minimum=1, maximum=language_model_max_layer, value=language_model_best_layer, step=1, label="visualization layers min"),
-            gr.Slider(minimum=1, maximum=language_model_max_layer, value=language_model_best_layer, step=1, label="visualization layers max"),
+            gr.Slider(minimum=1, maximum=language_model_max_layer, value=language_model_best_layer_min, step=1, label="visualization layers min"),
+            gr.Slider(minimum=1, maximum=language_model_max_layer, value=language_model_best_layer_max, step=1, label="visualization layers max"),
         ]
         return tuple(language_res + sliders)
     
@@ -295,11 +291,12 @@ def model_slider_change(model_type):
             layer.self_attn = ModifiedGemmaAttention(layer.self_attn)
         language_model_max_layer = 18
         vision_model_best_layer = 19
-        language_model_best_layer = 15
+        language_model_best_layer_min = 11
+        language_model_best_layer_max = 15
 
         sliders = [
-            gr.Slider(minimum=1, maximum=language_model_best_layer, value=language_model_best_layer, step=1, label="visualization layers min"),
-            gr.Slider(minimum=1, maximum=language_model_best_layer, value=language_model_best_layer, step=1, label="visualization layers max"),
+            gr.Slider(minimum=1, maximum=language_model_max_layer, value=language_model_best_layer_min, step=1, label="visualization layers min"),
+            gr.Slider(minimum=1, maximum=language_model_max_layer, value=language_model_best_layer_max, step=1, label="visualization layers max"),
         ]
         return tuple(language_res + sliders)
 
@@ -320,15 +317,15 @@ def focus_change(focus):
         if response_type.value == "answer + visualization":
             res = (
                 gr.Dropdown(choices=["GradCAM"], value="GradCAM", label="activation map type"),
-                gr.Slider(minimum=1, maximum=language_model_max_layer, value=language_model_best_layer, step=1, label="visualization layers min"), 
-                gr.Slider(minimum=1, maximum=language_model_max_layer, value=language_model_best_layer, step=1, label="visualization layers max")
+                gr.Slider(minimum=1, maximum=language_model_max_layer, value=language_model_best_layer_min, step=1, label="visualization layers min"), 
+                gr.Slider(minimum=1, maximum=language_model_max_layer, value=language_model_best_layer_max, step=1, label="visualization layers max")
             )
             return res
         else:
             res = (
                 gr.Dropdown(choices=["GradCAM"], value="GradCAM", label="activation map type"),
-                gr.Slider(minimum=1, maximum=language_model_max_layer, value=language_model_best_layer, step=1, label="visualization layers min"), 
-                gr.Slider(minimum=1, maximum=language_model_max_layer, value=language_model_best_layer, step=1, label="visualization layers max")
+                gr.Slider(minimum=1, maximum=language_model_max_layer, value=language_model_best_layer_min, step=1, label="visualization layers min"), 
+                gr.Slider(minimum=1, maximum=language_model_max_layer, value=language_model_best_layer_max, step=1, label="visualization layers max")
             )
             return res
 

demo/cam.py

@@ -1,674 +0,0 @@
-import cv2
-import numpy as np
-import types
-import torch
-import torch.nn.functional as F
-import matplotlib.pyplot as plt
-from PIL import Image
-from torch import nn
-import spaces
-from demo.modify_llama import *
-
-
-class AttentionGuidedCAM:
-    def __init__(self, model, register=True):
-        self.model = model
-        self.gradients = []
-        self.activations = []
-        self.hooks = []
-        if register:
-            self._register_hooks()
-
-    def _register_hooks(self):
-        for layer in self.target_layers:
-            self.hooks.append(layer.register_forward_hook(self._forward_hook))
-            self.hooks.append(layer.register_backward_hook(self._backward_hook))
-
-    def _forward_hook(self, module, input, output):
-        self.activations.append(output)
-
-    def _backward_hook(self, module, grad_in, grad_out):
-        self.gradients.append(grad_out[0])
-
-    
-    def remove_hooks(self):
-        for hook in self.hooks:
-            hook.remove()  
-    
-    @spaces.GPU(duration=120)
-    def generate_cam(self, input_tensor, class_idx=None):
-        raise NotImplementedError
-
-
-
-
-class AttentionGuidedCAMClip(AttentionGuidedCAM):
-    def __init__(self, model, target_layers):
-        self.target_layers = target_layers
-        super().__init__(model)
-
-    @spaces.GPU(duration=120)
-    def generate_cam(self, input_tensor, class_idx=None, visual_pooling_method="CLS"):
-        """ Generates Grad-CAM heatmap for ViT. """
-        
-        # Forward pass
-        output_full = self.model(**input_tensor)
-
-        if class_idx is None:
-            class_idx = torch.argmax(output_full.logits, dim=1).item()
-
-        if visual_pooling_method == "CLS":
-            output = output_full.image_embeds
-        elif visual_pooling_method == "avg":
-            output = self.model.visual_projection(output_full.vision_model_output.last_hidden_state).mean(dim=1)
-        else:
-            # project -> pooling
-            output, _ = self.model.visual_projection(output_full.vision_model_output.last_hidden_state).max(dim=1)
-
-            # pooling -> project
-            # output_mx, _ = output_full.vision_model_output.last_hidden_state.max(dim=1)
-            # output = self.model.visual_projection(output_mx)
-
-        output.backward(output_full.text_embeds[class_idx:class_idx+1], retain_graph=True)
-
-        # Aggregate activations and gradients from ALL layers
-        self.model.zero_grad()
-        cam_sum = None
-        for act, grad in zip(self.activations, self.gradients):
-
-            # act = torch.sigmoid(act[0])
-            act = F.relu(act[0])
-            
-            grad_weights = grad.mean(dim=-1, keepdim=True)
-            
-
-            print("act shape", act.shape)
-            print("grad_weights shape", grad_weights.shape)
-            
-            # cam = (act * grad_weights).sum(dim=-1)
-            cam, _ = (act * grad_weights).max(dim=-1)
-            # cam, _ = act.max(dim=-1)
-            # cam = cam.unsqueeze(0)
-            # cam, _ = grad_weights.max(dim=-1)
-            print("cam_shape: ", cam.shape)
-
-            # Sum across all layers
-            if cam_sum is None:
-                cam_sum = cam
-            else:
-                cam_sum += cam  
-                
-
-        # Normalize
-        cam_sum = F.relu(cam_sum)
-
-        # thresholding
-        cam_sum = cam_sum.to(torch.float32)
-        percentile = torch.quantile(cam_sum, 0.2)  # Adjust threshold dynamically
-        cam_sum[cam_sum < percentile] = 0
-
-        # Reshape
-        print("cam_sum shape: ", cam_sum.shape)
-        cam_sum = cam_sum[0, 1:]
-
-        num_patches = cam_sum.shape[-1]  # Last dimension of CAM output
-        grid_size = int(num_patches ** 0.5)
-        print(f"Detected grid size: {grid_size}x{grid_size}")
-        
-        cam_sum = cam_sum.view(grid_size, grid_size).detach()
-        cam_sum = (cam_sum - cam_sum.min()) / (cam_sum.max() - cam_sum.min())
-
-        return cam_sum, output_full, grid_size
-
-
-class AttentionGuidedCAMJanus(AttentionGuidedCAM):
-    def __init__(self, model, target_layers):
-        self.target_layers = target_layers
-        super().__init__(model)
-        self._modify_layers()
-        self._register_hooks_activations()
-
-    def _modify_layers(self):
-        for layer in self.target_layers:
-            setattr(layer, "attn_gradients", None)
-            setattr(layer, "attention_map", None)
-
-            layer.save_attn_gradients = types.MethodType(save_attn_gradients, layer)
-            layer.get_attn_gradients = types.MethodType(get_attn_gradients, layer)
-            layer.save_attn_map = types.MethodType(save_attn_map, layer)
-            layer.get_attn_map = types.MethodType(get_attn_map, layer)
-
-    def _forward_activate_hooks(self, module, input, output):
-        attn_output, attn_weights = output  # Unpack outputs
-        module.save_attn_map(attn_weights)
-        attn_weights.register_hook(module.save_attn_gradients)
-
-    def _register_hooks_activations(self):
-        for layer in self.target_layers:
-            if hasattr(layer, "q_proj"): # is an attention layer
-                self.hooks.append(layer.register_forward_hook(self._forward_activate_hooks))
-
-    @spaces.GPU(duration=120)
-    def generate_cam(self, input_tensor, tokenizer, temperature, top_p, class_idx=None, visual_pooling_method="CLS", focus="Visual Encoder"):
-        
-        torch.autograd.set_detect_anomaly(True)
-        for param in self.model.parameters():
-            param.requires_grad = False
-        
-        for layer in self.target_layers:
-            for param in layer.parameters():
-                param.requires_grad = True
-        
-        # Forward pass
-        image_embeddings, inputs_embeddings, outputs = self.model(input_tensor, tokenizer, temperature, top_p)
-
-
-        input_ids = input_tensor.input_ids
-
-        if focus == "Visual Encoder":
-            # Pooling
-            # if visual_pooling_method == "CLS":
-            #     image_embeddings_pooled = image_embeddings[:, 0, :]
-            # elif visual_pooling_method == "avg":
-            #     image_embeddings_pooled = image_embeddings[:, 1:, :].mean(dim=1)
-            # elif visual_pooling_method == "max":
-            #     image_embeddings_pooled, _ = image_embeddings[:, 1:, :].max(dim=1)
-
-            # print("image_embeddings_shape: ", image_embeddings_pooled.shape)
-            
-
-            start_idx = 620
-            # inputs_embeddings_pooled = inputs_embeddings[:, start_idx: -4].mean(dim=1)
-            self.model.zero_grad()
-            # image_embeddings_pooled.backward(inputs_embeddings_pooled, retain_graph=True)
-
-            loss = outputs.logits.max(dim=-1).values[0, start_idx + class_idx]
-            loss.backward()
-
-            cam_sum = None
-            for act, grad in zip(self.activations, self.gradients):
-                # act = torch.sigmoid(act)
-                act = F.relu(act[0])
-    
-
-                # Compute mean of gradients
-                print("grad shape:", grad.shape)
-                grad_weights = grad.mean(dim=-1, keepdim=True)
-
-                print("act shape", act.shape)
-                print("grad_weights shape", grad_weights.shape)
-
-                cam, _ = (act * grad_weights).max(dim=-1)
-                # cam, _ = grad_weights.max(dim=-1)
-                print(cam.shape)
-
-                # Sum across all layers
-                if cam_sum is None:
-                    cam_sum = cam
-                else:
-                    cam_sum += cam  
-
-            # Normalize
-            cam_sum = F.relu(cam_sum)
-            
-
-            # thresholding
-            cam_sum = cam_sum.to(torch.float32)
-            percentile = torch.quantile(cam_sum, 0.2)  # Adjust threshold dynamically
-            cam_sum[cam_sum < percentile] = 0
-
-            # Reshape
-            # if visual_pooling_method == "CLS":
-            cam_sum = cam_sum[0, 1:]
-            print("cam_sum shape: ", cam_sum.shape)
-            num_patches = cam_sum.shape[-1]  # Last dimension of CAM output
-            grid_size = int(num_patches ** 0.5)
-            print(f"Detected grid size: {grid_size}x{grid_size}")
-            
-            cam_sum = cam_sum.view(grid_size, grid_size)
-            cam_sum = (cam_sum - cam_sum.min()) / (cam_sum.max() - cam_sum.min())
-            cam_sum = cam_sum.detach().to("cpu")
-
-            return cam_sum, grid_size, start_idx
-
-
-
-
-
-
-        elif focus == "Language Model":
-            self.model.zero_grad()
-            loss = outputs.logits.max(dim=-1).values.sum()
-            loss.backward()
-            
-            self.activations = [layer.get_attn_map() for layer in self.target_layers]
-            self.gradients = [layer.get_attn_gradients() for layer in self.target_layers]
-
-            cam_sum = None
-            for act, grad in zip(self.activations, self.gradients):
-                # act = torch.sigmoid(act)
-                print("act_shape:", act.shape)
-                # print("act1_shape:", act[1].shape)
-                
-                act = act.mean(dim=1)
-    
-                # Compute mean of gradients
-                print("grad_shape:", grad.shape)
-                grad_weights = F.relu(grad.mean(dim=1))
-
-                cam = act * grad_weights
-                print(cam.shape)
-
-                # Sum across all layers
-                if cam_sum is None:
-                    cam_sum = cam
-                else:
-                    cam_sum += cam  
-
-            # Normalize
-            cam_sum = F.relu(cam_sum)
-
-            # thresholding
-            cam_sum = cam_sum.to(torch.float32)
-            percentile = torch.quantile(cam_sum, 0.2)  # Adjust threshold dynamically
-            cam_sum[cam_sum < percentile] = 0
-
-
-            # cam_sum shape: [1, seq_len, seq_len]
-            cam_sum_lst = []
-            cam_sum_raw = cam_sum
-            start = 620
-            for i in range(start, cam_sum_raw.shape[1]):
-                cam_sum = cam_sum_raw[:, i, :] # shape: [1: seq_len]
-                cam_sum = cam_sum[input_tensor.images_seq_mask].unsqueeze(0) # shape: [1, 576]
-                print("cam_sum shape: ", cam_sum.shape)
-                num_patches = cam_sum.shape[-1]  # Last dimension of CAM output
-                grid_size = int(num_patches ** 0.5)
-                print(f"Detected grid size: {grid_size}x{grid_size}")
-
-                # Fix the reshaping step dynamically
-                
-                cam_sum = cam_sum.view(grid_size, grid_size)
-                cam_sum = (cam_sum - cam_sum.min()) / (cam_sum.max() - cam_sum.min())
-                cam_sum = cam_sum.detach().to("cpu")
-                cam_sum_lst.append(cam_sum)
-
-
-            return cam_sum_lst, grid_size, start
-
-
-
-class AttentionGuidedCAMLLaVA(AttentionGuidedCAM):
-    def __init__(self, model, target_layers):
-        self.target_layers = target_layers
-        super().__init__(model, register=False)
-        self._modify_layers()
-        self._register_hooks_activations()
-
-    def _modify_layers(self):
-        for layer in self.target_layers:
-            setattr(layer, "attn_gradients", None)
-            setattr(layer, "attention_map", None)
-
-            layer.save_attn_gradients = types.MethodType(save_attn_gradients, layer)
-            layer.get_attn_gradients = types.MethodType(get_attn_gradients, layer)
-            layer.save_attn_map = types.MethodType(save_attn_map, layer)
-            layer.get_attn_map = types.MethodType(get_attn_map, layer)
-
-    def _forward_activate_hooks(self, module, input, output):
-        attn_output, attn_weights = output  # Unpack outputs
-        attn_weights.requires_grad_()
-        module.save_attn_map(attn_weights)
-        attn_weights.register_hook(module.save_attn_gradients)
-
-    def _register_hooks_activations(self):
-        for layer in self.target_layers:
-            if hasattr(layer, "q_proj"): # is an attention layer
-                self.hooks.append(layer.register_forward_hook(self._forward_activate_hooks))
-
-    @spaces.GPU(duration=120)
-    def generate_cam(self, inputs, tokenizer, temperature, top_p, class_idx=None, visual_pooling_method="CLS", focus="Visual Encoder"):
-        
-        # Forward pass
-        torch.autograd.set_detect_anomaly(True)
-        for param in self.model.parameters():
-            param.requires_grad = False
-        
-        for layer in self.target_layers:
-            for param in layer.parameters():
-                param.requires_grad = True
-        
-        outputs_raw = self.model(**inputs)
-
-        self.model.zero_grad()
-        print("outputs_raw", outputs_raw)
-
-        loss = outputs_raw.logits.max(dim=-1).values.sum()
-        loss.backward()
-
-        # get image masks
-        image_mask = []
-        last = 0
-        for i in range(inputs["input_ids"].shape[1]):
-            decoded_token = tokenizer.decode(inputs["input_ids"][0][i].item())
-            if (decoded_token == "<image>"):
-                image_mask.append(True)
-                last = i
-            else:
-                image_mask.append(False)
-
-
-        # Aggregate activations and gradients from ALL layers
-        self.activations = [layer.get_attn_map() for layer in self.target_layers]
-        self.gradients = [layer.get_attn_gradients() for layer in self.target_layers]
-        cam_sum = None
-
-        for act, grad in zip(self.activations, self.gradients):
-
-            print("act shape", act.shape)
-            print("grad shape", grad.shape)
-
-            grad = F.relu(grad)
-
-
-            cam = act * grad # shape: [1, heads, seq_len, seq_len]
-            cam = cam.sum(dim=1) # shape: [1, seq_len, seq_len]
-
-            # Sum across all layers
-            if cam_sum is None:
-                cam_sum = cam
-            else:
-                cam_sum += cam 
-
-        cam_sum = F.relu(cam_sum)
-        cam_sum = cam_sum.to(torch.float32)
-
-
-        # cam_sum shape: [1, seq_len, seq_len]
-        cam_sum_lst = []
-        cam_sum_raw = cam_sum
-        start_idx = last + 1
-        for i in range(start_idx, cam_sum_raw.shape[1]):
-            cam_sum = cam_sum_raw[0, i, :] # shape: [1: seq_len]
-
-            cam_sum = cam_sum[image_mask].unsqueeze(0) # shape: [1, img_seq_len]
-            print("cam_sum shape: ", cam_sum.shape)
-            num_patches = cam_sum.shape[-1]  # Last dimension of CAM output
-            grid_size = int(num_patches ** 0.5)
-            print(f"Detected grid size: {grid_size}x{grid_size}")
-            
-            cam_sum = cam_sum.view(grid_size, grid_size)
-            cam_sum = (cam_sum - cam_sum.min()) / (cam_sum.max() - cam_sum.min())
-            cam_sum_lst.append(cam_sum)
-
-
-        return cam_sum_lst, grid_size, start_idx
-
-
-
-
-
-
-class AttentionGuidedCAMChartGemma(AttentionGuidedCAM):
-    def __init__(self, model, target_layers):
-        self.target_layers = target_layers
-        super().__init__(model, register=True)
-        self._modify_layers()
-        self._register_hooks_activations()
-    
-    def _modify_layers(self):
-        for layer in self.target_layers:
-            setattr(layer, "attn_gradients", None)
-            setattr(layer, "attention_map", None)
-
-            layer.save_attn_gradients = types.MethodType(save_attn_gradients, layer)
-            layer.get_attn_gradients = types.MethodType(get_attn_gradients, layer)
-            layer.save_attn_map = types.MethodType(save_attn_map, layer)
-            layer.get_attn_map = types.MethodType(get_attn_map, layer)
-
-    def _forward_activate_hooks(self, module, input, output):
-        attn_output, attn_weights = output  # Unpack outputs
-        print("attn_output shape:", attn_output.shape)
-        print("attn_weights shape:", attn_weights.shape)
-        module.save_attn_map(attn_weights)
-        attn_weights.register_hook(module.save_attn_gradients)
-
-    def _register_hooks_activations(self):
-        for layer in self.target_layers:
-            if hasattr(layer, "q_proj"): # is an attention layer
-                self.hooks.append(layer.register_forward_hook(self._forward_activate_hooks))
-    
-    @spaces.GPU(duration=120)
-    def generate_cam(self, inputs, tokenizer, temperature, top_p, class_idx=None, visual_pooling_method="CLS", focus="Visual Encoder"):
-        
-        # Forward pass
-        torch.autograd.set_detect_anomaly(True)
-        for param in self.model.parameters():
-            param.requires_grad = False
-        
-        for layer in self.target_layers:
-            for param in layer.parameters():
-                param.requires_grad = True
-        
-        outputs_raw = self.model(**inputs, output_hidden_states=True)
-
-
-
-        # get image masks
-        image_mask = []
-        last = 0
-        for i in range(inputs["input_ids"].shape[1]):
-            decoded_token = tokenizer.decode(inputs["input_ids"][0][i].item())
-            if (decoded_token == "<image>"):
-                image_mask.append(True)
-                last = i
-            else:
-                image_mask.append(False)
-        start_idx = last + 1
-
-
-
-        if focus == "Visual Encoder":
-            # image_embeddings = outputs_raw.image_hidden_states
-            # inputs_embeddings = outputs_raw.hidden_states[0]
-            # # Pooling
-            # if visual_pooling_method == "avg":
-            #     image_embeddings_pooled = image_embeddings.mean(dim=1) # end of image: 618
-            # elif visual_pooling_method == "max":
-            #     image_embeddings_pooled, _ = image_embeddings.max(dim=1)
-
-            # print("image_embeddings_shape: ", image_embeddings_pooled.shape)
-            
-
-
-            # inputs_embeddings_pooled = inputs_embeddings[:, start_idx:].mean(dim=1)
-            self.model.zero_grad()
-            # image_embeddings_pooled.backward(inputs_embeddings_pooled, retain_graph=True)
-
-            loss = outputs_raw.logits.max(dim=-1).values[0, start_idx + class_idx]
-            loss.backward()
-
-            cam_sum = None
-            for act, grad in zip(self.activations, self.gradients):
-                # act = torch.sigmoid(act)
-                act = F.relu(act[0])
-    
-
-                # Compute mean of gradients
-                print("grad shape:", grad.shape)
-                grad_weights = grad.mean(dim=-1, keepdim=True)
-
-                print("act shape", act.shape)
-                print("grad_weights shape", grad_weights.shape)
-
-                cam = (act * grad_weights).sum(dim=-1)
-                # cam, _ = (act * grad_weights).max(dim=-1)
-                # cam, _ = grad_weights.max(dim=-1)
-                print(cam.shape)
-
-                # Sum across all layers
-                if cam_sum is None:
-                    cam_sum = cam
-                else:
-                    cam_sum += cam  
-
-            # Normalize
-            cam_sum = F.relu(cam_sum)
-            
-
-            # thresholding
-            cam_sum = cam_sum.to(torch.float32).detach().cpu()
-            percentile = torch.quantile(cam_sum, 0.2)  # Adjust threshold dynamically
-            cam_sum[cam_sum < percentile] = 0
-
-            # Reshape
-            print("cam_sum shape: ", cam_sum.shape)
-            num_patches = cam_sum.shape[-1]  # Last dimension of CAM output
-            grid_size = int(num_patches ** 0.5)
-            print(f"Detected grid size: {grid_size}x{grid_size}")
-            
-            cam_sum = cam_sum.view(grid_size, grid_size)
-            cam_sum = (cam_sum - cam_sum.min()) / (cam_sum.max() - cam_sum.min())
-
-            return cam_sum, grid_size, start_idx
-
-        elif focus == "Language Model":
-            self.model.zero_grad()
-            print("logits shape:", outputs_raw.logits.shape)
-            # loss = outputs_raw.logits.max(dim=-1).values.sum()
-            if class_idx == -1:
-                loss = outputs_raw.logits.max(dim=-1).values.sum()
-            else:
-                loss = outputs_raw.logits.max(dim=-1).values[0, start_idx + class_idx]
-            loss.backward()
-
-
-
-            # Aggregate activations and gradients from ALL layers
-            self.activations = [layer.get_attn_map() for layer in self.target_layers]
-            self.gradients = [layer.get_attn_gradients() for layer in self.target_layers]
-            print(f"layers shape: {len(self.target_layers)}")
-            print("activations & gradients shape", len(self.activations), len(self.gradients))
-
-            cams = []
-            
-            # Ver 2
-            for act, grad in zip(self.activations, self.gradients):
-
-                print("act shape", act.shape)
-                print("grad shape", grad.shape)
-                
-                grad = F.relu(grad)
-
-                # cam = grad
-                cam = act * grad # shape: [1, heads, seq_len, seq_len]
-                cam = cam.sum(dim=1) # shape: [1, seq_len, seq_len]
-                cam = cam.to(torch.float32).detach().cpu()
-                cams.append(cam)
-
-            # cam_sum = F.relu(cam_sum)
-            # cam_sum = cam_sum.to(torch.float32)
-
-            # cams shape: [layers, 1, seq_len, seq_len]
-            cam_sum_lst = []
-
-            start_idx = last + 1
-            for i in range(start_idx, cams[0].shape[1]):
-                cam_sum = None
-                for layer, cam_l in enumerate(cams):
-                    cam_l_i = cam_l[0, i, :] # shape: [1: seq_len]
-
-                    cam_l_i = cam_l_i[image_mask].unsqueeze(0) # shape: [1, img_seq_len]
-                    # print(f"layer: {layer}, token index: {i}")
-                    # print("cam_sum shape: ", cam_l_i.shape)
-                    num_patches = cam_l_i.shape[-1]  # Last dimension of CAM output
-                    grid_size = int(num_patches ** 0.5)
-                    # print(f"Detected grid size: {grid_size}x{grid_size}")
-
-                    # Fix the reshaping step dynamically
-                    cam_reshaped = cam_l_i.view(grid_size, grid_size)
-                    # print(f"max: {cam_reshaped.max()}, min: {cam_reshaped.min()}")
-                    # cam_reshaped = (cam_reshaped - cam_reshaped.min()) / (cam_reshaped.max() - cam_reshaped.min())
-                    if cam_sum == None:
-                        cam_sum = cam_reshaped
-                    else:
-                        cam_sum += cam_reshaped
-                    # print(f"normalized: max: {cam_normalized.max()}, min: {cam_normalized.min()}")
-
-                # print(f"sum: max: {cam_sum.max()}, min: {cam_sum.min()}")
-                cam_sum = (cam_sum - cam_sum.min()) / (cam_sum.max() - cam_sum.min())
-                cam_sum_lst.append(cam_sum)
-
-
-        return cam_sum_lst, grid_size, start_idx
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-def generate_gradcam(
-    cam, 
-    image,
-    size = (384, 384),
-    alpha=0.5, 
-    colormap=cv2.COLORMAP_JET, 
-    aggregation='mean',
-    normalize=False
-):
-    """
-    Generates a Grad-CAM heatmap overlay on top of the input image.
-
-    Parameters:
-      attributions (torch.Tensor): A tensor of shape (C, H, W) representing the
-        intermediate activations or gradients at the target layer.
-      image (PIL.Image): The original image.
-      alpha (float): The blending factor for the heatmap overlay (default 0.5).
-      colormap (int): OpenCV colormap to apply (default cv2.COLORMAP_JET).
-      aggregation (str): How to aggregate across channels; either 'mean' or 'sum'.
-
-    Returns:
-      PIL.Image: The image overlaid with the Grad-CAM heatmap.
-    """
-    # print("Generating Grad-CAM with shape:", cam.shape)
-
-    if normalize:
-        cam_min, cam_max = cam.min(), cam.max()
-        cam = cam - cam_min
-        cam = cam / (cam_max - cam_min)
-    # Convert tensor to numpy array
-    cam = torch.nn.functional.interpolate(cam.unsqueeze(0).unsqueeze(0), size=size, mode='bilinear').squeeze()
-    cam_np = cam.squeeze().detach().cpu().numpy()
-
-    # Apply Gaussian blur for smoother heatmaps
-    cam_np = cv2.GaussianBlur(cam_np, (5,5), sigmaX=0.8)
-
-    # Resize the cam to match the image size
-    width, height = size
-    cam_resized = cv2.resize(cam_np, (width, height))
-
-    # Convert the normalized map to a heatmap (0-255 uint8)
-    heatmap = np.uint8(255 * cam_resized)
-    heatmap = cv2.applyColorMap(heatmap, colormap)
-    # OpenCV produces heatmaps in BGR, so convert to RGB for consistency
-    heatmap = cv2.cvtColor(heatmap, cv2.COLOR_BGR2RGB)
-
-    # Convert original image to a numpy array
-    image_np = np.array(image)
-    image_np = cv2.resize(image_np, (width, height))
-
-    # Blend the heatmap with the original image
-    overlay = cv2.addWeighted(image_np, 1 - alpha, heatmap, alpha, 0)
-
-    return Image.fromarray(overlay)
-

demo/visualization.py

@@ -145,7 +145,7 @@ class Visualization:
         return cams
     
     
-    def process(self, cam_sum, thresholding=True, remove_cls=True, normalize=True):
+    def process(self, cam_sum, thresholding=True, remove_cls=False, normalize=True):
 
         cam_sum = cam_sum.to(torch.float32)
 

janus/models/modeling_vlm.py

@@ -256,7 +256,7 @@ class MultiModalityCausalLM(MultiModalityPreTrainedModel):
         inputs_embeds = self.language_model.get_input_embeddings()(input_ids)
 
         # replace with the image embeddings
-        images_embeds = images_embeds[:, 1:, :]
+        # images_embeds = images_embeds[:, 1:, :]
         inputs_embeds[images_seq_mask] = images_embeds[images_emb_mask]
 
         return inputs_embeds
@@ -293,7 +293,8 @@ class MultiModalityCausalLM(MultiModalityPreTrainedModel):
         inputs_embeds = self.language_model.get_input_embeddings()(input_tensor.input_ids)
         # print("input_embeddings: ", inputs_embeds)
 
-        images_embeds_rest = images_embeds[:, 1:, :]
+        # images_embeds_rest = images_embeds[:, 1:, :]
+        images_embeds_rest = images_embeds[:, :, :]
 
         # images_embeds_pooled = images_embeds.mean(dim=1)
 

janus/models/siglip_vit.py

@@ -655,9 +655,9 @@ def create_siglip_vit(
     else:
         layers = min(vision_cfg.layers, select_layer)
 
-    # Requre CLS token
-    vision_cfg.class_token = True
-    print("Usage Class Token: ", vision_cfg.class_token)
+    # Require CLS token
+    # vision_cfg.class_token = True
+    # print("Usage Class Token: ", vision_cfg.class_token)
     
     model = VisionTransformer(
         img_size=image_size,

questions/VLAT.py

@@ -49,7 +49,7 @@ VLAT_questions=[
 
     [
         "StackedArea",
-        "The number of girls named 'Olivia' was raising or falling from 2009 to 2012?", 
+        "The number of girls named 'Olivia' was increasing or decreasing from 2009 to 2012?", 
         "images/mini-VLAT/StackedArea.png"
     ],
 
@@ -115,7 +115,7 @@ VLAT_questions=[
 
     [
         "LineChart",
-        "Over the course of the first quarter of 2020, the price of a barrel of oil was rising or falling?", 
+        "Over the course of the first quarter of 2020, the price of a barrel of oil was increasing or decreasing?", 
         "images/mini-VLAT/LineChart.png" 
     ],
 
@@ -175,7 +175,7 @@ VLAT_questions=[
 
     [
         "AreaChart",
-        "Over the first six months of 2018, the price of a pound of coffee beans was roughly falling or rising?",
+        "Over the first six months of 2018, the price of a pound of coffee beans was roughly decreasing or increasing?",
         "images/mini-VLAT/AreaChart.png"
     ],