update

README copy.md

@@ -0,0 +1,26 @@
+---
+title: Association
+emoji: 💬
+colorFrom: yellow
+colorTo: purple
+sdk: gradio
+sdk_version: 4.36.1
+app_file: app.py
+pinned: false
+license: cc-by-4.0
+---
+
+- Association-Demo
+
+- HMDB51
+- OCL-3D
+    - data
+        - resources
+            - OCL_selected_test_affordance.pkl
+            - OCL_annot_test.pkl
+
+- 26000: /home/lihong/workspace/pangea
+
+
+
+An example chatbot using [Gradio](https://gradio.app), [`huggingface_hub`](https://huggingface.co/docs/huggingface_hub/v0.22.2/en/index), and the [Hugging Face Inference API](https://huggingface.co/docs/api-inference/index).

app.py

@@ -0,0 +1,665 @@
+import gradio as gr 
+import random
+import os
+import pickle
+from PIL import Image
+
+
+class Process:
+    gt_image= ""
+    gt_image_idx = (0, 0)
+    raw_image_path = ""
+    candidate_image1_path = ""
+    candidate_image2_path = ""
+    candidata_image1_idx = (0, 0)
+    candidate_image2_idx = (0, 0)
+    candidate_image1_group = "negative"
+    candidate_image2_group = "negative"
+    concept_choices = None
+    pkl_data = None
+    positive_cand = []
+    negative_cand = []
+    positive1_cand = []
+    positive2_cand = []
+    positive_common_cand = []
+    schedule = 0
+    idx_to_chain = {}
+    
+global process
+process = Process()
+
+def load_data_and_produce_list(dataset,exp_mode, concept_choices):
+    
+    if dataset == "ocl_attribute":
+        #TODO
+        attr_name = ['wooden', 'metal', 'flying', 'ripe', 'fresh', 'natural', 'cooked', 'painted', 'rusty', 'furry']
+        attr2idx = {item:idx for idx,item in enumerate(attr_name)}
+        idx_2_attr = {value:key for key,value in attr2idx.items()}
+        pkl_path = "Data/OCL_data/OCL_selected_test_attribute_refined.pkl"
+        image_dir = "Data/OCL_data/data" 
+        
+        with open(pkl_path,"rb") as f:
+            data = pickle.load(f)
+        
+        with open('Data/OCL_data/OCL_annot_test.pkl', "rb") as f:
+            process.pkl_data = pickle.load(f)
+
+        if exp_mode == "One concept":
+            process.positive_cand = data['selected_individual_pkl'][process.idx_to_chain[concept_choices]]
+            process.negative_cand = data['negative_pkl']
+
+        else:
+            
+            selected_concept_group = process.idx_to_chain[concept_choices].split("-")
+            selected_paired_pkl = data['selected_paired_pkl'][process.idx_to_chain[concept_choices]]
+            process.positive1_cand = selected_paired_pkl[selected_concept_group[0]]
+            process.positive2_cand = selected_paired_pkl[selected_concept_group[1]]
+            process.positive_common_cand = selected_paired_pkl[process.idx_to_chain[concept_choices]]
+            process.negative_cand = data['negative_pkl']         
+                
+    elif dataset == "ocl_affordance":
+        aff_name = ['break', 'carry', 'clean','cut','push','sit','write']
+        aff2idx = {item:idx for idx,item in enumerate(aff_name)}
+        idx_2_attr = {value:key for key,value in aff2idx.items()}
+        pkl_path = "Data/OCL_data/OCL_selected_test_affordance_refined.pkl"
+        image_dir = "Data/OCL_data/data" 
+        
+        with open(pkl_path,"rb") as f:
+            data = pickle.load(f)
+        
+        with open('Data/OCL_data/OCL_annot_test.pkl', "rb") as f:
+            process.pkl_data = pickle.load(f)
+        if exp_mode == "One concept":
+            process.positive_cand = data['selected_individual_pkl'][process.idx_to_chain[concept_choices]]
+            process.negative_cand = data['negative_pkl']
+        else:
+            selected_concept_group = process.idx_to_chain[concept_choices].split("-")
+            selected_paired_pkl = data['selected_paired_pkl'][process.idx_to_chain[concept_choices]]
+            process.positive1_cand = selected_paired_pkl[selected_concept_group[0]]
+            process.positive2_cand = selected_paired_pkl[selected_concept_group[1]]
+            process.positive_common_cand = selected_paired_pkl[process.idx_to_chain[concept_choices]]
+            process.negative_cand = data['negative_pkl']         
+    elif dataset == "Pangea":
+        attr_name = ["hit-18.1","run-51.3.2","dress-41.1.1-1-1","drive-11.5","cooking-45.3","build-26.1","shake-22.3-2","cut-21.1-1"]
+        attr2idx = {item:idx for idx,item in enumerate(attr_name)}
+        idx_2_attr = {value:key for key,value in attr2idx.items()}
+        pkl_path = "Data/pangea/pangea_test_refined.pkl"
+        image_dir = "Data/pangea/pangea" 
+        with open(pkl_path,"rb") as f:
+            data = pickle.load(f)
+        
+        with open("Data/pangea/B123_test_KIN-FULL_with_node.pkl", "rb") as f:
+            process.pkl_data = pickle.load(f)
+            
+        if exp_mode == "One concept":
+            process.positive_cand = data['selected_pkl'][process.idx_to_chain[concept_choices]]
+            process.negative_cand = data['negative_pkl']
+        else:
+            selected_concept_group = process.idx_to_chain[concept_choices].split("_")
+            selected_paired_pkl = data['selected_paired_pkl'][process.idx_to_chain[concept_choices]]
+            process.positive1_cand = selected_paired_pkl[selected_concept_group[0]]
+            process.positive2_cand = selected_paired_pkl[selected_concept_group[1]]
+            process.positive_common_cand = selected_paired_pkl[process.idx_to_chain[concept_choices]]
+            process.negative_cand = data['negative_pkl']  
+                                                                            
+    elif dataset == "hmdb":
+        attr_name = ['brush_hair','clap', 'dive', 'shake_hands','hug' ,'sit','smoke','eat']
+        attr2idx = {key:item for key,item in enumerate(attr_name)}
+        image_dir = "Data/refined_HMDB"
+        pkl_path = "Data/refined_HMDB.pkl"
+
+
+        with open(pkl_path,"rb") as f:
+            data = pickle.load(f)
+
+        if exp_mode == "One concept":
+            positive_cand = []
+            negative_cand = []
+            for each_data in data:
+                each_data['name'] = os.path.join(image_dir,each_data['name'])
+                if process.idx_to_chain[concept_choices] in each_data["label"]:
+                    positive_cand.append(each_data)
+                else:
+                    negative_cand.append(each_data)
+
+                if len(positive_cand) > 30 and len(negative_cand) > 100:
+                    break
+            
+            process.positive_cand = positive_cand
+            process.negative_cand = negative_cand
+            
+        else:
+            
+            negative_cand = []
+            positive1_cand = []
+            positive2_cand = []
+            positive_common_cand = []
+            
+            for each_data in data:
+                each_data['name'] = os.path.join(image_dir,each_data['name'])
+                
+                selected_concept_group = process.idx_to_chain[concept_choices].split("-")
+                
+                if selected_concept_group[0] in each_data["name"] and selected_concept_group[1] in each_data["name"]:
+                    positive_common_cand.append(each_data)
+                elif selected_concept_group[0] in each_data["name"]:
+                    positive1_cand.append(each_data)
+                elif selected_concept_group[1] in each_data["name"]:
+                    positive2_cand.append(each_data)
+                else:
+                    if len(negative_cand) <= 100:
+                        negative_cand.append(each_data)  
+                
+                
+            process.positive1_cand = positive1_cand
+            process.positive2_cand = positive2_cand
+            process.positive_common_cand = positive_common_cand
+            process.negative_cand = negative_cand
+                
+                
+TARGET_SIZE = (200,200)
+
+
+def load_images(dataset, raw_image_path, candidate_image1_path, candidate_image2_path):
+    if dataset == "ocl_attribute" or dataset == "ocl_affordance":
+        image_dir = "Data/OCL_data/data"
+        raw_data = process.pkl_data[raw_image_path[0]]
+
+        img_path = os.path.join(image_dir,raw_data["name"])
+        raw_image = Image.open(img_path).crop(raw_data['objects'][raw_image_path[1]]['box']).resize(TARGET_SIZE)
+        
+        candidate_data1 = process.pkl_data[candidate_image1_path[0]]
+        cand1_img_path = os.path.join(image_dir,candidate_data1["name"])
+        candidate_image1 = Image.open(cand1_img_path).crop(candidate_data1['objects'][candidate_image1_path[1]]['box']).resize(TARGET_SIZE)
+        
+        candidate_data2 = process.pkl_data[candidate_image2_path[0]]
+        cand2_img_path = os.path.join(image_dir,candidate_data2["name"])
+        candidate_image2 = Image.open(cand2_img_path).crop(candidate_data2['objects'][candidate_image2_path[1]]['box']).resize(TARGET_SIZE)
+    elif dataset == "Pangea":
+        mapping_dataset_directory = {'ActvityNet_hico_style_batch1':'ActivityNet_hico_batch1','charadesEgo_hico_style':'charadesego_frame', 'HAG_hico_style_new':'hag_frame','HACS_hico_style':'hacs_frame','kinetics_hico_style':'kinetics_dataset/k700-2020/train'}
+        image_dir = "Data/pangea/pangea"
+        raw_data = process.pkl_data[raw_image_path]       
+        img_path = os.path.join(image_dir,mapping_dataset_directory[raw_data[0]], raw_data[1])
+        raw_image = Image.open(img_path).resize(TARGET_SIZE)    
+
+        candidate_data1 = process.pkl_data[candidate_image1_path]
+        cand1_img_path = os.path.join(image_dir,mapping_dataset_directory[candidate_data1[0]], candidate_data1[1])
+        candidate_image1 = Image.open(cand1_img_path).resize(TARGET_SIZE)
+        
+        candidate_data2 = process.pkl_data[candidate_image2_path]
+        cand2_img_path = os.path.join(image_dir,mapping_dataset_directory[candidate_data2[0]], candidate_data2[1])
+        candidate_image2 = Image.open(cand2_img_path).resize(TARGET_SIZE)                            
+    else:
+        raw_image = Image.open(raw_image_path['name']).resize(TARGET_SIZE)
+        candidate_image1 = Image.open(candidate_image1_path['name']).resize(TARGET_SIZE)
+        candidate_image2 = Image.open(candidate_image2_path['name']).resize(TARGET_SIZE)
+    
+    return raw_image, candidate_image1, candidate_image2
+
+def load_candidate_images(dataset, cand_image,candidate_image1_path,candidate_image2_path):
+    raw_image = cand_image
+    if dataset == "ocl_attribute" or dataset == "ocl_affordance":
+        image_dir = "Data/OCL_data/data"
+        candidate_data1 = process.pkl_data[candidate_image1_path[0]]
+        cand1_img_path = os.path.join(image_dir, candidate_data1["name"])
+        candidate_image1 = Image.open(cand1_img_path).crop(candidate_data1['objects'][candidate_image1_path[1]]['box']).resize(TARGET_SIZE)
+        
+        candidate_data2 = process.pkl_data[candidate_image2_path[0]]
+        cand2_img_path = os.path.join(image_dir, candidate_data2["name"])
+        candidate_image2 = Image.open(cand2_img_path).crop(candidate_data2['objects'][candidate_image2_path[1]]['box']).resize(TARGET_SIZE)
+    elif dataset == "Pangea":
+        mapping_dataset_directory = {'ActvityNet_hico_style_batch1':'ActivityNet_hico_batch1','charadesEgo_hico_style':'charadesego_frame', 'HAG_hico_style_new':'hag_frame','HACS_hico_style':'hacs_frame','kinetics_hico_style':'kinetics_dataset/k700-2020/train'}
+        image_dir = "Data/pangea/pangea"
+        candidate_data1 = process.pkl_data[candidate_image1_path]
+        cand1_img_path = os.path.join(image_dir,mapping_dataset_directory[candidate_data1[0]],candidate_data1[1])  
+        candidate_image1 = Image.open(cand1_img_path).resize(TARGET_SIZE)    
+        
+        candidate_data2 = process.pkl_data[candidate_image2_path]
+        cand2_img_path = os.path.join(image_dir,mapping_dataset_directory[candidate_data2[0]],candidate_data2[1])
+        candidate_image2 = Image.open(cand2_img_path).resize(TARGET_SIZE)           
+    else:
+        candidate_image1 = Image.open(candidate_image1_path['name']).resize(TARGET_SIZE)
+        candidate_image2 = Image.open(candidate_image2_path['name']).resize(TARGET_SIZE)
+
+    return raw_image,candidate_image1,candidate_image2
+
+class InferenceDemo(object):
+    def __init__(self,args,dataset,exp_mode,concept_choices):
+        print("init success")
+      
+
+def get_concept_choices(dataset,exp_mode):
+    # if dataset == "ocl":
+    if dataset == "ocl_affordance":
+        if exp_mode == "One concept":
+            choices = [f"Chain_{i}" for i in range(8)]
+        else:
+            choices = [f"Chain_{i}" for i in range(4)]
+    elif dataset == "Pangea":
+        if exp_mode == "One concept":
+            choices = [f"Chain_{i}" for i in range(8)]
+        else:
+            choices = [f"Chain_{i}" for i in range(4)]        
+    else:
+        if exp_mode == "One concept":
+            choices = [f"Chain_{i}" for i in range(8)]
+        else:
+            choices = [f"Chain_{i}" for i in range(4)]
+        
+    return gr.update(choices=choices)
+
+def load_images_and_concepts(dataset,exp_mode,concept_choices):
+    
+    process.concept_choices = concept_choices
+    idx_2_chain = {}
+    if dataset == "ocl_attribute":
+        if exp_mode == "One concept":
+            concept = ["furry","metal","fresh","cooked","natural","ripe","painted","rusty"]
+            for idx in range(8):
+                idx_2_chain[f"Chain_{idx}"] = concept[idx]
+        else:
+            concept = ["furry-metal","fresh-cooked","natural-ripe","painted-rusty"]
+            for idx in range(4):
+                idx_2_chain[f"Chain_{idx}"] = concept[idx]
+    elif dataset == "ocl_affordance":
+        if exp_mode == "One concept":
+            concept = ['break', 'carry', 'clean','cut','open','push','sit','write']
+            for idx in range(8):
+                idx_2_chain[f"Chain_{idx}"] = concept[idx]
+        else:
+            concept = ['sit-write','push-carry','cut-clean','open-break']
+            for idx in range(4):
+                idx_2_chain[f"Chain_{idx}"] = concept[idx]
+    elif dataset == "Pangea":
+        if exp_mode == "One concept":
+            concept = ["hit-18.1","run-51.3.2","dress-41.1.1-1-1","drive-11.5","cooking-45.3","build-26.1","shake-22.3-2","cut-21.1-1"]
+            for idx in range(8):
+                idx_2_chain[f"Chain_{idx}"] = concept[idx]
+        else:
+            concept = ['run-51.3.2_hit-18.1', 'drive-11.5_dress-41.1.1-1-1', 'cooking-45.3_build-26.1','shake-22.3-2_cut-21.1-1']
+            for idx in range(4):
+                idx_2_chain[f"Chain_{idx}"] = concept[idx]        
+    else:
+        if exp_mode == "One concept":
+            concept = ["brush_hair","dive","clap","hug","shake_hands","sit","smoke","eat"]
+            for idx in range(8):
+                idx_2_chain[f"Chain_{idx}"] = concept[idx]
+        else:
+            concept = ["brush_hair-dive","clap-hug","shake_hands-sit","smoke-eat"]
+            for idx in range(4):
+                idx_2_chain[f"Chain_{idx}"] = concept[idx]
+    process.idx_to_chain = idx_2_chain
+    
+    load_data_and_produce_list(dataset,exp_mode,concept_choices)
+    
+    if exp_mode == "One concept":
+        if random.random() < 0.5:
+            process.raw_image_path = random.choice(process.positive_cand)
+            process.candidate_image1_idx = process.candidate_image1_path = random.choice(process.positive_cand)
+            process.candidate_image2_idx = process.candidate_image2_path = random.choice(process.negative_cand)
+            process.candidate_image1_group, process.candidate_image2_group = "positive", "negative"
+            process.gt_image = "Image1"
+            process.gt_image_idx = process.candidata_image1_idx
+        else:
+            process.raw_image_path = random.choice(process.positive_cand)
+            process.candidate_image1_idx = process.candidate_image1_path = random.choice(process.negative_cand)
+            process.candidate_image2_idx = process.candidate_image2_path = random.choice(process.positive_cand)
+            process.candidate_image1_group, process.candidate_image2_group = "negative", "positive"
+            process.gt_image = "Image2"
+            process.gt_image_idx = process.candidate_image2_idx
+    else:
+        if random.random() < 0.5:
+            process.raw_image_path = random.choice(process.positive1_cand)
+            process.candidate_image1_idx = process.candidate_image1_path = random.choice(process.positive1_cand)
+            process.candidate_image2_idx = process.candidate_image2_path = random.choice(process.negative_cand)
+            process.candidate_image1_group, process.candidate_image2_group = "positive1", "negative"
+            process.gt_image = "Image1"
+            process.gt_image_idx = process.candidata_image1_idx
+        else:
+            process.raw_image_path = random.choice(process.positive1_cand)
+            process.candidate_image1_idx = process.candidate_image1_path = random.choice(process.negative_cand)
+            process.candidate_image2_idx = process.candidate_image2_path = random.choice(process.positive1_cand)
+            process.candidate_image1_group, process.candidate_image2_group = "negative", "positive1"
+            process.gt_image = "Image2"
+            process.gt_image_idx = process.candidate_image2_idx
+    raw_image,candidate_image1,candidate_image2 = load_images(dataset, process.raw_image_path,process.candidate_image1_path,process.candidate_image2_path)
+    if dataset == "Pangea":
+        concept = ["hit", "run", "dress", "drive", "cooking", "build", "shake", "cut"]
+    elif dataset == "ocl_attribute":
+        concept = ["furry","metal","fresh","cooked","natural","ripe","painted","rusty"]
+    elif dataset == "ocl_affordance":
+        concept = ['break', 'carry', 'clean','cut','open','push','sit','write']
+    return raw_image,candidate_image1,candidate_image2, str(concept)
+    
+
+def count_and_reload_images(dataset,exp_mode, select_input,show_result, steps,raw_image,candidate_image1,candidate_image2):
+    
+    if select_input != None:
+        if select_input == process.gt_image or int(steps) < 6 or select_input == 'Uncertain':
+            if select_input == 'Uncertain':
+                if process.gt_image == 'Image1':
+                    negative_sample = 'Image2'
+                else:
+                    negative_sample = 'Image1'
+                filter_images(dataset, exp_mode, process.concept_choices, negative_sample)
+            if select_input == process.gt_image:
+                show_result = "Success!"
+            elif select_input == 'Uncertain':
+                show_result = 'Skip'
+            else:
+                show_result = "Error!"
+                
+            if exp_mode == "One concept":
+                if process.gt_image == "Image1":
+                    candidate_image = candidate_image1
+                else:
+                    candidate_image = candidate_image2                
+                if random.random() < 0.5:
+                    process.candidate_image1_idx = process.candidate_image1_path = random.choice([x for x in process.positive_cand if x!=process.gt_image_idx])
+                    process.candidate_image2_idx = process.candidate_image2_path = random.choice(process.negative_cand)
+                    process.candidate_image1_group, process.candidate_image2_group = "positive", "negative"
+                    process.gt_image = "Image1"
+                    process.gt_image_idx = process.candidate_image1_idx
+                else:
+                    process.candidate_image1_idx = process.candidate_image1_path = random.choice(process.negative_cand)
+                    process.candidate_image2_idx = process.candidate_image2_path = random.choice([x for x in process.positive_cand if x!=process.gt_image_idx])
+                    process.candidate_image1_group, process.candidate_image2_group = "negative", "positive"
+                    process.gt_image = "Image2"
+                    process.gt_image_idx = process.candidate_image2_idx
+                    
+                raw_image,candidate_image1,candidate_image2 = load_candidate_images(dataset,candidate_image,process.candidate_image1_path,process.candidate_image2_path)
+            else:
+                if process.gt_image == "Image1":
+                    candidate_image = candidate_image1
+                else:
+                    candidate_image = candidate_image2 
+                    
+                if random.random() < 0.5:
+                    if process.schedule < 3:
+                        process.candidate_image1_idx = process.candidate_image1_path = random.choice([x for x in process.positive1_cand if x!=process.gt_image_idx])
+                        process.candidate_image2_idx = process.candidate_image2_path = random.choice(process.negative_cand)
+                        process.candidate_image1_group, process.candidate_image2_group = "positive1", "negative"
+                        raw_image,candidate_image1,candidate_image2 = load_candidate_images(dataset,candidate_image,process.candidate_image1_path,process.candidate_image2_path)
+                        process.schedule += 1
+                    elif process.schedule == 3:
+                        if len(process.positive_common_cand) != 0:
+                            process.candidate_image1_idx = process.candidate_image1_path = random.choice([x for x in process.positive_common_cand if x!=process.gt_image_idx])
+                            process.candidate_image2_idx = process.candidate_image2_path = random.choice(process.negative_cand)
+                            process.candidate_image1_group, process.candidate_image2_group = "positive_com", "negative"
+                            raw_image,candidate_image1,candidate_image2 = load_candidate_images(dataset,candidate_image,process.candidate_image1_path,process.candidate_image2_path)
+                        else:
+                            process.raw_image_path = random.choice(process.positive2_cand)
+                            process.candidate_image1_idx = process.candidate_image1_path = random.choice([x for x in process.positive2_cand if x!=process.gt_image_idx])
+                            process.candidate_image2_idx = process.candidate_image2_path = random.choice(process.negative_cand)
+                            process.candidate_image1_group, process.candidate_image2_group = "positive2", "negative"
+                            raw_image,candidate_image1,candidate_image2 = load_images(dataset,process.raw_image_path,process.candidate_image1_path,process.candidate_image2_path)
+                        process.schedule += 1
+                    elif process.schedule < 7:
+                        process.candidate_image1_idx = process.candidate_image1_path = random.choice([x for x in process.positive2_cand if x!=process.gt_image_idx])
+                        process.candidate_image2_idx = process.candidate_image2_path = random.choice(process.negative_cand)
+                        process.candidate_image1_group, process.candidate_image2_group = "positive2", "negative"
+                        raw_image,candidate_image1,candidate_image2 = load_candidate_images(dataset,candidate_image,process.candidate_image1_path,process.candidate_image2_path)
+                        process.schedule += 1
+                    elif process.schedule == 7:
+                        if len(process.positive_common_cand) != 0:
+                            process.candidate_image1_path = random.choice([x for x in process.positive_common_cand if x!=process.gt_image_idx])
+                            process.candidate_image2_path = random.choice(process.negative_cand)
+                            raw_image,candidate_image1,candidate_image2 = load_candidate_images(dataset,candidate_image,process.candidate_image1_path,process.candidate_image2_path)
+                        else:
+                            process.raw_image_path = random.choice(process.positive1_cand)
+                            process.candidate_image1_path = random.choice([x for x in process.positive1_cand if x!=process.gt_image_idx])
+                            process.candidate_image2_path = random.choice(process.negative_cand)
+                            raw_image,candidate_image1,candidate_image2 = load_images(dataset,process.raw_image_path,process.candidate_image1_path,process.candidate_image2_path)
+                        process.schedule = 0
+                    process.gt_image = "Image1"
+                    process.gt_image_idx = process.candidate_image1_idx
+                else:
+                    if process.schedule < 3:
+                        process.candidate_image2_idx = process.candidate_image2_path = random.choice([x for x in process.positive1_cand if x!=process.gt_image_idx])
+                        process.candidate_image1_idx = process.candidate_image1_path = random.choice(process.negative_cand)
+                        process.candidate_image1_group, process.candidate_image2_group = "negative", "positive1"
+                        raw_image,candidate_image1,candidate_image2 = load_candidate_images(dataset,candidate_image,process.candidate_image1_path,process.candidate_image2_path)
+                        process.schedule += 1
+                    elif process.schedule == 3:
+                        if len(process.positive_common_cand) != 0:
+                            process.candidate_image2_idx = process.candidate_image2_path = random.choice([x for x in process.positive_common_cand if x!=process.gt_image_idx])
+                            process.candidate_image1_idx = process.candidate_image1_path = random.choice(process.negative_cand)
+                            process.candidate_image1_group, process.candidate_image2_group = "negative", "positive_com"
+                            raw_image,candidate_image1,candidate_image2 = load_candidate_images(dataset,candidate_image,process.candidate_image1_path,process.candidate_image2_path)
+                        else:
+                            process.raw_image_path = random.choice(process.positive2_cand)
+                            process.candidate_image2_idx = process.candidate_image2_path = random.choice([x for x in process.positive2_cand if x!=process.gt_image_idx])
+                            process.candidate_image1_idx = process.candidate_image1_path = random.choice(process.negative_cand)
+                            process.candidate_image1_group, process.candidate_image2_group = "negative", "positive2"
+                            raw_image,candidate_image1,candidate_image2 = load_images(dataset,process.raw_image_path,process.candidate_image1_path,process.candidate_image2_path)
+                        process.schedule += 1
+                    elif process.schedule < 7:
+                        process.candidate_image2_idx = process.candidate_image2_path = random.choice([x for x in process.positive2_cand if x!=process.gt_image_idx])
+                        process.candidate_image1_idx = process.candidate_image1_path = random.choice(process.negative_cand)
+                        process.candidate_image1_group, process.candidate_image2_group = "negative", "positive2"
+                        raw_image,candidate_image1,candidate_image2 = load_candidate_images(dataset,candidate_image,process.candidate_image1_path,process.candidate_image2_path)
+                        process.schedule += 1
+                    elif process.schedule == 7:
+                        if len(process.positive_common_cand) != 0:
+                            process.candidate_image2_idx = process.candidate_image2_path = random.choice([x for x in process.positive_common_cand if x!=process.gt_image_idx])
+                            process.candidate_image1_idx = process.candidate_image1_path = random.choice(process.negative_cand)
+                            process.candidate_image1_group, process.candidate_image2_group = "negative", "positive_com"
+                            raw_image,candidate_image1,candidate_image2 = load_candidate_images(dataset,candidate_image,process.candidate_image1_path,process.candidate_image2_path)
+                        else:
+                            process.raw_image_path = random.choice(process.positive1_cand)
+                            process.candidate_image2_idx = process.candidate_image2_path = random.choice([x for x in process.positive1_cand if x!=process.gt_image_idx])
+                            process.candidate_image1_idx = process.candidate_image1_path = random.choice(process.negative_cand)
+                            process.candidate_image1_group, process.candidate_image2_group = "negative", "positive1"
+                            raw_image,candidate_image1,candidate_image2 = load_images(dataset,process.raw_image_path,process.candidate_image1_path,process.candidate_image2_path)
+                        process.schedule = 0
+                        
+                    process.gt_image = "Image2"
+                    process.gt_image_idx = process.candidate_image2_idx
+                    
+            if select_input != 'Uncertain':
+                steps = int(steps) + 1
+            select_input = None 
+
+        else:
+            show_result = "Error, Please reset!"
+            process.gt_image = None
+    
+    return select_input,show_result, steps,raw_image,candidate_image1,candidate_image2
+
+def filter_images(dataset, exp_mode, concept_choices, image_filtered):      
+    if image_filtered == None:
+        return None
+    if dataset == "ocl_attribute" or dataset == "ocl_affordance":
+        if dataset == "ocl_attribute":
+            pkl_path = "Data/OCL_data/OCL_selected_test_attribute_refined.pkl"
+        else:
+            pkl_path = "Data/OCL_data/OCL_selected_test_affordance_refined.pkl"
+        with open(pkl_path,"rb") as f:
+            data = pickle.load(f)
+        if exp_mode == "One concept":
+            if image_filtered == "Image1":
+                print(process.candidate_image1_idx)
+                if process.candidate_image1_group == "positive":
+                    if process.candidate_image1_idx in data['selected_individual_pkl'][process.idx_to_chain[concept_choices]]:
+                        data['selected_individual_pkl'][process.idx_to_chain[concept_choices]].remove(process.candidate_image1_idx)
+                elif process.candidate_image1_group == "negative":
+                    if process.candidate_image1_idx in data["negative_pkl"]:
+                        data["negative_pkl"].remove(process.candidate_image1_idx)
+                else:
+                    print('Error')
+            else:
+                print(process.candidate_image2_idx)
+                if process.candidate_image2_group == "positive":
+                    if process.candidate_image2_idx in data['selected_individual_pkl'][process.idx_to_chain[concept_choices]]:
+                        data['selected_individual_pkl'][process.idx_to_chain[concept_choices]].remove(process.candidate_image2_idx)
+                elif process.candidate_image2_group == "negative":
+                    if process.candidate_image2_idx in data["negative_pkl"]:
+                        data["negative_pkl"].remove(process.candidate_image2_idx)
+                else:
+                    print('Error')
+        else:
+            selected_concept_group = process.idx_to_chain[concept_choices].split("_")
+            selected_paired_pkl = data['selected_paired_pkl'][process.idx_to_chain[concept_choices]]
+            if image_filtered == "Image1":
+                print(process.candidate_image1_idx)
+                if process.candidate_image1_group == "positive1":
+                    if process.candidate_image1_idx in selected_paired_pkl[selected_concept_group[0]]:
+                        selected_paired_pkl[selected_concept_group[0]].remove(process.candidate_image1_idx)
+                elif process.candidate_image1_group == "positive2":
+                    if process.candidate_image1_idx in selected_paired_pkl[selected_concept_group[1]]:
+                        selected_paired_pkl[selected_concept_group[1]].remove(process.candidate_image1_idx)
+                elif process.candidate_image1_group == "positive_com":
+                    if process.candidate_image1_idx in selected_paired_pkl[process.idx_to_chain[concept_choices]]:
+                        selected_paired_pkl[process.idx_to_chain[concept_choices]].remove(process.candidate_image1_idx)
+                elif process.candidate_image1_group == "negative":
+                    if process.candidate_image1_idx in data["negative_pkl"]:
+                        data["negative_pkl"].remove(process.candidate_image1_idx)
+                else:
+                    print('Error')
+            else:
+                print(process.candidate_image2_idx)
+                if process.candidate_image2_group == "positive1":
+                    if process.candidate_image2_idx in selected_paired_pkl[selected_concept_group[0]]:
+                        selected_paired_pkl[selected_concept_group[0]].remove(process.candidate_image2_idx)
+                elif process.candidate_image2_group == "positive2":
+                    if process.candidate_image2_idx in selected_paired_pkl[selected_concept_group[1]]:
+                        selected_paired_pkl[selected_concept_group[1]].remove(process.candidate_image2_idx)
+                elif process.candidate_image2_group == "positive_com":
+                    if process.candidate_image2_idx in selected_paired_pkl[process.idx_to_chain[concept_choices]]:
+                        selected_paired_pkl[process.idx_to_chain[concept_choices]].remove(process.candidate_image2_idx)
+                elif process.candidate_image2_group == "negative":
+                    if process.candidate_image2_idx in data["negative_pkl"]:
+                        data["negative_pkl"].remove(process.candidate_image2_idx)
+                else:
+                    print('Error')
+        with open(pkl_path, "wb") as f:
+            pickle.dump(data, f)
+            
+    elif dataset == "Pangea":
+        pkl_path = "Data/pangea/pangea_test_refined.pkl"
+        with open(pkl_path,"rb") as f:
+            data = pickle.load(f)
+        if exp_mode == "One concept":
+            if image_filtered == "Image1":
+                print(process.candidate_image1_idx)
+                if process.candidate_image1_group == "positive":
+                    if process.candidate_image1_idx in data['selected_pkl'][process.idx_to_chain[concept_choices]]:
+                        data['selected_pkl'][process.idx_to_chain[concept_choices]].remove(process.candidate_image1_idx)
+                elif process.candidate_image1_group == "negative":
+                    if process.candidate_image1_idx in data["negative_pkl"]:
+                        data["negative_pkl"].remove(process.candidate_image1_idx)
+                else:
+                    print('Error')
+            else:
+                print(process.candidate_image2_idx)
+                if process.candidate_image2_group == "positive":
+                    if process.candidate_image2_idx in data['selected_pkl'][process.idx_to_chain[concept_choices]]:
+                        data['selected_pkl'][process.idx_to_chain[concept_choices]].remove(process.candidate_image2_idx)
+                elif process.candidate_image2_group == "negative":
+                    if process.candidate_image2_idx in data["negative_pkl"]:
+                        data["negative_pkl"].remove(process.candidate_image2_idx)
+                else:
+                    print('Error')
+        else:
+            selected_concept_group = process.idx_to_chain[concept_choices].split("-")
+            selected_paired_pkl = data['selected_paired_pkl'][process.idx_to_chain[concept_choices]]
+            if image_filtered == "Image1":
+                print(process.candidate_image1_idx)
+                if process.candidate_image1_group == "positive1":
+                    if process.candidate_image1_idx in selected_paired_pkl[selected_concept_group[0]]:
+                        selected_paired_pkl[selected_concept_group[0]].remove(process.candidate_image1_idx)
+                elif process.candidate_image1_group == "positive2":
+                    if process.candidate_image1_idx in selected_paired_pkl[selected_concept_group[1]]:
+                        selected_paired_pkl[selected_concept_group[1]].remove(process.candidate_image1_idx)
+                elif process.candidate_image1_group == "positive_com":
+                    if process.candidate_image1_idx in selected_paired_pkl[process.idx_to_chain[concept_choices]]:
+                        selected_paired_pkl[process.idx_to_chain[concept_choices]].remove(process.candidate_image1_idx)
+                elif process.candidate_image1_group == "negative":
+                    if process.candidate_image1_idx in data["negative_pkl"]:
+                        data["negative_pkl"].remove(process.candidate_image1_idx)
+                else:
+                    print('Error')
+            else:
+                print(process.candidate_image2_idx)
+                if process.candidate_image2_group == "positive1":
+                    if process.candidate_image2_idx in selected_paired_pkl[selected_concept_group[0]]:
+                        selected_paired_pkl[selected_concept_group[0]].remove(process.candidate_image2_idx)
+                elif process.candidate_image2_group == "positive2":
+                    if process.candidate_image2_idx in selected_paired_pkl[selected_concept_group[1]]:
+                        selected_paired_pkl[selected_concept_group[1]].remove(process.candidate_image2_idx)
+                elif process.candidate_image2_group == "positive_com":
+                    if process.candidate_image2_idx in selected_paired_pkl[process.idx_to_chain[concept_choices]]:
+                        selected_paired_pkl[process.idx_to_chain[concept_choices]].remove(process.candidate_image2_idx)
+                elif process.candidate_image2_group == "negative":
+                    if process.candidate_image2_idx in data["negative_pkl"]:
+                        data["negative_pkl"].remove(process.candidate_image2_idx)
+                else:
+                    print('Error')
+        with open(pkl_path, "wb") as f:
+            pickle.dump(data, f)
+    else:
+        print("Error")
+                        
+    return None
+        
+with gr.Blocks() as demo:
+    
+    title_markdown = ("""
+        # MLLM Associstion
+        [[Paper]](https://mvig-rhos.com)  [[Code]](https://github.com/lihong2303/MLLMs_Association)
+    """)
+        # ![RHOS]("images/android-chrome-192x192.png") 
+    cur_dir = os.path.dirname(os.path.abspath(__file__))
+    gr.Markdown(title_markdown)
+    
+    with gr.Row():
+        with gr.Column():
+            raw_image = gr.Image(label="Raw Image",interactive=False)
+        with gr.Column():
+            candidate_image1 = gr.Image(label="Candidate Image 1",interactive=False)
+        with gr.Column():
+            candidate_image2 = gr.Image(label="Candidate Image 2",interactive=False)
+            
+    with gr.Row():
+        candidate_concepts = gr.Label(value="", label="Candidate Concepts")
+        filter_Images = gr.Radio(choices=["Image1", "Image2"],label="Filter low quality image") 
+        
+    with gr.Row():
+        dataset = gr.Dropdown(choices=["ocl_attribute","ocl_affordance","hmdb", "Pangea"],label="Select a dataset",interactive=True)
+        exp_mode = gr.Dropdown(choices=["One concept","Two concepts"],label="Select a test mode",interactive=True)
+        concept_choices = gr.Dropdown(choices=[],label = "Select the chain",interactive=True)
+        
+          
+    with gr.Row():
+        select_input = gr.Radio(choices=["Image1","Image2","Uncertain"],label="Select candidate image")
+        steps = gr.Label(value="0",label="Steps")
+        show_result = gr.Label(value="",label="Selected Result")
+        # reset_button = gr.Button(text="Reset")
+        
+    
+    exp_mode.change(fn=get_concept_choices,inputs=[dataset,exp_mode],outputs=concept_choices) 
+    
+    
+    concept_choices.change(fn=load_images_and_concepts,
+                           inputs=[dataset,exp_mode,concept_choices],
+                           outputs=[raw_image,candidate_image1,candidate_image2, candidate_concepts]) 
+    filter_Images.change(fn=filter_images, inputs=[dataset, exp_mode, concept_choices, filter_Images], outputs=[filter_Images])
+    select_input.change(fn=count_and_reload_images,inputs=[dataset,exp_mode,select_input,show_result,steps,raw_image,candidate_image1,candidate_image2],outputs=[select_input,show_result,steps,raw_image,candidate_image1,candidate_image2])
+    
+demo.queue()
+
+if __name__ == "__main__":
+    demo.launch()
+    # demo.launch(server_port=6126)
+    # import argparse
+    # argparser = argparse.ArgumentParser()
+    # argparser.add_argument("--server_name", default="0.0.0.0", type=str)
+    # argparser.add_argument("--port", default="6123", type=str)
+    # args = argparser.parse_args()
+    # try:
+    #     demo.launch(server_name=args.server_name, server_port=int(args.port),share=False)
+    # except Exception as e:
+    #     args.port=int(args.port)+1
+    #     print(f"Port {args.port} is occupied, try port {args.port}")
+    #     demo.launch(server_name=args.server_name, server_port=int(args.port),share=False)
+    

pangea.py

@@ -0,0 +1,260 @@
+
+import os
+import pickle
+import numpy as np
+
+import shutil
+
+mapping_dataset_directory = {'ActvityNet_hico_style_batch1':'ActivityNet_hico_batch1','charadesEgo_hico_style':'charadesego_frame', 'HAG_hico_style_new':'hag_frame','HACS_hico_style':'hacs_frame','kinetics_hico_style':'kinetics_dataset/k700-2020/train'}
+
+train_pkl = "/home/lihong/chenyuanjie/Sandwich/Data/B123_train_KIN-FULL_with_node.pkl"
+split_test_path = "/home/lihong/chenyuanjie/Sandwich/Data/B123_test_KIN-FULL_with_node.pkl"
+
+with open(split_test_path, 'rb') as f:
+    data = pickle.load(f)
+
+
+# split_test_pkl = []
+
+# action_num_dict = {}
+
+# for data_idx, data_item in enumerate(data):
+#     if data_item[0] in mapping_dataset_directory.keys():
+#         dataset = mapping_dataset_directory[data_item[0]]
+#     else:
+#         dataset = data_item[0]
+    
+#     orig_label = data_item[2]
+#     node_labels = data_item[3]
+    
+#     for nod_lab in node_labels:
+#         if nod_lab in action_num_dict.keys():
+#             action_num_dict[nod_lab] += 1
+#         else:
+#             action_num_dict[nod_lab] = 1
+    
+#     if data_idx %1000 == 0:
+#         print(len(data),data_idx)
+        
+
+# current_action_num_dict = {}
+# dataset_list = []
+# for data_idx, data_item in enumerate(data):
+#     if data_item[0] in mapping_dataset_directory.keys():
+#         dataset = mapping_dataset_directory[data_item[0]]
+#     else:
+#         dataset = data_item[0]
+#     image_path = '/data/xiaoqian/Images/' + dataset + '/' + data_item[1]
+#     if not os.path.isfile(image_path):
+#         if not dataset in dataset_list:
+#             dataset_list.append(dataset)
+#         continue
+    
+#     orig_label = data_item[2]
+#     node_labels = data_item[3]
+#     flag = False
+#     for nod_lab in node_labels:
+#         if nod_lab in current_action_num_dict.keys():
+#             if current_action_num_dict[nod_lab] < action_num_dict[nod_lab] * 0.1 and not flag:
+#                 split_test_pkl.append(data_item)
+#                 flag = True
+#             current_action_num_dict[nod_lab] += 1
+#         else:
+#             split_test_pkl.append(data_item)
+#             flag = True
+#             current_action_num_dict[nod_lab] = 1
+    
+#     if data_idx % 1000 == 0:
+#         print(len(data),data_idx)
+        
+# print(action_num_dict, current_action_num_dict)
+# with open(split_test_path, 'wb') as f:
+#     pickle.dump(split_test_pkl, f)
+# exit()
+
+## mapping node to idx
+mapping_node_index = pickle.load(open("/home/lihong/chenyuanjie/Sandwich/Data/mapping_node_index.pkl", "rb"))
+verbnet_topology = pickle.load(open("/home/lihong/chenyuanjie/Sandwich/Data/verbnet_topology_898.pkl", "rb"))
+
+Father2Son, objects = verbnet_topology["Father2Son"], verbnet_topology["objects"]
+
+objects = np.array(objects)
+objects_290 = objects[mapping_node_index]
+
+object_to_idx = {obj: idx for idx, obj in enumerate(objects_290)}
+
+# filtered_objects = [obj.split("-")[0] for obj in objects_290]
+
+selected_list = ["hit", "push","run","dress","drive","cook","throw","build","shake","cut"]
+true_selected_list = ["hit-18.1","push-12","run-51.3.2","dress-41.1.1-1-1","drive-11.5","cooking-45.3","throw-17.1-1","build-26.1","shake-22.3-2","cut-21.1-1"]
+true_selected_list_id = [object_to_idx[node] for node in true_selected_list]
+true_selected_paired_list = ['run-51.3.2_hit-18.1', 'drive-11.5_dress-41.1.1-1-1', 'cooking-45.3_build-26.1','shake-22.3-2_cut-21.1-1'] #,'throw-17.1-1_push-12'
+true_label = {}
+
+# {'hit-18.1cut-21.1-1': 86808, 'hit-18.1drive-11.5': 14935, 'hit-18.1run-51.3.2': 34237}
+# {'run-51.3.2run-51.3.2': 341324, 'run-51.3.2hit-18.1': 34237, 'run-51.3.2cut-21.1-1': 20389}
+# {'dress-41.1.1-1-1dress-41.1.1-1-1': 470063, 'dress-41.1.1-1-1run-51.3.2': 63862, 'dress-41.1.1-1-1cut-21.1-1': 47727, 'dress-41.1.1-1-1drive-11.5': 24965, 'dress-41.1.1-1-1hit-18.1': 23118, 'dress-41.1.1-1-1push-12': 11982, 'dress-41.1.1-1-1cooking-45.3': 469, 'dress-41.1.1-1-1build-26.1': 306}
+# {'drive-11.5drive-11.5': 238175, 'drive-11.5build-26.1': 15223, 'drive-11.5hit-18.1': 14935, 'drive-11.5cut-21.1-1': 30031, 'drive-11.5dress-41.1.1-1-1': 24965}
+# {'cooking-45.3cooking-45.3': 68577, 'cooking-45.3build-26.1': 37668, 'cooking-45.3cut-21.1-1': 15072, 'cooking-45.3dress-41.1.1-1-1': 469}
+# {'throw-17.1-1throw-17.1-1': 394887, 'throw-17.1-1hit-18.1': 92553, 'throw-17.1-1drive-11.5': 30348, 'throw-17.1-1dress-41.1.1-1-1': 97911, 'throw-17.1-1run-51.3.2': 30097, 'throw-17.1-1push-12': 20854, 'throw-17.1-1cut-21.1-1': 20714}
+# {'build-26.1cooking-45.3': 37668, 'build-26.1build-26.1': 95743, 'build-26.1drive-11.5': 15223, 'build-26.1cut-21.1-1': 23454, 'build-26.1shake-22.3-2': 23015, 'build-26.1dress-41.1.1-1-1': 306}
+# {'shake-22.3-2build-26.1': 23015, 'shake-22.3-2shake-22.3-2': 23015, 'shake-22.3-2cut-21.1-1': 13005}
+# {'cut-21.1-1cut-21.1-1': 553752, 'cut-21.1-1hit-18.1': 86808, 'cut-21.1-1drive-11.5': 30031, 'cut-21.1-1dress-41.1.1-1-1': 47727, 'cut-21.1-1build-26.1': 23454, 'cut-21.1-1cooking-45.3': 15072, 'cut-21.1-1run-51.3.2': 20389, 'cut-21.1-1throw-17.1-1': 20714, 'cut-21.1-1shake-22.3-2': 13005}
+
+selected_pkl = {}
+selected_paired_pkl = {}
+pangea_pkl = {}
+
+negative_pkl = []
+
+dataset_list = []
+num_images = 0
+
+for data_idx, data_item in enumerate(data):
+    save_flag = False
+    if data_item[0] in mapping_dataset_directory.keys():
+        dataset = mapping_dataset_directory[data_item[0]]
+    else:
+        dataset = data_item[0]
+    
+    image_path = '/data/xiaoqian/Images/' + dataset + '/' + data_item[1]
+    
+    if not os.path.isfile(image_path):
+        if not dataset in dataset_list:
+            dataset_list.append(dataset)
+        continue
+    
+    print(data_item)
+    exit()
+    orig_label = data_item[2]
+    node_labels = data_item[3]
+    node_labels_id = [object_to_idx[node] for node in node_labels]
+    
+    co_objects = list(set(node_labels_id).intersection(set(true_selected_list_id)))
+    
+    if len(co_objects) > 0:
+        
+        for sel_paired_objects in true_selected_list:
+            if object_to_idx[sel_paired_objects] in co_objects:
+                
+                if sel_paired_objects not in selected_pkl.keys():
+                    selected_pkl[sel_paired_objects] = [data_idx]
+                    save_flag = True
+                else:
+                    if len(selected_pkl[sel_paired_objects]) < 2000:
+                        save_flag = True
+                        selected_pkl[sel_paired_objects].append(data_idx)
+        
+        
+        for sel_paired_objects in true_selected_paired_list:
+            sel_obj1, sel_obj2 = sel_paired_objects.split("_")
+            
+            if object_to_idx[sel_obj1] in co_objects and object_to_idx[sel_obj2] in co_objects:
+                
+                if sel_paired_objects not in selected_paired_pkl.keys():
+                    selected_paired_pkl[sel_paired_objects] = {}
+                if sel_paired_objects not in selected_paired_pkl[sel_paired_objects].keys():
+                    save_flag = True
+                    selected_paired_pkl[sel_paired_objects][sel_paired_objects] = [data_idx]
+                else:
+                    if len(selected_paired_pkl[sel_paired_objects][sel_paired_objects]) < 2000:
+                        save_flag = True
+                        selected_paired_pkl[sel_paired_objects][sel_paired_objects].append(data_idx)
+            elif object_to_idx[sel_obj1] in co_objects:
+                
+                if sel_paired_objects not in selected_paired_pkl.keys():
+                    selected_paired_pkl[sel_paired_objects] = {}
+                if sel_obj1 not in selected_paired_pkl[sel_paired_objects].keys():
+                    save_flag = True
+                    selected_paired_pkl[sel_paired_objects][sel_obj1] = [data_idx]
+                else:
+                    if len(selected_paired_pkl[sel_paired_objects][sel_obj1]) < 2000:
+                        save_flag = True
+                        selected_paired_pkl[sel_paired_objects][sel_obj1].append(data_idx)
+            elif object_to_idx[sel_obj2] in co_objects:
+                
+                if sel_paired_objects not in selected_paired_pkl.keys():
+                    selected_paired_pkl[sel_paired_objects] = {}
+                if sel_obj2 not in selected_paired_pkl[sel_paired_objects].keys():
+                    save_flag = True
+                    selected_paired_pkl[sel_paired_objects][sel_obj2] = [data_idx]
+                else:
+                    if len(selected_paired_pkl[sel_paired_objects][sel_obj2]) < 2000:
+                        save_flag = True
+                        selected_paired_pkl[sel_paired_objects][sel_obj2].append(data_idx)
+    else:
+        if len(negative_pkl) < 3000:
+            
+            neg_flag = False
+            for sel_list in selected_list:
+                for nod_lab in node_labels:
+                    if sel_list in nod_lab:
+                        neg_flag = True
+                        break
+                    if neg_flag:
+                        break
+            
+            if not neg_flag:
+                save_flag = True
+                negative_pkl.append(data_idx)
+    if save_flag:
+        num_images += 1
+        if not os.path.exists(os.path.dirname(os.path.join("/home/lihong/workspace/pangea/pangea", dataset, data_item[1]))):
+            os.makedirs(os.path.dirname(os.path.join("/home/lihong/workspace/pangea/pangea", dataset, data_item[1])))
+        shutil.copy(image_path, os.path.join("/home/lihong/workspace/pangea/pangea", dataset, data_item[1]))
+    
+    if data_idx % 1000 == 0:
+        print(len(data),data_idx)
+
+for name in selected_pkl.keys():
+    print(f"selected {name} affordance has {len(selected_pkl[name])} objects")
+    
+for name in selected_paired_pkl.keys():
+    for sub_name in selected_paired_pkl[name].keys():
+        print(f"selected {name} paired actions {sub_name} has {len(selected_paired_pkl[name][sub_name])} objects")
+
+print("negative_pkl has {} objects".format(len(negative_pkl)))
+print("num_images has {} objects".format(num_images))
+
+pangea_pkl["selected_pkl"] = selected_pkl
+pangea_pkl["selected_paired_pkl"] = selected_paired_pkl
+pangea_pkl["negative_pkl"] = negative_pkl
+
+with open(os.path.join("/home/lihong/workspace/pangea","pangea_test.pkl"),"wb") as fp:
+    pickle.dump(pangea_pkl,fp)
+
+print(dataset_list)
+
+exit()         
+selected_images = {}
+
+save_data = []
+
+for data_idx, data_item in enumerate(data):
+    if data_item[0] in mapping_dataset_directory.keys():
+        dataset = mapping_dataset_directory[data_item[0]]
+    else:
+        dataset = data_item[0]
+    image_path = '/data/xiaoqian/Images/' + dataset + '/' + data_item[1]
+    
+    if not os.path.isfile(image_path):
+        if not dataset in dataset_list:
+            dataset_list.append(dataset)
+        continue
+    
+    orig_label = data_item[2]
+    node_labels = data_item[3]
+    if true_selected_list[9] in node_labels:
+        for i in range(len(true_selected_list)):
+            if true_selected_list[i] in node_labels:
+                if true_selected_list[9]+ true_selected_list[i] in selected_images.keys():
+                    selected_images[true_selected_list[9]+ true_selected_list[i]] += 1
+                else:
+                    selected_images[true_selected_list[9]+ true_selected_list[i]] = 1
+    
+    if data_idx %1000 == 0:
+        print(data_idx)
+    
+    
+print(selected_images)
+

preprocess_ocl_data.py

@@ -0,0 +1,207 @@
+import os
+import json
+import random
+import pickle
+
+import shutil
+
+
+def load_pickle_and_assign_split(pkl_dir,split):
+    pkl_path = os.path.join(pkl_dir, f"OCL_annot_{split}.pkl")
+    
+    with open(pkl_path, 'rb') as fp:
+        pkl = pickle.load(fp)
+    for x in pkl:
+        x['split'] = split
+    return pkl
+def load_class_json(name):
+    with open(os.path.join("/home/lixingyu/workspace_lxy2/Data/OCL_data/data/resources",f"OCL_class_{name}.json"),"r") as fp:
+        return json.load(fp)
+
+pkl_dir = "/home/lixingyu/workspace_lxy2/Data/OCL_data/data/resources"
+
+attrs = load_class_json("attribute")
+aff_dict = load_class_json("affordance")
+
+aff = []
+for aff_item in aff_dict:
+    # aff.append(aff_item["word"])
+    if aff_item["word"][0] not in aff:
+        aff.append(aff_item["word"][0])
+    else:
+        if len(aff_item["word"]) > 1:
+            random_aff = random.choice(aff_item["word"])
+            while random_aff in aff and random_aff == aff_item["word"][0]:
+                random_aff = random.choice(aff_item["word"])
+            aff.append(random_aff)
+        else:
+            if len(aff_dict[aff.index(aff_item["word"][0])]["word"]) > 1:
+                random_aff = random.choice(aff_dict[aff.index(aff_item["word"][0])]["word"])
+                while random_aff in aff and random_aff == aff_item["word"][0]:
+                    random_aff = random.choice(aff_dict[aff.index(aff_item["word"][0])]["word"])
+                    
+                aff[aff.index(aff_item["word"][0])] = random_aff
+                aff.append(aff_item["word"][0])
+            else:
+                aff.append(aff_item["word"][0] + '_1')
+        
+
+print(len(attrs))
+exit()
+
+attrs_2_idx = {attr_item:idx for idx,attr_item in enumerate(attrs)}
+
+aff_2_idx = {aff_item:idx for idx,aff_item in enumerate(aff)}
+
+selected_affordance = ['break', 'carry', 'clean','close','cut','eat','open','push','sit','write']
+selected_paired_affs = ['sit-write','push-carry','cut-clean','open-break', 'cut-close']
+selected_aff_id = [aff_2_idx[aff_item] for aff_item in selected_affordance]
+ocl_test_aff_pkl = {}
+selected_pkl = {}
+selected_paired_pkl = {}
+negative_aff = []
+
+for aff_idx, aff_item in enumerate(aff_dict):
+    
+    aff_flag = False
+    for sel_aff in selected_affordance:
+        if sel_aff in aff_item["word"]:
+            aff_flag = True
+            break
+    if not aff_flag:
+        negative_aff.append(aff[aff_idx])
+      
+negative_aff_id = [aff_2_idx[aff_item] for aff_item in negative_aff]
+
+
+negative_pkl = []
+for data_idx,each_item in enumerate(test_pkl):
+    if os.path.exists(os.path.join("/home/lixingyu/workspace_lxy2/Data/OCL_data/data",each_item["name"])):
+        for obj_idx,each_object in enumerate(each_item['objects']):
+            if (abs(each_object['box'][0] - each_object['box'][2])) * (abs(each_object['box'][1] - each_object['box'][3])) > 50000:
+                if len(list(set(each_object['aff']).intersection(set(selected_aff_id)))) > 0:
+                    for each_sel_aff in selected_affordance:
+                        if aff_2_idx[each_sel_aff] in each_object['aff']:
+                            if each_sel_aff not in selected_pkl.keys():
+                                selected_pkl[each_sel_aff] = [[data_idx, obj_idx]]
+                            else:
+                                selected_pkl[each_sel_aff].append([data_idx, obj_idx])
+                    
+                    for each_paired_sel_aff in selected_paired_affs:
+                        aff1,aff2 = each_paired_sel_aff.split("-")
+                        if aff_2_idx[aff1] in each_object['aff'] and aff_2_idx[aff2] in each_object['aff']:
+                            if each_paired_sel_aff not in selected_paired_pkl.keys():
+                                selected_paired_pkl[each_paired_sel_aff] = {}
+                            if each_paired_sel_aff not in selected_paired_pkl[each_paired_sel_aff].keys():
+                                selected_paired_pkl[each_paired_sel_aff][each_paired_sel_aff] = [[data_idx, obj_idx]]
+                            else:
+                                selected_paired_pkl[each_paired_sel_aff][each_paired_sel_aff].append([data_idx, obj_idx])
+                        elif aff_2_idx[aff1] in each_object['aff']:
+                            if each_paired_sel_aff not in selected_paired_pkl.keys():
+                                selected_paired_pkl[each_paired_sel_aff] = {}
+                            if aff1 not in selected_paired_pkl[each_paired_sel_aff].keys():
+                                selected_paired_pkl[each_paired_sel_aff][aff1] = [[data_idx, obj_idx]]
+                            else:
+                                selected_paired_pkl[each_paired_sel_aff][aff1].append([data_idx, obj_idx])
+                        elif aff_2_idx[aff2] in each_object['aff']:
+                            if each_paired_sel_aff not in selected_paired_pkl.keys():
+                                selected_paired_pkl[each_paired_sel_aff] = {}
+                            if aff2 not in selected_paired_pkl[each_paired_sel_aff].keys():
+                                selected_paired_pkl[each_paired_sel_aff][aff2] = [[data_idx, obj_idx]]
+                            else:
+                                selected_paired_pkl[each_paired_sel_aff][aff2].append([data_idx, obj_idx])
+                    
+                else:
+                    if len(list(set(each_object['aff']).intersection(set(negative_aff_id)))) == len(each_object['aff']):
+                        negative_pkl.append([data_idx,obj_idx])
+                if not os.path.exists(os.path.dirname(os.path.join("/home/lixingyu/workspace_lxy2/Data/OCL_data/data/saved_test_data",each_item["name"]))):
+                    os.makedirs(os.path.dirname(os.path.join("/home/lixingyu/workspace_lxy2/Data/OCL_data/data/saved_test_data",each_item["name"])))
+                shutil.copy(os.path.join("/home/lixingyu/workspace_lxy2/Data/OCL_data/data",each_item["name"]),os.path.join("/home/lixingyu/workspace_lxy2/Data/OCL_data/data/saved_test_data",each_item["name"]))
+
+print("negative_pkl has {} objects".format(len(negative_pkl)))
+
+for name in selected_pkl.keys():
+    print(f"selected {name} affordance has {len(selected_pkl[name])} objects")
+
+for name in selected_paired_pkl.keys():
+    for sub_name in selected_paired_pkl[name].keys():
+        print(f"selected {name} paired affordance {sub_name} has {len(selected_paired_pkl[name][sub_name])} objects")
+
+ocl_test_aff_pkl["selected_individual_pkl"] = selected_pkl
+ocl_test_aff_pkl["selected_paired_pkl"] = selected_paired_pkl
+ocl_test_aff_pkl["negative_pkl"] = negative_pkl
+
+with open(os.path.join(pkl_dir,"OCL_selected_test_affordance.pkl"),"wb") as fp:
+    pickle.dump(ocl_test_aff_pkl,fp)
+
+
+selected_attrs = ['wooden', 'metal', 'flying', 'ripe', 'fresh', 'natural', 'cooked', 'painted', 'rusty', 'furry'] 
+selected_attrs_id = [attrs_2_idx[attr_item] for attr_item in selected_attrs]
+selected_paired_attrs = ["furry-metal","fresh-cooked","natural-ripe","painted-rusty"]
+
+selected_affs = []
+selected_paired_affs = []
+
+
+ocl_test_attr_pkl = {}
+
+selected_pkl = {}
+selected_paired_pkl = {}
+
+negative_pkl = []
+for data_idx,each_item in enumerate(test_pkl):
+    if os.path.exists(os.path.join("/home/lixingyu/workspace_lxy2/Data/OCL_data/data",each_item["name"])):
+        for obj_idx,each_object in enumerate(each_item['objects']):
+            if (abs(each_object['box'][0] - each_object['box'][2])) * (abs(each_object['box'][1] - each_object['box'][3])) > 50000:
+                if len(list(set(each_object['attr']).intersection(set(selected_attrs_id)))) > 0:
+                    for each_sel_attr in selected_attrs:
+                        if attrs_2_idx[each_sel_attr] in each_object['attr']:
+                            if each_sel_attr not in selected_pkl.keys():
+                                selected_pkl[each_sel_attr] = [[data_idx, obj_idx]]
+                            else:
+                                selected_pkl[each_sel_attr].append([data_idx, obj_idx])
+                    
+                    for each_paired_sel_attr in selected_paired_attrs:
+                        attr1,attr2 = each_paired_sel_attr.split("-")
+                        if attrs_2_idx[attr1] in each_object['attr'] and attrs_2_idx[attr2] in each_object['attr']:
+                            if each_paired_sel_attr not in selected_paired_pkl.keys():
+                                selected_paired_pkl[each_paired_sel_attr] = {}
+                            if each_paired_sel_attr not in selected_paired_pkl[each_paired_sel_attr].keys():
+                                selected_paired_pkl[each_paired_sel_attr][each_paired_sel_attr] = [[data_idx, obj_idx]]
+                            else:
+                                selected_paired_pkl[each_paired_sel_attr][each_paired_sel_attr].append([data_idx, obj_idx])
+                        elif attrs_2_idx[attr1] in each_object['attr']:
+                            if each_paired_sel_attr not in selected_paired_pkl.keys():
+                                selected_paired_pkl[each_paired_sel_attr] = {}
+                            if attr1 not in selected_paired_pkl[each_paired_sel_attr].keys():
+                                selected_paired_pkl[each_paired_sel_attr][attr1] = [[data_idx, obj_idx]]
+                            else:
+                                selected_paired_pkl[each_paired_sel_attr][attr1].append([data_idx, obj_idx])
+                        elif attrs_2_idx[attr2] in each_object['attr']:
+                            if each_paired_sel_attr not in selected_paired_pkl.keys():
+                                selected_paired_pkl[each_paired_sel_attr] = {}
+                            if attr2 not in selected_paired_pkl[each_paired_sel_attr].keys():
+                                selected_paired_pkl[each_paired_sel_attr][attr2] = [[data_idx, obj_idx]]
+                            else:
+                                selected_paired_pkl[each_paired_sel_attr][attr2].append([data_idx, obj_idx])
+                    
+                else:
+                    negative_pkl.append([data_idx,obj_idx])
+            
+
+print("finshed!!!")
+print("negative_pkl has {} objects".format(len(negative_pkl)))
+for name in selected_pkl.keys():
+    print(f"selected {name} attribute has {len(selected_pkl[name])} objects")
+
+for name in selected_paired_pkl.keys():
+    for sub_name in selected_paired_pkl[name].keys():
+        print(f"selected {name} paired attribute {sub_name} has {len(selected_paired_pkl[name][sub_name])} objects")
+    
+ocl_test_attr_pkl["selected_individual_pkl"] = selected_pkl
+ocl_test_attr_pkl["selected_paired_pkl"] = selected_paired_pkl
+ocl_test_attr_pkl["negative_pkl"] = negative_pkl
+with open(os.path.join(pkl_dir,"OCL_selected_test_attribute.pkl"),"wb") as fp:
+    pickle.dump(ocl_test_attr_pkl,fp)
+
+

requirements.txt

@@ -0,0 +1 @@
+huggingface_hub==0.22.2