debug pipeline

app_split.py

@@ -5,5 +5,5 @@ from runners.data_spliter import DataSpliter
 class DataSplitApp:
     @staticmethod
     def start():
-        DataSpliter("configs/server/split_dataset_config.yaml").run()
+        DataSpliter("configs/server/server_split_dataset_config.yaml").run()
         

configs/server/server_split_dataset_config.yaml

@@ -10,13 +10,13 @@ runner:
     root_dir: "experiments"
   
   split: # 
-    root_dir: "/home/data/hofee/project/nbv_rec/data/nbv_rec_data_512_preproc_npy"
+    root_dir: "/data/hofee/data/packed_preprocessed_data"
     type: "unseen_instance" # "unseen_category"
     datasets:
       OmniObject3d_train: 
-        path: "../data/sample_for_training_preprocessed/OmniObject3d_train.txt"
+        path: "/data/hofee/data/OmniObject3d_train.txt"
         ratio: 0.9
 
       OmniObject3d_test: 
-        path: "../data/sample_for_training_preprocessed/OmniObject3d_test.txt"
+        path: "/data/hofee/data/OmniObject3d_test.txt"
         ratio: 0.1

configs/server/server_train_config.yaml

@@ -7,13 +7,13 @@ runner:
     parallel: False
     
   experiment:
-    name: full_w_global_feat_wo_local_pts_feat
+    name: test_new_pipeline_train_overfit
     root_dir: "experiments"
     use_checkpoint: False
     epoch: -1 # -1 stands for last epoch
     max_epochs: 5000
     save_checkpoint_interval: 1
-    test_first: True  
+    test_first: False  
 
   train:
     optimizer:
@@ -25,46 +25,46 @@ runner:
   test:
     frequency: 3 # test frequency
     dataset_list:
-      - OmniObject3d_test
+      #- OmniObject3d_test
       - OmniObject3d_val
 
-  pipeline: nbv_reconstruction_global_pts_pipeline
+  pipeline: nbv_reconstruction_global_pts_n_num_pipeline
   
 dataset:
   OmniObject3d_train:
-    root_dir: "/home/data/hofee/project/nbv_rec/data/nbv_rec_data_512_preproc_npy"
+    root_dir: "/data/hofee/data/packed_preprocessed_data"
     model_dir: "../data/scaled_object_meshes"
     source: nbv_reconstruction_dataset
-    split_file: "/home/data/hofee/project/nbv_rec/data/OmniObject3d_train.txt"
+    split_file: "/data/hofee/data/OmniObject3d_train_overfit.txt"
     type: train
     cache: True
     ratio: 1
-    batch_size: 160
+    batch_size: 16
     num_workers: 16
     pts_num: 4096
     load_from_preprocess: True
 
-  OmniObject3d_test:
-    root_dir: "/home/data/hofee/project/nbv_rec/data/nbv_rec_data_512_preproc_npy"
-    model_dir: "../data/scaled_object_meshes"
-    source: nbv_reconstruction_dataset
-    split_file: "/home/data/hofee/project/nbv_rec/data/OmniObject3d_test.txt"
-    type: test
-    cache: True
-    filter_degree: 75
-    eval_list:
-      - pose_diff
-    ratio: 0.05
-    batch_size: 160
-    num_workers: 12
-    pts_num: 4096
-    load_from_preprocess: True
+  # OmniObject3d_test:
+  #   root_dir: "/data/hofee/data/packed_preprocessed_data"
+  #   model_dir: "../data/scaled_object_meshes"
+  #   source: nbv_reconstruction_dataset
+  #   split_file: "/data/hofee/data/OmniObject3d_test.txt"
+  #   type: test
+  #   cache: True
+  #   filter_degree: 75
+  #   eval_list:
+  #     - pose_diff
+  #   ratio: 0.05
+  #   batch_size: 160
+  #   num_workers: 12
+  #   pts_num: 4096
+  #   load_from_preprocess: True
 
   OmniObject3d_val:
-    root_dir: "/home/data/hofee/project/nbv_rec/data/nbv_rec_data_512_preproc_npy"
+    root_dir: "/data/hofee/data/packed_preprocessed_data"
     model_dir: "../data/scaled_object_meshes"
     source: nbv_reconstruction_dataset
-    split_file: "/home/data/hofee/project/nbv_rec/data/OmniObject3d_train.txt"
+    split_file: "/data/hofee/data/OmniObject3d_train_overfit.txt"
     type: test
     cache: True
     filter_degree: 75
@@ -96,6 +96,15 @@ pipeline:
     eps: 1e-5
     global_scanned_feat: True
 
+  nbv_reconstruction_global_pts_n_num_pipeline:
+    modules:
+      pts_encoder: pointnet_encoder
+      transformer_seq_encoder: transformer_seq_encoder
+      pose_encoder: pose_encoder
+      view_finder: gf_view_finder
+      pts_num_encoder: pts_num_encoder
+    eps: 1e-5
+    global_scanned_feat: True
 
     
 module:
@@ -107,7 +116,7 @@ module:
     feature_transform: False
 
   transformer_seq_encoder:
-    embed_dim: 1344
+    embed_dim: 384
     num_heads: 4
     ffn_dim: 256
     num_layers: 3
@@ -116,7 +125,7 @@ module:
   gf_view_finder:
     t_feat_dim: 128
     pose_feat_dim: 256
-    main_feat_dim: 2048
+    main_feat_dim: 3072
     regression_head: Rx_Ry_and_T
     pose_mode: rot_matrix
     per_point_feature: False
@@ -128,6 +137,9 @@ module:
     pose_dim: 9
     out_dim: 256
 
+  pts_num_encoder:
+    out_dim: 64
+
 loss_function:
   gf_loss:
 

core/global_pts_n_num_pipeline.py

@@ -117,22 +117,20 @@ class NBVReconstructionGlobalPointsPipeline(nn.Module):
             for seq_idx in range(seq_len): 
                 partial_idx_in_combined_pts = scanned_mask == seq_idx # Ndarray(V), N->V idx mask
                 partial_perpoint_feat = perpoint_scanned_feat[partial_idx_in_combined_pts] # Ndarray(V x Dl)
-                partial_feat = torch.mean(partial_perpoint_feat, dim=0)[0] # Tensor(Dl)
+                partial_feat = torch.mean(partial_perpoint_feat, dim=0) # Tensor(Dl)
                 partial_feat_seq.append(partial_feat)
                 scanned_target_pts_num.append(partial_perpoint_feat.shape[0])
                 
-            scanned_target_pts_num = torch.tensor(scanned_target_pts_num, dtype=torch.int32).to(device) # Tensor(S)
+            scanned_target_pts_num = torch.tensor(scanned_target_pts_num, dtype=torch.float32).to(device).unsqueeze(-1) # Tensor(S)
             scanned_n_to_world_pose_9d = scanned_n_to_world_pose_9d.to(device)  # Tensor(S x 9)
             
             pose_feat_seq = self.pose_encoder.encode_pose(scanned_n_to_world_pose_9d)  # Tensor(S x Dp)
             pts_num_feat_seq = self.pts_num_encoder.encode_pts_num(scanned_target_pts_num)  # Tensor(S x Dn)
             partial_feat_seq = torch.stack(partial_feat_seq) # Tensor(S x Dl)
-            
             seq_embedding = torch.cat([pose_feat_seq, pts_num_feat_seq, partial_feat_seq], dim=-1) # Tensor(S x (Dp+Dn+Dl))
             embedding_list_batch.append(seq_embedding) # List(B): Tensor(S x (Dp+Dn+Dl))
 
         seq_feat = self.transformer_seq_encoder.encode_sequence(embedding_list_batch) # Tensor(B x Ds)
-
         main_feat = torch.cat([seq_feat, global_scanned_feat], dim=-1) # Tensor(B x (Ds+Dg))
 
         if torch.isnan(main_feat).any():

core/nbv_dataset.py

@@ -8,7 +8,7 @@ import torch
 import os
 import sys
 
-sys.path.append(r"/home/data/hofee/project/nbv_rec/nbv_reconstruction")
+sys.path.append(r"/data/hofee/project/nbv_rec/nbv_reconstruction")
 
 from utils.data_load import DataLoadUtil
 from utils.pose import PoseUtil
@@ -31,7 +31,7 @@ class NBVReconstructionDataset(BaseDataset):
         self.load_from_preprocess = config.get("load_from_preprocess", False)
 
         if self.type == namespace.Mode.TEST:
-            self.model_dir = config["model_dir"]
+            #self.model_dir = config["model_dir"]
             self.filter_degree = config["filter_degree"]
         if self.type == namespace.Mode.TRAIN:
             scale_ratio = 1
@@ -66,7 +66,9 @@ class NBVReconstructionDataset(BaseDataset):
                 if max_coverage_rate > scene_max_coverage_rate:
                     scene_max_coverage_rate = max_coverage_rate
                 max_coverage_rate_list.append(max_coverage_rate)
-            mean_coverage_rate = np.mean(max_coverage_rate_list)
+                
+            if max_coverage_rate_list:
+                mean_coverage_rate = np.mean(max_coverage_rate_list)
 
             for seq_idx in range(seq_num):
                 label_path = DataLoadUtil.get_label_path(
@@ -122,7 +124,7 @@ class NBVReconstructionDataset(BaseDataset):
             scanned_views_pts,
             scanned_coverages_rate,
             scanned_n_to_world_pose,
-        ) = ([], [], [], [])
+        ) = ([], [], [])
         for view in scanned_views:
             frame_idx = view[0]
             coverage_rate = view[1]
@@ -164,19 +166,14 @@ class NBVReconstructionDataset(BaseDataset):
             combined_scanned_views_pts, self.pts_num, require_idx=True
         )
         
-        combined_scanned_views_pts_mask = np.zeros(len(scanned_views_pts), dtype=np.uint8)
+        combined_scanned_views_pts_mask = np.zeros(len(combined_scanned_views_pts), dtype=np.uint8)
         
         start_idx = 0
         for i in range(len(scanned_views_pts)):
             end_idx = start_idx + len(scanned_views_pts[i])
             combined_scanned_views_pts_mask[start_idx:end_idx] = i
             start_idx = end_idx
-            
         fps_downsampled_combined_scanned_pts_mask = combined_scanned_views_pts_mask[fps_idx]
-        
-        
-        
-        
         data_item = {
             "scanned_pts": np.asarray(scanned_views_pts, dtype=np.float32), # Ndarray(S x Nv x 3)
             "scanned_pts_mask": np.asarray(fps_downsampled_combined_scanned_pts_mask,dtype=np.uint8), # Ndarray(N), range(0, S)
@@ -241,10 +238,9 @@ if __name__ == "__main__":
     torch.manual_seed(seed)
     np.random.seed(seed)
     config = {
-        "root_dir": "/home/data/hofee/project/nbv_rec/data/nbv_rec_data_512_preproc_npy",
-        "model_dir": "/home/data/hofee/project/nbv_rec/data/scaled_object_meshes",
+        "root_dir": "/data/hofee/data/packed_preprocessed_data",
         "source": "nbv_reconstruction_dataset",
-        "split_file": "/home/data/hofee/project/nbv_rec/data/OmniObject3d_test.txt",
+        "split_file": "/data/hofee/data/OmniObject3d_train.txt",
         "load_from_preprocess": True,
         "ratio": 0.5,
         "batch_size": 2,

utils/data_load.py

@@ -34,6 +34,8 @@ class DataLoadUtil:
     @staticmethod
     def get_label_num(root, scene_name):
         label_dir = os.path.join(root, scene_name, "label")
+        if not os.path.exists(label_dir):
+            return 0
         return len(os.listdir(label_dir))
 
     @staticmethod