diff --git a/configs/local/strategy_generate_config.yaml b/configs/local/strategy_generate_config.yaml index 3ef8221..3e4576d 100644 --- a/configs/local/strategy_generate_config.yaml +++ b/configs/local/strategy_generate_config.yaml @@ -12,7 +12,8 @@ runner: generate: voxel_threshold: 0.01 - overlap_threshold: 0.5 + soft_overlap_threshold: 0.3 + hard_overlap_threshold: 0.6 filter_degree: 75 to_specified_dir: True # if True, output_dir is used, otherwise, root_dir is used save_points: True @@ -20,17 +21,17 @@ runner: save_best_combined_points: False save_mesh: True overwrite: False - seq_num: 50 + seq_num: 10 dataset_list: - OmniObject3d datasets: OmniObject3d: #"/media/hofee/data/data/temp_output" - root_dir: "/media/hofee/repository/new_sample" + root_dir: "/media/hofee/repository/new_full_box_data" model_dir: "/media/hofee/data/data/scaled_object_meshes" from: 0 - to: -1 # -1 means all + to: -1 # -1 means end #output_dir: "/media/hofee/data/data/label_output" diff --git a/configs/local/view_generate_config.yaml b/configs/local/view_generate_config.yaml index aae49c0..d11eef6 100644 --- a/configs/local/view_generate_config.yaml +++ b/configs/local/view_generate_config.yaml @@ -7,9 +7,12 @@ runner: name: debug root_dir: experiments generate: + port: 5003 + from: 3000 + to: -1 # -1 means all object_dir: /media/hofee/data/data/scaled_object_meshes table_model_path: /media/hofee/data/data/others/table.obj - output_dir: /media/hofee/repository/new_nbv_reconstruction_data_512 + output_dir: /media/hofee/repository/new_full_data binocular_vision: true plane_size: 10 max_views: 512 diff --git a/runners/strategy_generator.py b/runners/strategy_generator.py index db7cc2f..21ac203 100644 --- a/runners/strategy_generator.py +++ b/runners/strategy_generator.py @@ -35,7 +35,7 @@ class StrategyGenerator(Runner): def run(self): dataset_name_list = ConfigManager.get("runner", "generate", "dataset_list") - voxel_threshold, overlap_threshold = ConfigManager.get("runner","generate","voxel_threshold"), ConfigManager.get("runner","generate","overlap_threshold") + voxel_threshold, soft_overlap_threshold, hard_overlap_threshold = ConfigManager.get("runner","generate","voxel_threshold"), ConfigManager.get("runner","generate","soft_overlap_threshold"), ConfigManager.get("runner","generate","hard_overlap_threshold") for dataset_idx in range(len(dataset_name_list)): dataset_name = dataset_name_list[dataset_idx] status_manager.set_progress("generate_strategy", "strategy_generator", "dataset", dataset_idx, len(dataset_name_list)) @@ -47,23 +47,23 @@ class StrategyGenerator(Runner): if to_idx == -1: to_idx = len(scene_name_list) cnt = 0 - total = len(scene_name_list) + total = len(scene_name_list[from_idx:to_idx]) Log.info(f"Processing Dataset: {dataset_name}, From: {from_idx}, To: {to_idx}") for scene_name in scene_name_list[from_idx:to_idx]: Log.info(f"({dataset_name})Processing [{cnt}/{total}]: {scene_name}") status_manager.set_progress("generate_strategy", "strategy_generator", "scene", cnt, total) - diag = DataLoadUtil.get_bbox_diag(model_dir, scene_name) - voxel_threshold = diag*0.02 + #diag = DataLoadUtil.get_bbox_diag(model_dir, scene_name) + voxel_threshold = 0.002 status_manager.set_status("generate_strategy", "strategy_generator", "voxel_threshold", voxel_threshold) output_label_path = DataLoadUtil.get_label_path(root_dir, scene_name,0) if os.path.exists(output_label_path) and not self.overwrite: Log.info(f"Scene <{scene_name}> Already Exists, Skip") cnt += 1 continue - try: - self.generate_sequence(root_dir, model_dir, scene_name,voxel_threshold, overlap_threshold) - except Exception as e: - Log.error(f"Scene <{scene_name}> Failed, Error: {e}") + + self.generate_sequence(root_dir, model_dir, scene_name,voxel_threshold, soft_overlap_threshold, hard_overlap_threshold) + # except Exception as e: + # Log.error(f"Scene <{scene_name}> Failed, Error: {e}") cnt += 1 status_manager.set_progress("generate_strategy", "strategy_generator", "scene", total, total) status_manager.set_progress("generate_strategy", "strategy_generator", "dataset", len(dataset_name_list), len(dataset_name_list)) @@ -76,7 +76,7 @@ class StrategyGenerator(Runner): def load_experiment(self, backup_name=None): super().load_experiment(backup_name) - def generate_sequence(self, root, model_dir, scene_name, voxel_threshold, overlap_threshold): + def generate_sequence(self, root, model_dir, scene_name, voxel_threshold, soft_overlap_threshold, hard_overlap_threshold): status_manager.set_status("generate_strategy", "strategy_generator", "scene", scene_name) frame_num = DataLoadUtil.get_scene_seq_length(root, scene_name) model_points_normals = DataLoadUtil.load_points_normals(root, scene_name) @@ -84,47 +84,83 @@ class StrategyGenerator(Runner): down_sampled_model_pts = PtsUtil.voxel_downsample_point_cloud(model_pts, voxel_threshold) display_table_info = DataLoadUtil.get_display_table_info(root, scene_name) radius = display_table_info["radius"] - top = DataLoadUtil.get_display_table_top(root, scene_name) - scan_points = ReconstructionUtil.generate_scan_points(display_table_top=top,display_table_radius=radius) + scan_points_path = os.path.join(root,scene_name, "scan_points.txt") + if os.path.exists(scan_points_path): + scan_points = np.loadtxt(scan_points_path) + else: + scan_points = ReconstructionUtil.generate_scan_points(display_table_top=0,display_table_radius=radius) + np.savetxt(scan_points_path, scan_points) pts_list = [] scan_points_indices_list = [] - + non_zero_cnt = 0 for frame_idx in range(frame_num): - if self.load_pts and os.path.exists(os.path.join(root,scene_name, "pts", f"{frame_idx}.txt")): - sampled_point_cloud = np.loadtxt(os.path.join(root,scene_name, "pts", f"{frame_idx}.txt")) - indices = np.loadtxt(os.path.join(root,scene_name, "pts", f"{frame_idx}_indices.txt")).astype(np.int32).tolist() - status_manager.set_progress("generate_strategy", "strategy_generator", "loading frame", frame_idx, frame_num) + status_manager.set_progress("generate_strategy", "strategy_generator", "loading frame", frame_idx, frame_num) + pts_path = os.path.join(root,scene_name, "pts", f"{frame_idx}.txt") + if self.load_pts and pts_path: + with open(pts_path, 'r') as f: + pts_str = f.read() + if pts_str == "": + sampled_point_cloud = np.asarray([]) + else: + sampled_point_cloud = np.loadtxt(pts_path) + indices_path = os.path.join(root,scene_name, "covered_scan_pts", f"{frame_idx}_indices.txt") + with open(indices_path, 'r') as f: + indices_str = f.read() + if indices_str == "": + indices = [] + else: + indices = np.loadtxt(indices_path).astype(np.int32).tolist() + if isinstance(indices, int): + indices = [indices] + pts_list.append(sampled_point_cloud) + if sampled_point_cloud.shape[0] != 0: + non_zero_cnt += 1 scan_points_indices_list.append(indices) else: path = DataLoadUtil.get_path(root, scene_name, frame_idx) cam_params = DataLoadUtil.load_cam_info(path, binocular=True) - status_manager.set_progress("generate_strategy", "strategy_generator", "loading frame", frame_idx, frame_num) point_cloud, display_table_pts = DataLoadUtil.get_target_point_cloud_world_from_path(path, binocular=True, get_display_table_pts=True) - sampled_point_cloud = ReconstructionUtil.filter_points(point_cloud, model_points_normals, cam_pose=cam_params["cam_to_world"], voxel_size=voxel_threshold, theta=self.filter_degree) + + if point_cloud.shape[0] != 0: + sampled_point_cloud = ReconstructionUtil.filter_points(point_cloud, model_points_normals, cam_pose=cam_params["cam_to_world"], voxel_size=voxel_threshold, theta=self.filter_degree) + non_zero_cnt += 1 + else: + sampled_point_cloud = point_cloud + covered_pts, indices = ReconstructionUtil.compute_covered_scan_points(scan_points, display_table_pts) + if self.save_pts: pts_dir = os.path.join(root,scene_name, "pts") - covered_pts_dir = os.path.join(pts_dir, "covered_scan_pts") + #display_dir = os.path.join(root,scene_name, "display_pts") + covered_pts_dir = os.path.join(root,scene_name, "covered_scan_pts") if not os.path.exists(pts_dir): os.makedirs(pts_dir) if not os.path.exists(covered_pts_dir): os.makedirs(covered_pts_dir) + # if not os.path.exists(display_dir): + # os.makedirs(display_dir) np.savetxt(os.path.join(pts_dir, f"{frame_idx}.txt"), sampled_point_cloud) + #np.savetxt(os.path.join(display_dir, f"{frame_idx}.txt"), display_table_pts) np.savetxt(os.path.join(covered_pts_dir, f"{frame_idx}.txt"), covered_pts) - np.savetxt(os.path.join(pts_dir, f"{frame_idx}_indices.txt"), indices) + np.savetxt(os.path.join(covered_pts_dir, f"{frame_idx}_indices.txt"), indices) pts_list.append(sampled_point_cloud) scan_points_indices_list.append(indices) status_manager.set_progress("generate_strategy", "strategy_generator", "loading frame", frame_num, frame_num) - seq_num = min(self.seq_num, len(pts_list)) - init_view_list = range(seq_num) + seq_num = min(self.seq_num, non_zero_cnt) + init_view_list = [] + for i in range(seq_num): + if pts_list[i].shape[0] < 100: + continue + init_view_list.append(i) seq_idx = 0 for init_view in init_view_list: status_manager.set_progress("generate_strategy", "strategy_generator", "computing sequence", seq_idx, len(init_view_list)) - limited_useful_view, _, _ = ReconstructionUtil.compute_next_best_view_sequence_with_overlap(down_sampled_model_pts, pts_list,init_view=init_view, threshold=voxel_threshold, overlap_threshold=overlap_threshold, status_info=self.status_info) + limited_useful_view, _, _ = ReconstructionUtil.compute_next_best_view_sequence_with_overlap(down_sampled_model_pts, pts_list, scan_points_indices_list = scan_points_indices_list,init_view=init_view, + threshold=voxel_threshold, soft_overlap_threshold=soft_overlap_threshold, hard_overlap_threshold= hard_overlap_threshold, scan_points_threshold=10, status_info=self.status_info) data_pairs = self.generate_data_pairs(limited_useful_view) seq_save_data = { "data_pairs": data_pairs, diff --git a/utils/data_load.py b/utils/data_load.py index 318a6d5..584f4e4 100644 --- a/utils/data_load.py +++ b/utils/data_load.py @@ -322,7 +322,7 @@ class DataLoadUtil: random_downsample_N=65536, voxel_size=0.005, target_mask_label=(0, 255, 0, 255), - display_table_mask_label=(255, 0, 0, 255), + display_table_mask_label=(0, 0, 255, 255), get_display_table_pts=False ): cam_info = DataLoadUtil.load_cam_info(path, binocular=binocular) @@ -369,6 +369,12 @@ class DataLoadUtil: mask_R, display_table_mask_label, )["points_world"] + display_pts_L = PtsUtil.random_downsample_point_cloud( + display_pts_L, random_downsample_N + ) + point_cloud_R = PtsUtil.random_downsample_point_cloud( + display_pts_R, random_downsample_N + ) display_pts_overlap = DataLoadUtil.get_overlapping_points( display_pts_L, display_pts_R, voxel_size ) diff --git a/utils/pts.py b/utils/pts.py index b701af4..675f339 100644 --- a/utils/pts.py +++ b/utils/pts.py @@ -19,6 +19,8 @@ class PtsUtil: @staticmethod def random_downsample_point_cloud(point_cloud, num_points): + if point_cloud.shape[0] == 0: + return point_cloud idx = np.random.choice(len(point_cloud), num_points, replace=True) return point_cloud[idx] diff --git a/utils/reconstruction.py b/utils/reconstruction.py index 0b78e59..622e753 100644 --- a/utils/reconstruction.py +++ b/utils/reconstruction.py @@ -8,7 +8,7 @@ class ReconstructionUtil: def compute_coverage_rate(target_point_cloud, combined_point_cloud, threshold=0.01): kdtree = cKDTree(combined_point_cloud) distances, _ = kdtree.query(target_point_cloud) - covered_points = np.sum(distances < threshold) + covered_points = np.sum(distances < threshold*2) coverage_rate = covered_points / target_point_cloud.shape[0] return coverage_rate @@ -17,7 +17,10 @@ class ReconstructionUtil: kdtree = cKDTree(combined_point_cloud) distances, _ = kdtree.query(new_point_cloud) overlapping_points = np.sum(distances < threshold) - overlap_rate = overlapping_points / new_point_cloud.shape[0] + if new_point_cloud.shape[0] == 0: + overlap_rate = 0 + else: + overlap_rate = overlapping_points / new_point_cloud.shape[0] return overlap_rate @staticmethod @@ -43,12 +46,23 @@ class ReconstructionUtil: best_view = view_index return best_view, best_coverage_increase + @staticmethod + def get_new_added_points(old_combined_pts, new_pts, threshold=0.005): + if old_combined_pts.size == 0: + return new_pts + if new_pts.size == 0: + return np.array([]) + + tree = cKDTree(old_combined_pts) + distances, _ = tree.query(new_pts, k=1) + new_added_points = new_pts[distances > threshold] + return new_added_points @staticmethod - def compute_next_best_view_sequence_with_overlap(target_point_cloud, point_cloud_list, scan_points_indices_list, threshold=0.01, overlap_threshold=0.3, init_view = 0, status_info=None): + def compute_next_best_view_sequence_with_overlap(target_point_cloud, point_cloud_list, scan_points_indices_list, threshold=0.01, soft_overlap_threshold=0.5, hard_overlap_threshold=0.7, init_view = 0, scan_points_threshold=5, status_info=None): selected_views = [point_cloud_list[init_view]] combined_point_cloud = np.vstack(selected_views) - combined_scan_points_indices = scan_points_indices_list[init_view] + history_indices = [scan_points_indices_list[init_view]] down_sampled_combined_point_cloud = PtsUtil.voxel_downsample_point_cloud(combined_point_cloud,threshold) new_coverage = ReconstructionUtil.compute_coverage_rate(target_point_cloud, down_sampled_combined_point_cloud, threshold) current_coverage = new_coverage @@ -62,16 +76,23 @@ class ReconstructionUtil: best_coverage_increase = -1 for view_index in remaining_views: - + if point_cloud_list[view_index].shape[0] == 0: + continue + if selected_views: new_scan_points_indices = scan_points_indices_list[view_index] - if not ReconstructionUtil.check_scan_points_overlap(combined_scan_points_indices, new_scan_points_indices): - combined_old_point_cloud = np.vstack(selected_views) - down_sampled_old_point_cloud = PtsUtil.voxel_downsample_point_cloud(combined_old_point_cloud,threshold) - down_sampled_new_view_point_cloud = PtsUtil.voxel_downsample_point_cloud(point_cloud_list[view_index],threshold) - overlap_rate = ReconstructionUtil.compute_overlap_rate(down_sampled_new_view_point_cloud,down_sampled_old_point_cloud, threshold) - if overlap_rate < overlap_threshold: - continue + + if not ReconstructionUtil.check_scan_points_overlap(history_indices, new_scan_points_indices, scan_points_threshold): + overlap_threshold = hard_overlap_threshold + else: + overlap_threshold = soft_overlap_threshold + + combined_old_point_cloud = np.vstack(selected_views) + down_sampled_old_point_cloud = PtsUtil.voxel_downsample_point_cloud(combined_old_point_cloud,threshold) + down_sampled_new_view_point_cloud = PtsUtil.voxel_downsample_point_cloud(point_cloud_list[view_index],threshold) + overlap_rate = ReconstructionUtil.compute_overlap_rate(down_sampled_new_view_point_cloud,down_sampled_old_point_cloud, threshold) + if overlap_rate < overlap_threshold: + continue candidate_views = selected_views + [point_cloud_list[view_index]] combined_point_cloud = np.vstack(candidate_views) @@ -88,7 +109,7 @@ class ReconstructionUtil: break selected_views.append(point_cloud_list[best_view]) remaining_views.remove(best_view) - combined_scan_points_indices = ReconstructionUtil.combine_scan_points_indices(combined_scan_points_indices, scan_points_indices_list[best_view]) + history_indices.append(scan_points_indices_list[best_view]) current_coverage += best_coverage_increase cnt_processed_view += 1 if status_info is not None: @@ -110,7 +131,7 @@ class ReconstructionUtil: return view_sequence, remaining_views, down_sampled_combined_point_cloud @staticmethod - def filter_points(points, points_normals, cam_pose, voxel_size=0.005, theta=45): + def filter_points(points, points_normals, cam_pose, voxel_size=0.005, theta=75): sampled_points = PtsUtil.voxel_downsample_point_cloud(points, voxel_size) kdtree = cKDTree(points_normals[:,:3]) _, indices = kdtree.query(sampled_points) @@ -143,6 +164,7 @@ class ReconstructionUtil: @staticmethod def compute_covered_scan_points(scan_points, point_cloud, threshold=0.01): + tree = cKDTree(point_cloud) covered_points = [] indices = [] @@ -153,10 +175,10 @@ class ReconstructionUtil: return covered_points, indices @staticmethod - def check_scan_points_overlap(indices1, indices2, threshold=5): - return len(set(indices1).intersection(set(indices2))) > threshold - - @staticmethod - def combine_scan_points_indices(indices1, indices2): - combined_indices = set(indices1) | set(indices2) - return sorted(combined_indices) \ No newline at end of file + def check_scan_points_overlap(history_indices, indices2, threshold=5): + for indices1 in history_indices: + if len(set(indices1).intersection(set(indices2))) >= threshold: + return True + return False + + \ No newline at end of file