Adding separate methods to extract the timestamp at which the keyframes occur.

Katna/frame_extractor.py

@@ -95,33 +95,72 @@ def __extract_all_frames_from_video__(self, videopath):
 
             frame_diffs = []
             frames = []
+
             for _ in range(0, self.max_frames_in_chunk):
                 if ret:
-                    # Calling process frame function to calculate the frame difference and adding the difference 
-                    # in **frame_diffs** list and frame to **frames** list
                     prev_frame, curr_frame = self.__process_frame(frame, prev_frame, frame_diffs, frames)
                     i = i + 1
                     ret, frame = cap.read()
-                    # print(frame_count)
                 else:
                     cap.release()
                     break
             chunk_no = chunk_no + 1
             yield frames, frame_diffs
         cap.release()
 
+    def __extract_all_frames_from_video_with_time__(self, videopath, chunk_info):
+        """Generator function for extracting frames from a input video which are sufficiently different from each other, 
+        and return result back as list of opencv images in memory along with their timestamps
+
+        :param videopath: inputvideo path
+        :type videopath: `str`
+        :param chunk_info: list of chunk timing info
+        :type chunk_info: list of tuples (filepath, start_time, end_time)
+        :return: Generator with extracted frames in max_process_frames chunks, difference between frames, and timestamps
+        :rtype: generator object with content of type [numpy.ndarray, numpy.ndarray, list] 
+        """
+        cap = cv2.VideoCapture(str(videopath))
+
+        ret, frame = cap.read()
+        i = 1
+        chunk_no = 0
+
+        while ret:
+            curr_frame = None
+            prev_frame = None
+
+            frame_diffs = []
+            frames = []
+            timestamps = []  # List to store timestamps
+
+            for _ in range(0, self.max_frames_in_chunk):
+                if ret:
+                    # Get timestamp in seconds
+                    timestamp = cap.get(cv2.CAP_PROP_POS_MSEC) / 1000.0
+                    timestamps.append((chunk_info[1] + timestamp, chunk_info[1], chunk_info[2]))
+
+                    # Process frame and calculate differences
+                    prev_frame, curr_frame = self.__process_frame(frame, prev_frame, frame_diffs, frames)
+                    i = i + 1
+                    ret, frame = cap.read()
+                else:
+                    cap.release()
+                    break
+            chunk_no = chunk_no + 1
+            yield frames, frame_diffs, timestamps
+        cap.release()
+
     def __get_frames_in_local_maxima__(self, frames, frame_diffs):
         """ Internal function for getting local maxima of key frames 
         This functions Returns one single image with strongest change from its vicinity of frames 
         ( vicinity defined using window length ) 
 
-        :param object: base class inheritance
-        :type object: class:`Object`
         :param frames: list of frames to do local maxima on
         :type frames: `list of images`
         :param frame_diffs: list of frame difference values 
         :type frame_diffs: `list of images`
-
+        :return: list of extracted key frames
+        :rtype: list
         """
         extracted_key_frames = []
         diff_array = np.array(frame_diffs)
@@ -139,6 +178,38 @@ def __get_frames_in_local_maxima__(self, frames, frame_diffs):
         del frame_diffs[:]
         return extracted_key_frames
 
+    def __get_frames_in_local_maxima_with_time__(self, frames, frame_diffs, timestamps):
+        """ Internal function for getting local maxima of key frames with timestamps
+        This functions Returns one single image with strongest change from its vicinity of frames 
+        ( vicinity defined using window length ) 
+
+        :param frames: list of frames to do local maxima on
+        :type frames: `list of images`
+        :param frame_diffs: list of frame difference values 
+        :type frame_diffs: `list of images`
+        :param timestamps: list of timestamps
+        :type timestamps: list of tuples (timestamp, chunk_start, chunk_end)
+        :return: tuple of (list of extracted key frames, list of timestamps)
+        :rtype: tuple(list, list)
+        """
+        extracted_key_frames = []
+        extracted_timestamps = []
+        diff_array = np.array(frame_diffs)
+        # Normalizing the frame differences based on windows parameters
+        sm_diff_array = self.__smooth__(diff_array, self.len_window)
+
+        # Get the indexes of those frames which have maximum differences
+        frame_indexes = np.asarray(argrelextrema(sm_diff_array, np.greater))[0]
+
+        for frame_index in frame_indexes:
+            extracted_key_frames.append(frames[frame_index - 1])
+            extracted_timestamps.append(timestamps[frame_index - 1])
+        del frames[:]
+        del sm_diff_array
+        del diff_array
+        del frame_diffs[:]
+        return extracted_key_frames, extracted_timestamps
+
     def __smooth__(self, x, window_len, window=config.FrameExtractor.window_type):
         """smooth the data using a window with requested size.
         This method is based on the convolution of a scaled window with the signal.
@@ -188,36 +259,59 @@ def __smooth__(self, x, window_len, window=config.FrameExtractor.window_type):
         return y[window_len - 1 : -window_len + 1]
 
     def extract_candidate_frames(self, videopath):
-        """ Pubic function for this module , Given and input video path
+        """ Public function for this module , Given and input video path
         This functions Returns one list of all candidate key-frames  
 
-        :param object: base class inheritance
-        :type object: class:`Object`
         :param videopath: inputvideo path
         :type videopath: `str`
-        :return: opencv.Image.Image objects
+        :return: list of opencv.Image.Image objects
         :rtype: list
         """
-
         extracted_candidate_key_frames = []
 
         # Get all frames from video in chunks using python Generators
-        frame_extractor_from_video_generator = self.__extract_all_frames_from_video__(
-            videopath
-        )
+        frame_extractor_from_video_generator = self.__extract_all_frames_from_video__(videopath)
 
         # Loop over every frame in the frame extractor generator object and calculate the
         # local maxima of frames 
         for frames, frame_diffs in frame_extractor_from_video_generator:
-            extracted_candidate_key_frames_chunk = []
             if self.USE_LOCAL_MAXIMA:
-
                 # Getting the frame with maximum frame difference
                 extracted_candidate_key_frames_chunk = self.__get_frames_in_local_maxima__(
                     frames, frame_diffs
                 )
-                extracted_candidate_key_frames.extend(
-                    extracted_candidate_key_frames_chunk
-                )
+                extracted_candidate_key_frames.extend(extracted_candidate_key_frames_chunk)
 
         return extracted_candidate_key_frames
+
+    def extract_candidate_frames_with_time(self, args):
+        """ Public function for this module , Given an input video path and chunk info
+        This functions Returns one list of all candidate key-frames and their timestamps
+
+        :param args: tuple of (videopath, chunk_info)
+        :type args: tuple(str, list)
+        :return: tuple of (list of opencv.Image.Image objects, list of timestamps in seconds)
+        :rtype: tuple(list, list)
+        """
+        videopath, chunk_info = args
+
+        extracted_candidate_key_frames = []
+        extracted_timestamps = []
+
+        # Get all frames from video in chunks using python Generators
+        frame_extractor_from_video_generator = self.__extract_all_frames_from_video_with_time__(
+            videopath, chunk_info
+        )
+
+        # Loop over every frame in the frame extractor generator object and calculate the
+        # local maxima of frames 
+        for frames, frame_diffs, timestamps in frame_extractor_from_video_generator:
+            if self.USE_LOCAL_MAXIMA:
+                # Getting the frame with maximum frame difference
+                extracted_candidate_key_frames_chunk, extracted_timestamps_chunk = self.__get_frames_in_local_maxima_with_time__(
+                    frames, frame_diffs, timestamps
+                )
+                extracted_candidate_key_frames.extend(extracted_candidate_key_frames_chunk)
+                extracted_timestamps.extend(extracted_timestamps_chunk)
+
+        return extracted_candidate_key_frames, extracted_timestamps

Katna/image_selector.py

@@ -54,6 +54,8 @@ def __get_brightness_score__(self, image):
         :return: result of Brightness measurment 
         :rtype: float value between 0.0 to 100.0    
         """
+        if len(image.shape) == 2:  # If image is grayscale
+            image = cv2.cvtColor(image, cv2.COLOR_GRAY2BGR)
         hsv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)
         _, _, v = cv2.split(hsv)
         sum = np.sum(v, dtype=np.float32)
@@ -229,9 +231,14 @@ def __get_best_images_index_from_each_cluster__(
             n_images = np.arange(len(curr_row))
             variance_laplacians = self.__get_laplacian_scores(files, n_images)
 
-            # Selecting image with low burr(high laplacian) score
-            selected_frame_of_current_cluster = curr_row[np.argmax(variance_laplacians)]
-            filtered_items.append(selected_frame_of_current_cluster)
+            if len(variance_laplacians) > 0:
+                # Selecting image with low burr(high laplacian) score
+                selected_frame_of_current_cluster = curr_row[np.argmax(variance_laplacians)]
+                filtered_items.append(selected_frame_of_current_cluster)
+
+            else:
+                # temporary catch for situation where there are no variance laplacians
+                print("NO VARIANCE LAPLACIANS, downstream issue, ignore for now")
 
         return filtered_items
 
@@ -247,6 +254,73 @@ def __get_best_images_index_from_each_cluster__(
     #     """
     #     self.__dict__.update(state)
 
+    def select_best_frames_with_time(self, input_key_frames, number_of_frames, input_timestamps):
+        """[summary] Public function for Image selector class: takes list of key-frames images and number of required
+        frames as input, returns list of filtered keyframes
+
+        :param object: base class inheritance
+        :type object: class:`Object`
+        :param input_key_frames: list of input keyframes in list of opencv image format 
+        :type input_key_frames: python list opencv images
+        :param number_of_frames: Required number of images 
+        :type: int   
+        :param input_timestamps: list of timestamps for each keyframe
+        :type: python list of timestamps
+        :return: Returns list of filtered image files 
+        :rtype: python list of images
+        """
+
+        self.nb_clusters = number_of_frames
+
+        filtered_key_frames = []
+        filtered_images_list = []   
+        filtered_timestamps = []
+        # Repeat until number of frames 
+        min_brightness_values = np.arange(config.ImageSelector.min_brightness_value, -0.01, -self.brightness_step)
+        max_brightness_values = np.arange(config.ImageSelector.max_brightness_value, 100.01, self.brightness_step)
+        min_entropy_values = np.arange(config.ImageSelector.min_entropy_value, -0.01, -self.entropy_step)
+        max_entropy_values = np.arange(config.ImageSelector.max_entropy_value, 10.01, self.entropy_step)
+
+        for (min_brightness_value, max_brightness_value, min_entropy_value, max_entropy_value) in itertools.zip_longest(min_brightness_values, max_brightness_values, min_entropy_values, max_entropy_values): 
+            if min_brightness_value is None:
+                min_brightness_value = 0.0
+            if max_brightness_value is None:
+                max_brightness_value = 100.0
+            if min_entropy_value is None:
+                min_entropy_value = 0.0
+            if max_entropy_value is None:
+                max_entropy_value = 10.0
+            self.min_brightness_value = min_brightness_value
+            self.max_brightness_value = max_brightness_value
+            self.min_entropy_value = min_entropy_value
+            self.max_entropy_value = max_entropy_value
+            filtered_key_frames = self.__filter_optimum_brightness_and_contrast_images__(
+                input_key_frames, 
+            )
+            if len(filtered_key_frames) >= number_of_frames:
+                break
+
+        # Selecting the best images from each cluster by first preparing the clusters on basis of histograms 
+        # and then selecting the best images from every cluster
+        if len(filtered_key_frames) >= self.nb_clusters:
+            files_clusters_index_array = self.__prepare_cluster_sets__(filtered_key_frames)
+            selected_images_index = self.__get_best_images_index_from_each_cluster__(
+                filtered_key_frames, files_clusters_index_array
+            )
+
+            for index in selected_images_index:
+                img = filtered_key_frames[index]
+                filtered_images_list.append(img)
+                filtered_timestamps.append(input_timestamps[index])
+        else:
+            # if number of required files are less than requested key-frames return all the files
+            for i, img in enumerate(filtered_key_frames):
+                filtered_images_list.append(img)
+                filtered_timestamps.append(input_timestamps[i])
+        return filtered_images_list, filtered_timestamps
+
+
+
     def select_best_frames(self, input_key_frames, number_of_frames):
         """[summary] Public function for Image selector class: takes list of key-frames images and number of required
         frames as input, returns list of filtered keyframes