diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 78aa0ab0..2419973a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,16 +27,42 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install -e . - - name: Run pytest linux (linux-only) - if: matrix.os == 'ubuntu-22.04' + - name: Run pytest test_colors run: | cd ammico - python -m pytest -m "not gcv and not long" -svv --cov=. --cov-report=xml - - name: Run pytest windows(windows-only) - if: matrix.os == 'windows-latest' + python -m pytest test/test_colors.py -svv --cov=. --cov-report=xml + - name: Run pytest test_cropposts run: | cd ammico - python -m pytest -m "not gcv and not long and not win_skip" -svv --cov=. --cov-report=xml + python -m pytest test/test_cropposts.py -svv --cov=. --cov-report=xml + - name: Run pytest test_display + run: | + cd ammico + python -m pytest test/test_display.py -svv --cov=. --cov-report=xml + - name: Run pytest test_faces + run: | + cd ammico + python -m pytest test/test_faces.py -svv --cov=. --cov-report=xml + - name: Run pytest test_multimodal_search + run: | + cd ammico + python -m pytest test/test_multimodal_search.py -m "not long" -svv --cov=. --cov-report=xml + - name: Run pytest test_objects + run: | + cd ammico + python -m pytest test/test_objects.py -svv --cov=. --cov-report=xml + - name: Run pytest test_summary + run: | + cd ammico + python -m pytest test/test_summary.py -m "not long" -svv --cov=. --cov-report=xml + - name: Run pytest test_text + run: | + cd ammico + python -m pytest test/test_text.py -m "not gcv" -svv --cov=. --cov-report=xml + - name: Run pytest test_utils + run: | + cd ammico + python -m pytest test/test_utils.py -svv --cov=. --cov-report=xml - name: Upload coverage if: matrix.os == 'ubuntu-22.04' && matrix.python-version == '3.9' uses: codecov/codecov-action@v3 diff --git a/ammico/display.py b/ammico/display.py index 9ad7bf44..387e9269 100644 --- a/ammico/display.py +++ b/ammico/display.py @@ -24,6 +24,8 @@ "CAM16-UCS", "DIN99", ] +SUMMARY_ANALYSIS_TYPE = ["summary_and_questions", "summary", "questions"] +SUMMARY_MODEL = ["base", "large"] class AnalysisExplorer: @@ -111,6 +113,9 @@ def __init__(self, mydict: dict) -> None: State("setting_Emotion_emotion_threshold", "value"), State("setting_Emotion_race_threshold", "value"), State("setting_Color_delta_e_method", "value"), + State("setting_Summary_analysis_type", "value"), + State("setting_Summary_model", "value"), + State("setting_Summary_list_of_questions", "value"), prevent_initial_call=True, )(self._right_output_analysis) @@ -118,6 +123,7 @@ def __init__(self, mydict: dict) -> None: Output("settings_TextDetector", "style"), Output("settings_EmotionDetector", "style"), Output("settings_ColorDetector", "style"), + Output("settings_Summary_Detector", "style"), Input("Dropdown_select_Detector", "value"), )(self._update_detector_setting) @@ -240,6 +246,60 @@ def _create_setting_layout(self): ) ], ), + html.Div( + id="settings_Summary_Detector", + style={"display": "none"}, + children=[ + html.Div( + [ + dcc.Dropdown( + options=SUMMARY_ANALYSIS_TYPE, + value="summary_and_questions", + id="setting_Summary_analysis_type", + ) + ], + style={ + "width": "33%", + "display": "inline-block", + }, + ), + html.Div( + [ + dcc.Dropdown( + options=SUMMARY_MODEL, + value="base", + id="setting_Summary_model", + ) + ], + style={ + "width": "33%", + "display": "inline-block", + "margin-top": "10px", + }, + ), + html.Div( + [ + html.Div( + "Please enter a question", + style={ + "height": "50px", + "margin-top": "5px", + }, + ), + dcc.Input( + type="text", + id="setting_Summary_list_of_questions", + style={"height": "auto", "margin-bottom": "auto"}, + ), + ], + style={ + "width": "33%", + "display": "inline-block", + "margin-top": "10px", + }, + ), + ], + ), ], ) return settings_layout @@ -334,16 +394,19 @@ def _update_detector_setting(self, setting_input): } if setting_input == "TextDetector": - return display_flex, display_none, display_none + return display_flex, display_none, display_none, display_none if setting_input == "EmotionDetector": - return display_none, display_flex, display_none + return display_none, display_flex, display_none, display_none if setting_input == "ColorDetector": - return display_none, display_none, display_flex + return display_none, display_none, display_flex, display_none + + if setting_input == "SummaryDetector": + return display_none, display_none, display_none, display_flex else: - return display_none, display_none, display_none + return display_none, display_none, display_none, display_none def _right_output_analysis( self, @@ -355,6 +418,9 @@ def _right_output_analysis( setting_emotion_emotion_threshold: int, setting_emotion_race_threshold: int, setting_color_delta_e_method: str, + setting_summary_analysis_type: str, + setting_summary_model: str, + setting_summary_list_of_questions: str, ) -> dict: """Callback function to perform analysis on the selected image and return the output. @@ -396,6 +462,15 @@ def _right_output_analysis( image_copy, delta_e_method=setting_color_delta_e_method, ) + elif detector_value == "SummaryDetector": + detector_class = identify_function( + image_copy, + analysis_type=setting_summary_analysis_type, + summary_model_type=setting_summary_model, + list_of_questions=[setting_summary_list_of_questions] + if (setting_summary_list_of_questions is not None) + else None, + ) else: detector_class = identify_function(image_copy) return detector_class.analyse_image() diff --git a/ammico/summary.py b/ammico/summary.py index 50fc2ee5..21e5fe91 100644 --- a/ammico/summary.py +++ b/ammico/summary.py @@ -5,9 +5,91 @@ class SummaryDetector(AnalysisMethod): - def __init__(self, subdict: dict) -> None: + def __init__( + self, + subdict: dict = {}, + summary_model_type: str = "base", + analysis_type: str = "summary_and_questions", + list_of_questions: str = None, + summary_model=None, + summary_vis_processors=None, + summary_vqa_model=None, + summary_vqa_vis_processors=None, + summary_vqa_txt_processors=None, + ) -> None: + """ + SummaryDetector class for analysing images using the blip_caption model. + + Args: + subdict (dict, optional): Dictionary containing the image to be analysed. Defaults to {}. + summary_model_type (str, optional): Type of blip_caption model to use. Can be "base" or "large". Defaults to "base". + analysis_type (str, optional): Type of analysis to perform. Can be "summary", "questions" or "summary_and_questions". Defaults to "summary_and_questions". + list_of_questions (list, optional): List of questions to answer. Defaults to ["Are there people in the image?", "What is this picture about?"]. + summary_model ([type], optional): blip_caption model. Defaults to None. + summary_vis_processors ([type], optional): Preprocessors for visual inputs. Defaults to None. + summary_vqa_model ([type], optional): blip_vqa model. Defaults to None. + summary_vqa_vis_processors ([type], optional): Preprocessors for vqa visual inputs. Defaults to None. + summary_vqa_txt_processors ([type], optional): Preprocessors for vqa text inputs. Defaults to None. + + Raises: + ValueError: If analysis_type is not one of "summary", "questions" or "summary_and_questions". + + Returns: + None. + """ + super().__init__(subdict) + if analysis_type not in ["summary", "questions", "summary_and_questions"]: + raise ValueError( + "analysis_type must be one of 'summary', 'questions' or 'summary_and_questions'" + ) self.summary_device = "cuda" if cuda.is_available() else "cpu" + allowed_model_types = ["base", "large"] + if summary_model_type not in allowed_model_types: + raise ValueError( + "Model type is not allowed - please select one of {}".format( + allowed_model_types + ) + ) + self.summary_model_type = summary_model_type + self.analysis_type = analysis_type + if list_of_questions is None: + self.list_of_questions = [ + "Are there people in the image?", + "What is this picture about?", + ] + elif (not isinstance(list_of_questions, list)) or ( + not all(isinstance(i, str) for i in list_of_questions) + ): + raise ValueError("list_of_questions must be a list of string (questions)") + else: + self.list_of_questions = list_of_questions + if ( + (summary_model is None) + and (summary_vis_processors is None) + and (analysis_type != "questions") + ): + self.summary_model, self.summary_vis_processors = self.load_model( + model_type=summary_model_type + ) + else: + self.summary_model = summary_model + self.summary_vis_processors = summary_vis_processors + if ( + (summary_vqa_model is None) + and (summary_vqa_vis_processors is None) + and (summary_vqa_txt_processors is None) + and (analysis_type != "summary") + ): + ( + self.summary_vqa_model, + self.summary_vqa_vis_processors, + self.summary_vqa_txt_processors, + ) = self.load_vqa_model() + else: + self.summary_vqa_model = summary_vqa_model + self.summary_vqa_vis_processors = summary_vqa_vis_processors + self.summary_vqa_txt_processors = summary_vqa_txt_processors def load_model_base(self): """ @@ -63,32 +145,71 @@ def load_model(self, model_type: str): summary_model, summary_vis_processors = select_model[model_type](self) return summary_model, summary_vis_processors - def analyse_image(self, summary_model=None, summary_vis_processors=None): + def load_vqa_model(self): + """ + Load blip_vqa model and preprocessors for visual and text inputs from lavis.models. + + Args: + + Returns: + model (torch.nn.Module): model. + vis_processors (dict): preprocessors for visual inputs. + txt_processors (dict): preprocessors for text inputs. + + """ + ( + summary_vqa_model, + summary_vqa_vis_processors, + summary_vqa_txt_processors, + ) = load_model_and_preprocess( + name="blip_vqa", + model_type="vqav2", + is_eval=True, + device=self.summary_device, + ) + return summary_vqa_model, summary_vqa_vis_processors, summary_vqa_txt_processors + + def analyse_image(self): + """ + Analyse image with blip_caption model. + + Args: + + Returns: + self.subdict (dict): dictionary with analysis results. + """ + if self.analysis_type == "summary_and_questions": + self.analyse_summary() + self.analyse_questions(self.list_of_questions) + elif self.analysis_type == "summary": + self.analyse_summary() + elif self.analysis_type == "questions": + self.analyse_questions(self.list_of_questions) + + return self.subdict + + def analyse_summary(self): """ Create 1 constant and 3 non deterministic captions for image. Args: - summary_model (str): model. - summary_vis_processors (str): preprocessors for visual inputs. Returns: - self.subdict (dict): dictionary with constant image summary and 3 non deterministic summary. + self.subdict (dict): dictionary with analysis results. """ - if summary_model is None and summary_vis_processors is None: - summary_model, summary_vis_processors = self.load_model_base() path = self.subdict["filename"] raw_image = Image.open(path).convert("RGB") image = ( - summary_vis_processors["eval"](raw_image) + self.summary_vis_processors["eval"](raw_image) .unsqueeze(0) .to(self.summary_device) ) with no_grad(): - self.subdict["const_image_summary"] = summary_model.generate( + self.subdict["const_image_summary"] = self.summary_model.generate( {"image": image} )[0] - self.subdict["3_non-deterministic summary"] = summary_model.generate( + self.subdict["3_non-deterministic summary"] = self.summary_model.generate( {"image": image}, use_nucleus_sampling=True, num_captions=3 ) return self.subdict @@ -103,32 +224,37 @@ def analyse_questions(self, list_of_questions: list[str]) -> dict: Returns: self.subdict (dict): dictionary with answers to questions. """ - ( - summary_vqa_model, - summary_vqa_vis_processors, - summary_vqa_txt_processors, - ) = load_model_and_preprocess( - name="blip_vqa", - model_type="vqav2", - is_eval=True, - device=self.summary_device, - ) + if ( + (self.summary_vqa_model is None) + and (self.summary_vqa_vis_processors is None) + and (self.summary_vqa_txt_processors is None) + ): + ( + self.summary_vqa_model, + self.summary_vqa_vis_processors, + self.summary_vqa_txt_processors, + ) = load_model_and_preprocess( + name="blip_vqa", + model_type="vqav2", + is_eval=True, + device=self.summary_device, + ) if len(list_of_questions) > 0: path = self.subdict["filename"] raw_image = Image.open(path).convert("RGB") image = ( - summary_vqa_vis_processors["eval"](raw_image) + self.summary_vqa_vis_processors["eval"](raw_image) .unsqueeze(0) .to(self.summary_device) ) question_batch = [] for quest in list_of_questions: - question_batch.append(summary_vqa_txt_processors["eval"](quest)) + question_batch.append(self.summary_vqa_txt_processors["eval"](quest)) batch_size = len(list_of_questions) image_batch = image.repeat(batch_size, 1, 1, 1) with no_grad(): - answers_batch = summary_vqa_model.predict_answers( + answers_batch = self.summary_vqa_model.predict_answers( samples={"image": image_batch, "text_input": question_batch}, inference_method="generate", ) diff --git a/ammico/test/test_display.py b/ammico/test/test_display.py index 4f51b901..3a2253eb 100644 --- a/ammico/test/test_display.py +++ b/ammico/test/test_display.py @@ -1,6 +1,7 @@ import json import ammico.display as ammico_display import pytest +import sys def test_explore_analysis_faces(get_path): @@ -25,6 +26,7 @@ def test_explore_analysis_objects(get_path): assert sub_dict[key] == outs[key] +@pytest.mark.skipif(sys.platform == "darwin", reason="segmentation fault on mac") def test_AnalysisExplorer(get_path): path_img_1 = get_path + "IMG_2809.png" path_img_2 = get_path + "IMG_2746.png" @@ -46,18 +48,59 @@ def test_AnalysisExplorer(get_path): assert analysis_explorer.update_picture(None) is None analysis_explorer._right_output_analysis( - 2, all_options_dict, path_img_1, "ObjectDetector", True, 50, 50, "CIE 1976" + 2, + all_options_dict, + path_img_1, + "ObjectDetector", + True, + 50, + 50, + "CIE 1976", + "summary_and_questions", + "base", + "How many people are in the picture?", ) analysis_explorer._right_output_analysis( - 2, all_options_dict, path_img_1, "EmotionDetector", True, 50, 50, "CIE 1976" + 2, + all_options_dict, + path_img_1, + "EmotionDetector", + True, + 50, + 50, + "CIE 1976", + "summary_and_questions", + "base", + "How many people are in the picture?", ) + analysis_explorer._right_output_analysis( - 2, all_options_dict, path_img_1, "SummaryDetector", True, 50, 50, "CIE 1976" + 2, + all_options_dict, + path_img_1, + "SummaryDetector", + True, + 50, + 50, + "CIE 1976", + "summary_and_questions", + "base", + "How many people are in the picture?", ) analysis_explorer._right_output_analysis( - 2, all_options_dict, path_img_1, "ColorDetector", True, 50, 50, "CIE 1976" + 2, + all_options_dict, + path_img_1, + "ColorDetector", + True, + 50, + 50, + "CIE 1976", + "summary_and_questions", + "base", + "How many people are in the picture?", ) with pytest.raises(EnvironmentError): diff --git a/ammico/test/test_objects.py b/ammico/test/test_objects.py index 36b085ae..bcd536f5 100644 --- a/ammico/test/test_objects.py +++ b/ammico/test/test_objects.py @@ -2,6 +2,7 @@ import pytest import ammico.objects as ob import ammico.objects_cvlib as ob_cvlib +import sys OBJECT_1 = "cell phone" OBJECT_2 = "motorcycle" @@ -25,6 +26,7 @@ def test_objects_from_cvlib(default_objects): assert str(objects) == str(out_objects) +@pytest.mark.skipif(sys.platform == "darwin", reason="segmentation fault on mac") def test_analyse_image_cvlib(get_path): mydict = {"filename": get_path + TEST_IMAGE_1} ob_cvlib.ObjectCVLib().analyse_image(mydict) @@ -55,6 +57,7 @@ def test_init_default_objects(): assert init_objects[obj] == "no" +@pytest.mark.skipif(sys.platform == "darwin", reason="segmentation fault on mac") def test_analyse_image_from_file_cvlib(get_path): file_path = get_path + TEST_IMAGE_1 objs = ob_cvlib.ObjectCVLib().analyse_image_from_file(file_path) @@ -66,6 +69,7 @@ def test_analyse_image_from_file_cvlib(get_path): assert objs[key] == out_dict[key] +@pytest.mark.skipif(sys.platform == "darwin", reason="segmentation fault on mac") def test_detect_objects_cvlib(get_path): file_path = get_path + TEST_IMAGE_1 objs = ob_cvlib.ObjectCVLib().detect_objects_cvlib(file_path) @@ -82,6 +86,7 @@ def test_set_keys(default_objects, get_path): assert str(default_objects) == str(key_objs) +@pytest.mark.skipif(sys.platform == "darwin", reason="segmentation fault on mac") def test_analyse_image(get_path): mydict = {"filename": get_path + TEST_IMAGE_1} ob.ObjectDetector.set_client_to_cvlib() diff --git a/ammico/test/test_summary.py b/ammico/test/test_summary.py index 909c977f..b6388413 100644 --- a/ammico/test/test_summary.py +++ b/ammico/test/test_summary.py @@ -40,7 +40,7 @@ def get_dict(get_path): return mydict -@pytest.mark.long +# @pytest.mark.long def test_analyse_image(get_dict): reference_results = { "run1": { @@ -63,9 +63,12 @@ def test_analyse_image(get_dict): ) # run two different images for key in get_dict.keys(): - get_dict[key] = sm.SummaryDetector(get_dict[key]).analyse_image( - summary_model, summary_vis_processors - ) + get_dict[key] = sm.SummaryDetector( + get_dict[key], + analysis_type="summary", + summary_model=summary_model, + summary_vis_processors=summary_vis_processors, + ).analyse_image() assert len(get_dict) == 2 for key in get_dict.keys(): assert len(get_dict[key]["3_non-deterministic summary"]) == 3 @@ -84,9 +87,11 @@ def test_analyse_questions(get_dict): "What happends on the picture?", ] for key in get_dict: - get_dict[key] = sm.SummaryDetector(get_dict[key]).analyse_questions( - list_of_questions - ) + get_dict[key] = sm.SummaryDetector( + get_dict[key], + analysis_type="questions", + list_of_questions=list_of_questions, + ).analyse_image() assert len(get_dict) == 2 list_of_questions_ans = ["2", "100"] list_of_questions_ans2 = ["flood", "festival"] @@ -97,3 +102,50 @@ def test_analyse_questions(get_dict): test_answers2.append(get_dict[key][list_of_questions[1]]) assert sorted(test_answers) == sorted(list_of_questions_ans) assert sorted(test_answers2) == sorted(list_of_questions_ans2) + + +def test_init_summary(): + sd = sm.SummaryDetector({}, analysis_type="summary") + assert sd.analysis_type == "summary" + with pytest.raises(ValueError): + sm.SummaryDetector({}, analysis_type="something") + list_of_questions = ["Question 1", "Question 2"] + sd = sm.SummaryDetector({}, list_of_questions=list_of_questions) + assert sd.list_of_questions == list_of_questions + with pytest.raises(ValueError): + sm.SummaryDetector({}, list_of_questions={}) + with pytest.raises(ValueError): + sm.SummaryDetector({}, list_of_questions=[None]) + with pytest.raises(ValueError): + sm.SummaryDetector({}, list_of_questions=[0.1]) + + +@pytest.mark.long +def test_advanced_init_summary(): + sd = sm.SummaryDetector({}) + assert sd.summary_model + assert sd.summary_vis_processors + sd = sm.SummaryDetector({}, summary_model_type="large") + assert sd.summary_model + assert sd.summary_vis_processors + with pytest.raises(ValueError): + sm.SummaryDetector({}, summary_model_type="bla") + ( + summary_vqa_model, + summary_vqa_vis_processors, + summary_vqa_txt_processors, + ) = load_model_and_preprocess( + name="blip_vqa", + model_type="vqav2", + is_eval=True, + device="cpu", + ) + sd = sm.SummaryDetector( + {}, + summary_vqa_model=summary_vqa_model, + summary_vqa_vis_processors=summary_vqa_vis_processors, + summary_vqa_txt_processors=summary_vqa_txt_processors, + ) + assert sd.summary_vqa_model + assert sd.summary_vqa_vis_processors + assert sd.summary_vqa_txt_processors diff --git a/ammico/testing_all.py b/ammico/testing_all.py deleted file mode 100644 index 197fa6e3..00000000 --- a/ammico/testing_all.py +++ /dev/null @@ -1,13 +0,0 @@ -import ammico - -if __name__ == "__main__": - images = ammico.find_files(path=".") - mydict = ammico.initialize_dict(images) - for key in mydict: - mydict[key] = ammico.TextDetector( - mydict[key], analyse_text=True - ).analyse_image() - print(mydict) - outdict = ammico.append_data_to_dict(mydict) - df = ammico.dump_df(outdict) - df.to_csv("data_out.csv")