curious.bio/planktoscope
curious.bio/planktoscope
8
0
Fork 0
Code Issues 10 Pull requests Packages Projects 7 Releases Wiki Activity

segmenter: ecotaxa export, check that there are objects to export

Browse source 
(cherry picked from commit 31b8c83aa6f0db59c64a62f4caf73cf3974da4f6)
This commit is contained in:
Romain Bazile 2021-06-02 14:07:04 +02:00 committed by Romain Bazile
parent 86c322ccbe
commit 36eccd43f2
2 changed files with 35 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
scripts/planktoscope/segmenter/__init__.py
View file

@ -507,7 +507,7 @@ class SegmenterProcess(multiprocessing.Process):
) )
return (object_number, len(regionprops)) return (object_number, len(regionprops))
def _pipe(self): def _pipe(self, ecotaxa_export):
logger.info("Finding images") logger.info("Finding images")
images_list = self._find_files( images_list = self._find_files(
self.__working_path, ("JPG", "jpg", "JPEG", "jpeg") self.__working_path, ("JPG", "jpg", "JPEG", "jpeg")
@ -638,18 +638,27 @@ class SegmenterProcess(multiprocessing.Process):
f"We also found {total_objects} objects, or an average of {total_objects / (total_duration * 60)}objects per second" f"We also found {total_objects} objects, or an average of {total_objects / (total_duration * 60)}objects per second"
) )
planktoscope.segmenter.ecotaxa.ecotaxa_export( if ecotaxa_export:
self.__archive_fn, if "objects" in self.__global_metadata:
self.__global_metadata, if planktoscope.segmenter.ecotaxa.ecotaxa_export(
self.__working_obj_path, self.__archive_fn,
keep_files=True, self.__global_metadata,
) self.__working_obj_path,
keep_files=True,
):
logger.error("The ecotaxa export could not be completed")
else:
logger.succes("Ecotaxa archive export completed for this folder")
else:
logger.info("There are no objects to export")
else:
logger.info("We are not creating the ecotaxa output archive for this folder")
# cleanup # cleanup
# we're done free some mem # we're done free some mem
self.__flat = None self.__flat = None
def segment_all(self, paths: list): def segment_all(self, paths: list, force, ecotaxa_export):
"""Starts the segmentation in all the folders given recursively """Starts the segmentation in all the folders given recursively
Args: Args:
@ -660,9 +669,9 @@ class SegmenterProcess(multiprocessing.Process):
for x in os.walk(path): for x in os.walk(path):
if x[0] not in img_paths: if x[0] not in img_paths:
img_paths.append(x[0]) img_paths.append(x[0])
self.segment_list(img_paths) self.segment_list(img_paths, force, ecotaxa_export)
def segment_list(self, path_list: list, force=True): def segment_list(self, path_list: list, force=True, ecotaxa_export=True):
"""Starts the segmentation in the folders given """Starts the segmentation in the folders given
Args: Args:
@ -676,7 +685,7 @@ class SegmenterProcess(multiprocessing.Process):
# The file exists, let's check if we force or not # The file exists, let's check if we force or not
if force: if force:
# forcing, let's gooooo # forcing, let's gooooo
if not self.segment_path(path): if not self.segment_path(path, ecotaxa_export):
logger.error(f"There was en error while segmenting {path}") logger.error(f"There was en error while segmenting {path}")
else: else:
# we need to check for the presence of done.txt in each folder # we need to check for the presence of done.txt in each folder
@ -686,14 +695,14 @@ class SegmenterProcess(multiprocessing.Process):
f"Moving to the next folder, {path} has already been segmented" f"Moving to the next folder, {path} has already been segmented"
) )
else: else:
if not self.segment_path(path): if not self.segment_path(path, ecotaxa_export):
logger.error(f"There was en error while segmenting {path}") logger.error(f"There was en error while segmenting {path}")
else: else:
logger.debug(f"Moving to the next folder, {path} has no metadata.json") logger.debug(f"Moving to the next folder, {path} has no metadata.json")
# Publish the status "Done" to via MQTT to Node-RED # Publish the status "Done" to via MQTT to Node-RED
self.segmenter_client.client.publish("status/segmenter", '{"status":"Done"}') self.segmenter_client.client.publish("status/segmenter", '{"status":"Done"}')
def segment_path(self, path): def segment_path(self, path, ecotaxa_export):
"""Starts the segmentation in the given path """Starts the segmentation in the given path
Args: Args:
@ -755,7 +764,7 @@ class SegmenterProcess(multiprocessing.Process):
logger.info(f"Starting the pipeline in {path}") logger.info(f"Starting the pipeline in {path}")
try: try:
self._pipe() self._pipe(ecotaxa_export)
except Exception as e: except Exception as e:
logger.exception(f"There was an error in the pipeline {e}") logger.exception(f"There was an error in the pipeline {e}")
return False return False
@ -781,6 +790,7 @@ class SegmenterProcess(multiprocessing.Process):
path = None path = None
recursive = True recursive = True
force = False force = False
ecotaxa_export = True
# {"action":"segment"} # {"action":"segment"}
if "settings" in last_message: if "settings" in last_message:
if "force" in last_message["settings"]: if "force" in last_message["settings"]:
@ -789,6 +799,9 @@ class SegmenterProcess(multiprocessing.Process):
if "recursive" in last_message["settings"]: if "recursive" in last_message["settings"]:
# parse folders recursively starting from the given parameter # parse folders recursively starting from the given parameter
recursive = last_message["settings"]["recursive"] recursive = last_message["settings"]["recursive"]
if "ecotaxa" in last_message["settings"]:
# generate ecotaxa output archive
ecotaxa_export = last_message["settings"]["ecotaxa"]
# TODO eventually add customisation to segmenter parameters here # TODO eventually add customisation to segmenter parameters here
if "path" in last_message: if "path" in last_message:
@ -800,9 +813,9 @@ class SegmenterProcess(multiprocessing.Process):
) )
if path: if path:
if recursive: if recursive:
self.segment_all(path) self.segment_all(path, force, ecotaxa_export)
else: else:
self.segment_list(path) self.segment_list(path, force, ecotaxa_export)
else: else:
self.segment_all(self.__img_path) self.segment_all(self.__img_path)

7
scripts/planktoscope/segmenter/ecotaxa.py
View file

@ -199,7 +199,11 @@ def ecotaxa_export(archive_filepath, metadata, image_base_path, keep_files=False
# empty table, one line per object # empty table, one line per object
tsv_content = [] tsv_content = []
object_list = metadata.pop("objects") if "objects" in metadata:
object_list = metadata.pop("objects")
else:
logger.error("No objects metadata recorded, cannot continue the export")
return 0
# fix crappy old camera resolution that was not exported as string # fix crappy old camera resolution that was not exported as string
if type(metadata["acq_camera_resolution"]) != str: if type(metadata["acq_camera_resolution"]) != str:
@ -248,3 +252,4 @@ def ecotaxa_export(archive_filepath, metadata, image_base_path, keep_files=False
path_or_buf=tsv_file, sep="\t", encoding="utf-8", index=False path_or_buf=tsv_file, sep="\t", encoding="utf-8", index=False
) )
logger.success("Ecotaxa archive is ready!") logger.success("Ecotaxa archive is ready!")
return 1