Merge branch 'feature/2d-tileconfigs'

This feature implements a pre-processing step to apply a shading
correction model to the original data, plus creating projections of
the raw stacks. Additionally, it creates tile configuration files to
also stitch those 2D tilings (as opposed to stitching the stacks and
creating the projections afterwards).

src/micrometa/dataset.py

@@ -16,6 +16,7 @@ import io  # pylint: disable=unused-import
 import olefile
 
 from imcflibs.pathtools import parse_path, exists
+from imcflibs.strtools import strip_prefix
 
 from .log import LOG as log
 
@@ -630,6 +631,51 @@ class MosaicData(DataSet):
         log.debug('Dataset type: %s', type(img_ds))
         self.subvol.append(img_ds)
 
+    def files_and_coords(self, sort=False):
+        """Get a list of filenames and coordinates of the mosaic subvolumes.
+
+        Parameters
+        ----------
+        sort : bool, optional
+            If set to True the sequence of subvolumes will be re-ordered to be
+            line-wise from bottom-right to top-left. This is mostly intended for
+            being used with specific fusion methods of Fiji's Grid/Collection
+            stitcher, e.g. the "Random input tile" where the order of the tiles
+            affects the fusion result.
+
+        Returns
+        -------
+        list(list(str, list(float)))
+            A list of lists where the first element is the file name of the
+            respective sub-volume and the second element is a list of float
+            values (either two or three) with the corresponding coordinates.
+        """
+        tiles = list()
+        for vol in self.subvol:
+            # get storage path to subvolumes, make it relative to base path:
+            fname = strip_prefix(vol.storage['full'], self.storage['path'])
+            # always use forward slashes as path separator (works on all OS!)
+            fname = fname.replace('\\', '/')
+            pos = vol.position['relative']
+
+            try:
+                tiles.append([fname,
+                              [pos[0], pos[1], pos[2]]
+                             ])
+            except IndexError:
+                tiles.append([fname,
+                              [pos[0], pos[1]]
+                             ])
+
+        if sort:
+            # first sort by the 1st element of the coordinates, followed by
+            # sorting by the 2nd element:
+            tiles = sorted(sorted(tiles, key=lambda x: x[1][0], reverse=True),
+                           key=lambda x: x[1][1], reverse=True)
+
+        return tiles
+
+
 
 class MosaicDataCuboid(MosaicData):
 

src/micrometa/ijm_templates/stitching_body.ijm

@@ -33,10 +33,11 @@ if (ignore_z_stage) {
 tileconfigs = get_tileconfig_files(input_dir);
 for (i = 0; i < tileconfigs.length; i++) {
     layout_file = tileconfigs[i];
-    export_file  = input_dir + '/';
-    export_file += replace(layout_file, '.txt', export_format);
-    preview_file = input_dir + '/';
-    preview_file += replace(layout_file, '.txt', '_preview.jpg');
+    ds_name = replace(layout_file, '.txt', '');
+    ds_name = replace(ds_name, '.ics', '');
+    ds_name = replace(ds_name, '.ids', '');
+    export_file  = input_dir + '/' + ds_name + export_format;
+    preview_file = input_dir + '/' + ds_name + '_preview.jpg';
     param = tpl + "layout_file=[" + layout_file + "]";
     print(hr);
     print("*** [" + name + "]: processing " + layout_file);

src/micrometa/ijm_templates/stitching_head.ijm

@@ -31,7 +31,7 @@ function get_tileconfig_files(dir) {
      * Scan a directory for files matching a certain pattern and assemble a
      * new array with the filenames.
      */
-    pattern = 'mosaic_[0-9]+\.txt';
+    pattern = 'mosaic_[0-9]+.*\.txt';
     filelist = getFileList(dir);
     tileconfigs = newArray(filelist.length);
     ti = 0;  // the tileconfig index

src/micrometa/imagej.py

@@ -7,15 +7,15 @@
 from os import listdir
 from os.path import join, dirname, basename, splitext
 
-from imcflibs.iotools import readtxt
+import imcflibs
 from imcflibs.pathtools import exists
 
 from . import __version__
 from .log import LOG as log
 
 
-def gen_tile_config(mosaic_ds, sort=True, suffix=''):
-    """Generate a tile configuration for Fiji's stitcher.
+def gen_tile_config(mosaic, sort=True, suffix='', force_2d=False):
+    """Generate a tile configuration for Fiji's Grid/Collection stitcher.
 
     Generate a layout configuration file for a ceartain mosaic in the format
     readable by Fiji's "Grid/Collection stitching" plugin. The configuration is
@@ -24,9 +24,9 @@ def gen_tile_config(mosaic_ds, sort=True, suffix=''):
 
     Parameters
     ----------
-    mosaic_ds : micrometa.dataset.MosaicData
+    mosaic : dataset.MosaicData
         The mosaic dataset to generate the tile config for.
-    sort : bool
+    sort : bool, optional
         If set to True the sequence of tiles in the configuration will be
         re-ordered to be line-wise from bottom-right to top-left. This is mostly
         intended for being used with specific fusion methods of the
@@ -36,73 +36,68 @@ def gen_tile_config(mosaic_ds, sort=True, suffix=''):
         An optional suffix to use for the file names in the tile config instead
         of the original one. Can be used if the workflow requires a
         pre-processing step before the actual stitching where results will be
-        stored e.g. as ICS files or similar.
+        stored e.g. as ICS files or similar and / or when generating config
+        files for stitching projections of the original stacks (e.g. having a
+        suffix like "-max.ics").
+    force_2d : bool, optional (default=False)
+        By setting to True, the configuration will be generated for a 2D mosaic,
+        even if the mosaic is a 3D dataset. Can be used to generate a config for
+        stitching e.g. projections or single slices of the original stacks.
 
     Returns
     -------
     config : list(str)
         The tile configuration as a list of strings, one per line.
     """
-    conf = list()
-    conf.append('# Generated by %s (%s).\n#\n' % (__name__, __version__))
-    subvol_size_z = mosaic_ds.subvol[0].get_dimensions()['Z']
-    subvol_position_dim = len(mosaic_ds.subvol[0].position['relative'])
-    conf.append('# Define the number of dimensions we are working on\n')
-    if subvol_size_z > 1:
+    log.debug("Mosaic storage path: %s", mosaic.storage['path'])
+    tiles = mosaic.files_and_coords(sort)
+
+    conf = [
+        '# Generated by %s (%s).\n#\n' % (__name__, __version__),
+        '# Define the number of dimensions we are working on\n',
+    ]
+
+    # despite a subvolume being a stack (the 'Z' dimension is larger than 1),
+    # the coordinates might only be given as a 2D grid, without a z-component -
+    # in that case we have to fill in '0' as the z coordinate to make the
+    # Grid/Collection stitcher work in 3D:
+    coords_are_3d = len(tiles[0][1]) > 2
+    log.debug("Tile coordinates are given in 3D: %s", coords_are_3d)
+
+    is_stack = mosaic.subvol[0].get_dimensions()['Z'] > 1
+    log.debug("Original dataset is a stack: %s", is_stack)
+    is_stack = is_stack and not force_2d
+    if is_stack:
         conf.append('dim = 3\n')
-        if subvol_position_dim < 3:
-            coord_format = '(%f, %f, 0.0)\n'
-        else:
+        if coords_are_3d:
             coord_format = '(%f, %f, %f)\n'
+        else:
+            coord_format = '(%f, %f, 0.0)\n'
     else:
         conf.append('dim = 2\n')
         coord_format = '(%f, %f)\n'
+
     conf.append('# Define the image coordinates (in pixels)\n')
-    log.debug("Mosaic storage path: %s", mosaic_ds.storage['path'])
-    tiles = list()
-    for vol in mosaic_ds.subvol:
-        fname = vol.storage['full']
-        # convert path to subvolumes to be relative to mosaic file:
-        if fname.startswith(mosaic_ds.storage['path']):
-            fname = fname[len(mosaic_ds.storage['path']):]
-        # always use forward slashes as path separator (works on all OS!)
-        fname = fname.replace('\\', '/')
+    for tile_details in tiles:
+        fname = tile_details[0]
         if suffix:
             try:
-                # remove everything up to the last dot:
-                fname = fname[:fname.rindex('.') + 1]
-                # now append the new suffix:
-                fname += suffix
+                # remove all from the last dot on, then append the new suffix
+                fname = fname[:fname.rindex('.')] + suffix
             except ValueError:
                 msg = "File name doesn't contain a dot: %s" % fname
                 log.error(msg)
                 raise ValueError(msg)
 
-        pos = vol.position['relative']
+        line = '%s; ; ' % fname
+        line += coord_format % tuple(tile_details[1])
 
-        if subvol_position_dim > 2:
-            tiles.append([fname, pos[0], pos[1]], pos[2])
-        else:
-            tiles.append([fname, pos[0], pos[1]])
-
-    if sort:
-        tiles = sorted(sorted(tiles, key=lambda x: x[1], reverse=True),
-                       key=lambda x: x[2], reverse=True)
-
-    for tile_details in tiles:
-        line = '%s; ; ' % tile_details[0]
-        if subvol_position_dim > 2:
-            line += coord_format % (tile_details[1],
-                                    tile_details[2],
-                                    tile_details[3])
-        else:
-            line += coord_format % (tile_details[1], tile_details[2])
         conf.append(line)
 
     return conf
 
 
-def write_tile_config(mosaic_ds, outdir='', padlen=0, suffix=''):
+def write_tile_config(mosaic, outdir='', padlen=0, suffix='', force_2d=False):
     """Generate and write the tile configuration file.
 
     Call the function to generate the corresponding tile configuration and
@@ -111,7 +106,7 @@ def write_tile_config(mosaic_ds, outdir='', padlen=0, suffix=''):
 
     Parameters
     ----------
-    mosaic_ds : dataset.MosaicData
+    mosaic : dataset.MosaicData
         The mosaic dataset to write the tile config for.
     outdir : str
         The output directory, if empty the input directory is used.
@@ -120,22 +115,23 @@ def write_tile_config(mosaic_ds, outdir='', padlen=0, suffix=''):
         file name, e.g. '2' will result in names like 'mosaic_01.txt' and so on.
     suffix : str, optional
         An optional suffix to be passed on to the gen_tile_config() call.
+    force_2d : bool, optional (default=False)
+        See gen_tile_config() for details.
     """
-    log.info('write_tile_config(%i)', mosaic_ds.supplement['index'])
-    config = gen_tile_config(mosaic_ds, suffix=suffix)
-    fname = 'mosaic_%0' + str(padlen) + 'i.txt'
-    fname = fname % mosaic_ds.supplement['index']
+    log.info('write_tile_config(%i)', mosaic.supplement['index'])
+    config = gen_tile_config(mosaic, suffix=suffix, force_2d=force_2d)
+    fname = 'mosaic_%0' + str(padlen) + 'i%s.txt'
+    fname = fname % (mosaic.supplement['index'], suffix)
     if outdir == '':
-        fname = join(mosaic_ds.storage['path'], fname)
+        fname = join(mosaic.storage['path'], fname)
     else:
         fname = join(outdir, fname)
-    out = open(fname, 'w')
+    with open(fname, 'w') as out:
         out.writelines(config)
-    out.close()
         log.warn('Wrote tile config to %s', out.name)
 
 
-def write_all_tile_configs(experiment, outdir='', suffix=''):
+def write_all_tile_configs(experiment, outdir='', suffix='', force_2d=False):
     """Wrapper to generate all TileConfiguration.txt files.
 
     All arguments are directly passed on to write_tile_config().
@@ -143,11 +139,24 @@ def write_all_tile_configs(experiment, outdir='', suffix=''):
     padlen = len(str(len(experiment)))
     log.debug("Padding tile configuration file indexes to length %i", padlen)
     for mosaic_ds in experiment:
-        write_tile_config(mosaic_ds, outdir, padlen, suffix)
+        write_tile_config(mosaic_ds, outdir, padlen, suffix, force_2d)
 
 
 def locate_templates(tplpath=''):
-    """Locate path to templates, possibly in a .zip or .jar file."""
+    """Locate path to templates, possibly in a .zip or .jar file.
+
+    Parameters
+    ----------
+    tplpath : str, optional
+        The path to a directory or a .zip / .jar file containing the template
+        files (the default is '', which will result in the current directory
+        being searched for a subdirectory with the name 'ijm_templates').
+
+    Returns
+    -------
+    tplpath : str
+        The path to a directory or .zip / .jar file containing the templates.
+    """
     # by default templates are expected in a subdir of the current package:
     if tplpath == '':
         tplpath = join(dirname(__file__), 'ijm_templates')
@@ -155,9 +164,9 @@ def locate_templates(tplpath=''):
         if not exists(tplpath):
             tplpath += '.zip'
             log.debug('Looking for template directory: %s', tplpath)
-    # extended magic to look for templates in jar files having a version number
-    # (and possibly a 'SNAPSHOT' part in their filename) without having to
-    # hard-code those strings here:
+    # some logic to look for templates in jar files having a version number and
+    # possibly a 'SNAPSHOT' part in their filename without having to hard-code
+    # those strings here:
     if tplpath.lower().endswith('.jar'):
         candidates = list()
         jar_dir = dirname(tplpath)
@@ -176,7 +185,7 @@ def locate_templates(tplpath=''):
     return tplpath
 
 
-def gen_stitching_macro_code(experiment, pfx, path='', tplpath='', opts={}):
+def gen_stitching_macro(name, path, tplpfx, tplpath='', opts={}):
     """Generate code in ImageJ's macro language to stitch the mosaics.
 
     Take two template files ("head" and "body") and generate an ImageJ
@@ -187,13 +196,13 @@ def gen_stitching_macro_code(experiment, pfx, path='', tplpath='', opts={}):
 
     Parameters
     ----------
-    experiment : micrometa.experiment.MosaicExperiment
-        The object containing all information about the mosaic.
-    pfx : str
-        The prefix for the two template files, will be completed with the
-        corresponding suffixes "_head.ijm" and "_body.ijm".
+    name : str
+        The dataset name, to be used as a reference in macro log messages.
     path : str
         The path to use as input directory *INSIDE* the macro.
+    tplpfx : str
+        The prefix for the two template files, will be completed with the
+        corresponding suffixes "_head.ijm" and "_body.ijm".
     tplpath : str
         The path to a directory or zip file containing the templates.
     opts : dict (optional)
@@ -214,11 +223,11 @@ def gen_stitching_macro_code(experiment, pfx, path='', tplpath='', opts={}):
     ijm = []
     ijm.append('// Generated by %s (%s).\n\n' % (__name__, __version__))
     ijm.append("// =================== BEGIN macro HEAD ===================\n")
-    ijm += readtxt(pfx + '_head.ijm', templates)
+    ijm += imcflibs.iotools.readtxt(tplpfx + '_head.ijm', templates)
     ijm.append("// ==================== END macro HEAD ====================\n")
     ijm.append('\n')
 
-    ijm.append('name = "%s";\n' % experiment.infile['dname'])
+    ijm.append('name = "%s";\n' % name)
     # windows path separator (in)sanity:
     path = path.replace('\\', '\\\\')
     ijm.append('input_dir="%s";\n' % path)
@@ -226,14 +235,9 @@ def gen_stitching_macro_code(experiment, pfx, path='', tplpath='', opts={}):
     for option, value in opts.items():
         ijm.append('%s = %s;\n' % (option, value))
 
-    # If the overlap is below a certain level (5 percent), we disable
-    # computing the actual positions and subpixel accuracy:
-    if experiment[0].get_overlap('pct') < 5.0:
-        ijm.append('compute = false;\n')
-
     ijm.append('\n')
     ijm.append("// =================== BEGIN macro BODY ===================\n")
-    ijm += readtxt(pfx + '_body.ijm', templates)
+    ijm += imcflibs.iotools.readtxt(tplpfx + '_body.ijm', templates)
     ijm.append("// ==================== END macro BODY ====================\n")
     log.debug('--- ijm ---\n%s\n--- ijm ---', ijm)
     return ijm