1__all__ = ['version',
    2    'mm_read',
    3    'mm_read_to_3d',
    4    'image_write_with_thumbnail',
    5    'nrg_format_path',
    6    'highest_quality_repeat',
    7    'match_modalities',
    8    'mc_resample_image_to_target',
    9    'nrg_filelist_to_dataframe',
   10    'merge_timeseries_data',
   11    'timeseries_reg',
   12    'merge_dwi_data',
   13    'outlierness_by_modality',
   14    'bvec_reorientation',
   15    'get_dti',
   16    'dti_reg',
   17    'mc_reg',
   18    'get_data',
   19    'get_models',
   20    'get_valid_modalities',
   21    'dewarp_imageset',
   22    'super_res_mcimage',
   23    'segment_timeseries_by_meanvalue',
   24    'get_average_rsf',
   25    'get_average_dwi_b0',
   26    'dti_template',
   27    't1_based_dwi_brain_extraction',
   28    'mc_denoise',
   29    'tsnr',
   30    'dvars',
   31    'slice_snr',
   32    'impute_fa',
   33    'trim_dti_mask',
   34    'dipy_dti_recon',
   35    'concat_dewarp',
   36    'joint_dti_recon',
   37    'middle_slice_snr',
   38    'foreground_background_snr',
   39    'quantile_snr',
   40    'mask_snr',
   41    'dwi_deterministic_tracking',
   42    'dwi_closest_peak_tracking',
   43    'dwi_streamline_pairwise_connectivity',
   44    'dwi_streamline_connectivity',
   45    'hierarchical_modality_summary',
   46    'tra_initializer',
   47    'neuromelanin',
   48    'resting_state_fmri_networks',
   49    'write_bvals_bvecs',
   50    'crop_mcimage',
   51    'mm',
   52    'write_mm',
   53    'mm_nrg',
   54    'mm_csv',
   55    'collect_blind_qc_by_modality',
   56    'alffmap',
   57    'alff_image',
   58    'down2iso',
   59    'read_mm_csv',
   60    'assemble_modality_specific_dataframes',
   61    'bind_wide_mm_csvs',
   62    'merge_mm_dataframe',
   63    'augment_image',
   64    'boot_wmh',
   65    'threaded_bind_wide_mm_csvs',
   66    'get_names_from_data_frame',
   67    'average_mm_df',
   68    'quick_viz_mm_nrg',
   69    'blind_image_assessment',
   70    'average_blind_qc_by_modality',
   71    'best_mmm',
   72    'nrg_2_bids',
   73    'bids_2_nrg',
   74    'parse_nrg_filename',
   75    'novelty_detection_svm',
   76    'novelty_detection_ee',
   77    'novelty_detection_lof',
   78    'novelty_detection_loop',
   79    'novelty_detection_quantile',
   80    'generate_mm_dataframe',
   81    'aggregate_antspymm_results',
   82    'aggregate_antspymm_results_sdf',
   83    'study_dataframe_from_matched_dataframe',
   84    'merge_wides_to_study_dataframe',
   85    'filter_image_files',
   86    'docsamson',
   87    'enantiomorphic_filling_without_mask',
   88    'wmh',
   89    'remove_elements_from_numpy_array',
   90    'score_fmri_censoring',
   91    'remove_volumes_from_timeseries',
   92    'loop_timeseries_censoring',
   93    'clean_tmp_directory',
   94    'validate_nrg_file_format',
   95    'ants_to_nibabel_affine',
   96    'dict_to_dataframe']
   97
   98from pathlib import Path
   99from pathlib import PurePath
  100import os
  101import pandas as pd
  102import math
  103import os.path
  104from os import path
  105import pickle
  106import sys
  107import numpy as np
  108import random
  109import functools
  110from operator import mul
  111from scipy.sparse.linalg import svds
  112from scipy.stats import pearsonr
  113import re
  114import datetime as dt
  115from collections import Counter
  116import tempfile
  117import uuid
  118import warnings
  119
  120from dipy.core.histeq import histeq
  121import dipy.reconst.dti as dti
  122from dipy.core.gradients import (gradient_table, gradient_table_from_gradient_strength_bvecs)
  123from dipy.io.gradients import read_bvals_bvecs
  124from dipy.segment.mask import median_otsu
  125from dipy.reconst.dti import fractional_anisotropy, color_fa
  126import nibabel as nib
  127
  128import ants
  129import antspynet
  130import antspyt1w
  131import siq
  132import tensorflow as tf
  133
  134from multiprocessing import Pool
  135import glob as glob
  136
  137antspyt1w.set_global_scientific_computing_random_seed(
  138    antspyt1w.get_global_scientific_computing_random_seed( )
  139)
  140
  141DATA_PATH = os.path.expanduser('~/.antspymm/')
  142
  143def version( ):
  144    """
  145    report versions of this package and primary dependencies
  146
  147    Arguments
  148    ---------
  149    None
  150
  151    Returns
  152    -------
  153    a dictionary with package name and versions
  154
  155    Example
  156    -------
  157    >>> import antspymm
  158    >>> antspymm.version()
  159    """
  160    import pkg_resources
  161    return {
  162              'tensorflow': pkg_resources.get_distribution("tensorflow").version,
  163              'antspyx': pkg_resources.get_distribution("antspyx").version,
  164              'antspynet': pkg_resources.get_distribution("antspynet").version,
  165              'antspyt1w': pkg_resources.get_distribution("antspyt1w").version,
  166              'antspymm': pkg_resources.get_distribution("antspymm").version
  167              }
  168
  169def nrg_filename_to_subjectvisit(s, separator='-'):
  170    """
  171    Extracts a pattern from the input string.
  172    
  173    Parameters:
  174    - s: The input string from which to extract the pattern.
  175    - separator: The separator used in the string (default is '-').
  176    
  177    Returns:
  178    - A string in the format of 'PREFIX-Number-Date'
  179    """
  180    parts = os.path.basename(s).split(separator)
  181    # Assuming the pattern is always in the form of PREFIX-Number-Date-...
  182    # and PREFIX is always "PPMI", extract the first three parts
  183    extracted = separator.join(parts[:3])
  184    return extracted
  185
  186
  187def validate_nrg_file_format(path, separator):
  188    """
  189    is your path nrg-etic?
  190    Validates if a given path conforms to the NRG file format, taking into account known extensions
  191    and the expected directory structure.
  192
  193    :param path: The file path to validate.
  194    :param separator: The separator used in the filename and directory structure.
  195    :return: A tuple (bool, str) indicating whether the path is valid and a message explaining the validation result.
  196
  197    : example
  198
  199    ntfn='/Users/ntustison/Data/Stone/LIMBIC/NRG/ANTsLIMBIC/sub08C105120Yr/ses-1/rsfMRI_RL/000/ANTsLIMBIC_sub08C105120Yr_ses-1_rsfMRI_RL_000.nii.gz'
  200    ntfngood='/Users/ntustison/Data/Stone/LIMBIC/NRG/ANTsLIMBIC/sub08C105120Yr/ses_1/rsfMRI_RL/000/ANTsLIMBIC-sub08C105120Yr-ses_1-rsfMRI_RL-000.nii.gz'
  201
  202    validate_nrg_detailed(ntfngood, '-')
  203    print( validate_nrg_detailed(ntfn, '-') )
  204    print( validate_nrg_detailed(ntfn, '_') )
  205
  206    """
  207    import re    
  208
  209    def normalize_path(path):
  210        """
  211        Replace multiple repeated '/' with just a single '/'
  212        
  213        :param path: The file path to normalize.
  214        :return: The normalized file path with single '/'.
  215        """
  216        normalized_path = re.sub(r'/+', '/', path)
  217        return normalized_path
  218
  219    def strip_known_extension(filename, known_extensions):
  220        """
  221        Strips a known extension from the filename.
  222
  223        :param filename: The filename from which to strip the extension.
  224        :param known_extensions: A list of known extensions to strip from the filename.
  225        :return: The filename with the known extension stripped off, if found.
  226        """
  227        for ext in known_extensions:
  228            if filename.endswith(ext):
  229                # Strip the extension and return the modified filename
  230                return filename[:-len(ext)]
  231        # If no known extension is found, return the original filename
  232        return filename
  233
  234    import warnings
  235    if normalize_path( path ) != path:
  236        path = normalize_path( path )
  237        warnings.warn("Probably had multiple repeated slashes eg /// in the file path.  this might cause issues. clean up with re.sub(r'/+', '/', path)")
  238
  239    known_extensions = [".nii.gz", ".nii", ".mhd", ".nrrd", ".mha", ".json", ".bval", ".bvec"]
  240    known_extensions2 = [ext.lstrip('.') for ext in known_extensions]
  241    def get_extension(filename, known_extensions ):
  242        # List of known extensions in priority order
  243        for ext in known_extensions:
  244            if filename.endswith(ext):
  245                return ext.strip('.')
  246        return "Invalid extension"
  247    
  248    parts = path.split('/')
  249    if len(parts) < 7:  # Checking for minimum path structure
  250        return False, "Path structure is incomplete. Expected at least 7 components, found {}.".format(len(parts))
  251    
  252    # Extract directory components and filename
  253    directory_components = parts[1:-1]  # Exclude the root '/' and filename
  254    filename = parts[-1]
  255    filename_without_extension = strip_known_extension( filename, known_extensions )
  256    file_extension = get_extension( filename, known_extensions )
  257    
  258    # Validating file extension
  259    if file_extension not in known_extensions2:
  260        print( file_extension )
  261        return False, "Invalid file extension: {}. Expected 'nii.gz' or 'json'.".format(file_extension)
  262    
  263    # Splitting the filename to validate individual parts
  264    filename_parts = filename_without_extension.split(separator)
  265    if len(filename_parts) != 5:  # Expecting 5 parts based on the NRG format
  266        print( filename_parts )
  267        return False, "Filename does not have exactly 5 parts separated by '{}'. Found {} parts.".format(separator, len(filename_parts))
  268    
  269    # Reconstruct expected filename from directory components
  270    expected_filename_parts = directory_components[-5:]
  271    expected_filename = separator.join(expected_filename_parts)
  272    if filename_without_extension != expected_filename:
  273        print( filename_without_extension )
  274        print("--- vs expected ---")
  275        print( expected_filename )
  276        return False, "Filename structure does not match directory structure. Expected filename: {}.".format(expected_filename)
  277    
  278    # Validate directory structure against NRG format
  279    study_name, subject_id, session, modality = directory_components[-4:-1] + [directory_components[-1].split('/')[0]]
  280    if not all([study_name, subject_id, session, modality]):
  281        return False, "Directory structure does not follow NRG format. Ensure StudyName, SubjectID, Session (ses_x), and Modality are correctly specified."
  282    
  283    # If all checks pass
  284    return True, "The path conforms to the NRG format."
  285
  286
  287
  288def apply_transforms_mixed_interpolation(
  289    fixed,
  290    moving,
  291    transformlist,
  292    interpolator="linear",
  293    imagetype=0,
  294    whichtoinvert=None,
  295    mask=None,
  296    **kwargs
  297):
  298    """
  299    Apply ANTs transforms with mixed interpolation:
  300    - Linear interpolation inside `mask`
  301    - Nearest neighbor outside `mask`
  302
  303    Parameters
  304    ----------
  305    fixed : ANTsImage
  306        Fixed/reference image to define spatial domain.
  307
  308    moving : ANTsImage
  309        Moving image to be transformed.
  310
  311    transformlist : list of str
  312        List of filenames for transforms.
  313
  314    interpolator : str, optional
  315        Interpolator used inside the mask. Default is "linear".
  316
  317    imagetype : int
  318        Image type used by ANTs (0 = scalar, 1 = vector, etc.)
  319
  320    whichtoinvert : list of bool, optional
  321        List of booleans indicating which transforms to invert.
  322
  323    mask : ANTsImage
  324        Binary mask image indicating where to apply `interpolator` (e.g., "linear").
  325        Outside the mask, nearest neighbor is used.
  326
  327    kwargs : dict
  328        Additional arguments passed to `ants.apply_transforms`.
  329
  330    Returns
  331    -------
  332    ANTsImage
  333        Interpolated image using mixed interpolation, added across masked regions.
  334    """
  335    if mask is None:
  336        raise ValueError("A binary `mask` image must be provided.")
  337
  338    # Apply linear interpolation inside the mask
  339    interp_linear = ants.apply_transforms(
  340        fixed=fixed,
  341        moving=moving,
  342        transformlist=transformlist,
  343        interpolator=interpolator,
  344        imagetype=imagetype,
  345        whichtoinvert=whichtoinvert,
  346        **kwargs
  347    )
  348
  349    # Apply nearest-neighbor interpolation everywhere
  350    interp_nn = ants.apply_transforms(
  351        fixed=fixed,
  352        moving=moving,
  353        transformlist=transformlist,
  354        interpolator="nearestNeighbor",
  355        imagetype=imagetype,
  356        whichtoinvert=whichtoinvert,
  357        **kwargs
  358    )
  359
  360    # Combine: linear * mask + nn * (1 - mask)
  361    mixed_result = (interp_linear * mask) + (interp_nn * (1 - mask))
  362
  363    return mixed_result
  364
  365def get_antsimage_keys(dictionary):
  366    """
  367    Return the keys of the dictionary where the values are ANTsImages.
  368
  369    :param dictionary: A dictionary to inspect
  370    :return: A list of keys for which the values are ANTsImages
  371    """
  372    return [key for key, value in dictionary.items() if isinstance(value, ants.core.ants_image.ANTsImage)]
  373
  374def to_nibabel(img: "ants.core.ants_image.ANTsImage") -> nib.Nifti1Image:
  375    """
  376    Convert an ANTsPy image to a Nibabel Nifti1Image in-memory, using correct spatial affine.
  377
  378    Parameters:
  379        img (ants.ANTsImage): An image from ANTsPy.
  380
  381    Returns:
  382        nib.Nifti1Image: The corresponding Nibabel image with spatial orientation in RAS.
  383    """
  384    array_data = img.numpy()  # get voxel data as NumPy array
  385    affine = ants_to_nibabel_affine(img)
  386    return nib.Nifti1Image(array_data, affine)
  387
  388def ants_to_nibabel_affine(ants_img):
  389    """
  390    Convert an ANTsPy image (in LPS space) to a Nibabel-compatible affine (in RAS space).
  391    Handles 2D, 3D, 4D input (only spatial dimensions are encoded in the affine).
  392    
  393    Returns:
  394        4x4 np.ndarray affine matrix in RAS space.
  395    """
  396    spatial_dim = ants_img.dimension
  397    spacing = np.array(ants_img.spacing)
  398    origin = np.array(ants_img.origin)
  399    direction = np.array(ants_img.direction).reshape((spatial_dim, spatial_dim))
  400    # Compute rotation-scale matrix
  401    affine_linear = direction @ np.diag(spacing)
  402    # Build full 4x4 affine with identity in homogeneous bottom row
  403    affine = np.eye(4)
  404    affine[:spatial_dim, :spatial_dim] = affine_linear
  405    affine[:spatial_dim, 3] = origin
  406    affine[3, 3]=1
  407    # Convert LPS -> RAS by flipping x and y
  408    lps_to_ras = np.diag([-1, -1, 1, 1])
  409    affine = lps_to_ras @ affine
  410    return affine
  411
  412
  413def dict_to_dataframe(data_dict, convert_lists=True, convert_arrays=True, convert_images=True, verbose=False):
  414    """
  415    Convert a dictionary to a pandas DataFrame, excluding items that cannot be processed by pandas.
  416
  417    :param data_dict: Dictionary to be converted.
  418    :param convert_lists: boolean
  419    :param convert_arrays: boolean
  420    :param convert_images: boolean
  421    :param verbose: boolean
  422    :return: DataFrame representation of the dictionary.
  423    """
  424    processed_data = {}
  425    list_length = None
  426    def mean_of_list(lst):
  427        if not lst:  # Check if the list is not empty
  428            return 0  # Return 0 or appropriate value for an empty list
  429        all_numeric = all(isinstance(item, (int, float)) for item in lst)
  430        if all_numeric:
  431            return sum(lst) / len(lst)
  432        return None
  433    
  434    for key, value in data_dict.items():
  435        # Check if value is a scalar
  436        if isinstance(value, (int, float, str, bool)):
  437            processed_data[key] = [value]
  438        # Check if value is a list of scalars
  439        elif isinstance(value, list) and all(isinstance(item, (int, float, str, bool)) for item in value) and convert_lists:
  440            meanvalue = mean_of_list( value )
  441            newkey = key+"_mean"
  442            if verbose:
  443                print( " Key " + key + " is list with mean " + str(meanvalue) + " to " + newkey )
  444            if newkey not in data_dict.keys() and convert_lists:
  445                processed_data[newkey] = meanvalue
  446        elif isinstance(value, np.ndarray) and all(isinstance(item, (int, float, str, bool)) for item in value) and convert_arrays:
  447            meanvalue = value.mean()
  448            newkey = key+"_mean"
  449            if verbose:
  450                print( " Key " + key + " is nparray with mean " + str(meanvalue) + " to " + newkey )
  451            if newkey not in data_dict.keys():
  452                processed_data[newkey] = meanvalue
  453        elif isinstance(value, ants.core.ants_image.ANTsImage ) and convert_images:
  454            meanvalue = value.mean()
  455            newkey = key+"_mean"
  456            if newkey not in data_dict.keys():
  457                if verbose:
  458                    print( " Key " + key + " is antsimage with mean " + str(meanvalue) + " to " + newkey )
  459                processed_data[newkey] = meanvalue
  460            else:
  461                if verbose:
  462                    print( " Key " + key + " is antsimage with mean " + str(meanvalue) + " but " + newkey + " already exists" )
  463
  464    return pd.DataFrame.from_dict(processed_data)
  465
  466
  467def clean_tmp_directory(age_hours=1., use_sudo=False, extensions=[ '.nii', '.nii.gz' ], log_file_path=None):
  468    """
  469    Clean the /tmp directory by removing files and directories older than a certain number of hours.
  470    Optionally uses sudo and can filter files by extensions.
  471
  472    :param age_hours: Age in hours to consider files and directories for deletion.
  473    :param use_sudo: Whether to use sudo for removal commands.
  474    :param extensions: List of file extensions to delete. If None, all files are considered.
  475    :param log_file_path: Path to the log file. If None, a default path will be used based on the OS.
  476
  477    # Usage
  478    # Example: clean_tmp_directory(age_hours=1, use_sudo=True, extensions=['.log', '.tmp'])
  479    """
  480    import os
  481    import platform
  482    import subprocess
  483    from datetime import datetime, timedelta
  484
  485    if not isinstance(age_hours, float):
  486        return
  487
  488    # Determine the tmp directory based on the operating system
  489    tmp_dir = '/tmp'
  490
  491    # Set the log file path
  492    if log_file_path is not None:
  493        log_file = log_file_path
  494
  495    current_time = datetime.now()
  496    for item in os.listdir(tmp_dir):
  497        try:
  498            item_path = os.path.join(tmp_dir, item)
  499            item_stat = os.stat(item_path)
  500
  501            # Calculate the age of the file/directory
  502            item_age = current_time - datetime.fromtimestamp(item_stat.st_mtime)
  503            if item_age > timedelta(hours=age_hours):
  504                # Check for file extensions if provided
  505                if extensions is None or any(item.endswith(ext) for ext in extensions):
  506                    # Construct the removal command
  507                    rm_command = ['sudo', 'rm', '-rf', item_path] if use_sudo else ['rm', '-rf', item_path]
  508                    subprocess.run(rm_command)
  509
  510                if log_file_path is not None:
  511                    with open(log_file, 'a') as log:
  512                        log.write(f"{datetime.now()}: Deleted {item_path}\n")
  513        except Exception as e:
  514            if log_file_path is not None:
  515                with open(log_file, 'a') as log:
  516                    log.write(f"{datetime.now()}: Error deleting {item_path}: {e}\n")
  517
  518
  519
  520def docsamson(locmod, studycsv, outputdir, projid, sid, dtid, mysep, t1iid=None, verbose=True):
  521    """
  522    Processes image file names based on the specified imaging modality and other parameters.
  523
  524    The function selects file names from the provided dictionary `studycsv` based on the imaging modality.
  525    It supports various modalities like T1w, T2Flair, perf, NM2DMT, rsfMRI, DTI, and configures the filenames accordingly.
  526    The function can optionally print verbose output during processing.
  527
  528    Parameters:
  529    locmod (str): The imaging modality. Options include 'T1w', 'T2Flair', 'perf', 'NM2DMT', 'rsfMRI', 'DTI'.
  530    studycsv (dict): A dictionary with keys corresponding to imaging modalities and values as file names.
  531    outputdir (str): Base directory for output files.
  532    projid (str): Project identifier.
  533    sid (str): Subject identifier.
  534    dtid (str): Data acquisition time identifier.
  535    mysep (str): Separator used in file naming.
  536    t1iid (str, optional): Identifier related to T1-weighted images, used in naming output files when locmod is not 'T1w'.
  537    verbose (bool, optional): If True, prints detailed information during execution.
  538
  539    Returns:
  540    dict: A dictionary with keys 'modality', 'outprefix', and 'images'.
  541        - 'modality' (str): The imaging modality used.
  542        - 'outprefix' (str): The prefix for output file paths.
  543        - 'images' (list): A list of processed image file names.
  544
  545    Notes:
  546    - The function is designed to work within a specific workflow and might require adaptation for general use.
  547
  548    Examples:
  549    >>> result = docsamson('T1w', studycsv, outputdir, projid, sid, dtid, mysep)
  550    >>> print(result['modality'])
  551    'T1w'
  552    >>> print(result['outprefix'])
  553    '/path/to/output/directory/T1w/some_identifier'
  554    >>> print(result['images'])
  555    ['image1.nii', 'image2.nii']
  556    """
  557
  558    import os
  559    import re
  560
  561    myimgsInput = []
  562    myoutputPrefix = None
  563    imfns = ['filename', 'rsfid1', 'rsfid2', 'dtid1', 'dtid2', 'flairid']
  564    
  565    # Define image file names based on the modality
  566    if locmod == 'T1w':
  567        imfns=['filename']
  568    elif locmod == 'T2Flair':
  569        imfns=['flairid']
  570    elif locmod == 'perf':
  571        imfns=['perfid']
  572    elif locmod == 'pet3d':
  573        imfns=['pet3did']
  574    elif locmod == 'NM2DMT':
  575        imfns=[]
  576        for i in range(11):
  577            imfns.append('nmid' + str(i))
  578    elif locmod == 'rsfMRI':
  579        imfns=[]
  580        for i in range(4):
  581            imfns.append('rsfid' + str(i))
  582    elif locmod == 'DTI':
  583        imfns=[]
  584        for i in range(4):
  585            imfns.append('dtid' + str(i))
  586    else:
  587        raise ValueError("docsamson: no match of modality to filename id " + locmod )
  588
  589    # Process each file name
  590    for i in imfns:
  591        if verbose:
  592            print(i + " " + locmod)
  593        if i in studycsv.keys():
  594            fni = str(studycsv[i].iloc[0])
  595            if verbose:
  596                print(i + " " + fni + ' exists ' + str(os.path.exists(fni)))
  597            if os.path.exists(fni):
  598                myimgsInput.append(fni)
  599                temp = os.path.basename(fni)
  600                mysplit = temp.split(mysep)
  601                iid = re.sub(".nii.gz", "", mysplit[-1])
  602                iid = re.sub(".mha", "", iid)
  603                iid = re.sub(".nii", "", iid)
  604                iid2 = iid
  605                if locmod != 'T1w' and t1iid is not None:
  606                    iid2 = iid + "_" + t1iid
  607                else:
  608                    iid2 = t1iid
  609                myoutputPrefix = os.path.join(outputdir, projid, sid, dtid, locmod, iid, projid + mysep + sid + mysep + dtid + mysep + locmod + mysep + iid2)
  610    
  611    if verbose:
  612        print(locmod)
  613        print(myimgsInput)
  614        print(myoutputPrefix)
  615    
  616    return {
  617        'modality': locmod,
  618        'outprefix': myoutputPrefix,
  619        'images': myimgsInput
  620    }
  621
  622
  623def get_valid_modalities( long=False, asString=False, qc=False ):
  624    """
  625    return a list of valid modality identifiers used in NRG modality designation
  626    and that can be processed by this package.
  627
  628    long - return the long version
  629
  630    asString - concat list to string
  631    """
  632    if long:
  633        mymod = ["T1w", "NM2DMT", "rsfMRI", "rsfMRI_LR", "rsfMRI_RL", "rsfMRILR", "rsfMRIRL", "DTI", "DTI_LR","DTI_RL",  "DTILR","DTIRL","T2Flair", "dwi", "dwi_ap", "dwi_pa", "func", "func_ap", "func_pa", "perf", 'pet3d']
  634    elif qc:
  635        mymod = [ 'T1w', 'T2Flair', 'NM2DMT', 'DTI', 'DTIdwi','DTIb0', 'rsfMRI', "perf", 'pet3d' ]
  636    else:
  637        mymod = ["T1w", "NM2DMT", "DTI","T2Flair", "rsfMRI", "perf", 'pet3d' ]
  638    if not asString:
  639        return mymod
  640    else:
  641        mymodchar=""
  642        for x in mymod:
  643            mymodchar = mymodchar + " " + str(x)
  644        return mymodchar
  645
  646
  647def generate_mm_dataframe(
  648        projectID,
  649        subjectID,
  650        date,
  651        imageUniqueID,
  652        modality,
  653        source_image_directory,
  654        output_image_directory,
  655        t1_filename,
  656        flair_filename=[],
  657        rsf_filenames=[],
  658        dti_filenames=[],
  659        nm_filenames=[],
  660        perf_filename=[],
  661        pet3d_filename=[],
  662):
  663    """
  664    Generate a DataFrame for medical imaging data with extensive validation of input parameters.
  665
  666    This function creates a DataFrame containing information about medical imaging files,
  667    ensuring that filenames match expected patterns for their modalities and that all
  668    required images exist. It also validates the number of filenames provided for specific
  669    modalities like rsfMRI, DTI, and NM.
  670
  671    Parameters:
  672    - projectID (str): Project identifier.
  673    - subjectID (str): Subject identifier.
  674    - date (str): Date of the imaging study.
  675    - imageUniqueID (str): Unique image identifier.
  676    - modality (str): Modality of the imaging study.
  677    - source_image_directory (str): Directory of the source images.
  678    - output_image_directory (str): Directory for output images.
  679    - t1_filename (str): Filename of the T1-weighted image.
  680    - flair_filename (list): List of filenames for FLAIR images.
  681    - rsf_filenames (list): List of filenames for rsfMRI images.
  682    - dti_filenames (list): List of filenames for DTI images.
  683    - nm_filenames (list): List of filenames for NM images.
  684    - perf_filename (list): List of filenames for perfusion images.
  685    - pet3d_filename (list): List of filenames for pet3d images.
  686
  687    Returns:
  688    - pandas.DataFrame: A DataFrame containing the validated imaging study information.
  689
  690    Raises:
  691    - ValueError: If any validation checks fail or if the number of columns does not match the data.
  692    """
  693    def check_pd_construction(data, columns):
  694        # Check if the length of columns matches the length of data in each row
  695        if all(len(row) == len(columns) for row in data):
  696            return True
  697        else:
  698            return False
  699    from os.path import exists
  700    valid_modalities = get_valid_modalities()
  701    if not isinstance(t1_filename, str):
  702        raise ValueError("t1_filename is not a string")
  703    if not exists(t1_filename):
  704        raise ValueError("t1_filename does not exist")
  705    if modality not in valid_modalities:
  706        raise ValueError('modality ' + str(modality) + " not a valid mm modality:  " + get_valid_modalities(asString=True))
  707    # if not exists( output_image_directory ):
  708    #    raise ValueError("output_image_directory does not exist")
  709    if not exists( source_image_directory ):
  710        raise ValueError("source_image_directory does not exist")
  711    if len( rsf_filenames ) > 2:
  712        raise ValueError("len( rsf_filenames ) > 2")
  713    if len( dti_filenames ) > 3:
  714        raise ValueError("len( dti_filenames ) > 3")
  715    if len( nm_filenames ) > 11:
  716        raise ValueError("len( nm_filenames ) > 11")
  717    if len( rsf_filenames ) < 2:
  718        for k in range(len(rsf_filenames),2):
  719            rsf_filenames.append(None)
  720    if len( dti_filenames ) < 3:
  721        for k in range(len(dti_filenames),3):
  722            dti_filenames.append(None)
  723    if len( nm_filenames ) < 10:
  724        for k in range(len(nm_filenames),10):
  725            nm_filenames.append(None)
  726    # check modality names
  727    if not "T1w" in t1_filename:
  728        raise ValueError("T1w is not in t1 filename " + t1_filename)
  729    if flair_filename is not None:
  730        if isinstance(flair_filename,list):
  731            if (len(flair_filename) == 0):
  732                flair_filename=None
  733            else:
  734                print("Take first entry from flair_filename list")
  735                flair_filename=flair_filename[0]
  736    if flair_filename is not None and not "lair" in flair_filename:
  737            raise ValueError("flair is not flair filename " + flair_filename)
  738    ## perfusion
  739    if perf_filename is not None:
  740        if isinstance(perf_filename,list):
  741            if (len(perf_filename) == 0):
  742                perf_filename=None
  743            else:
  744                print("Take first entry from perf_filename list")
  745                perf_filename=perf_filename[0]
  746    if perf_filename is not None and not "perf" in perf_filename:
  747            raise ValueError("perf_filename is not perf filename " + perf_filename)
  748
  749    if pet3d_filename is not None:
  750        if isinstance(pet3d_filename,list):
  751            if (len(pet3d_filename) == 0):
  752                pet3d_filename=None
  753            else:
  754                print("Take first entry from pet3d_filename list")
  755                pet3d_filename=pet3d_filename[0]
  756    if pet3d_filename is not None and not "pet" in pet3d_filename:
  757            raise ValueError("pet3d_filename is not pet filename " + pet3d_filename)
  758    
  759    for k in nm_filenames:
  760        if k is not None:
  761            if not "NM" in k:
  762                raise ValueError("NM is not flair filename " + k)
  763    for k in dti_filenames:
  764        if k is not None:
  765            if not "DTI" in k and not "dwi" in k:
  766                raise ValueError("DTI/DWI is not dti filename " + k)
  767    for k in rsf_filenames:
  768        if k is not None:
  769            if not "fMRI" in k and not "func" in k:
  770                raise ValueError("rsfMRI/func is not rsfmri filename " + k)
  771    if perf_filename is not None:
  772        if not "perf" in perf_filename:
  773                raise ValueError("perf_filename is not a valid perfusion (perf) filename " + k)
  774    allfns = [t1_filename] + [flair_filename] + nm_filenames + dti_filenames + rsf_filenames + [perf_filename] + [pet3d_filename]
  775    for k in allfns:
  776        if k is not None:
  777            if not isinstance(k, str):
  778                raise ValueError(str(k) + " is not a string")
  779            if not exists( k ):
  780                raise ValueError( "image " + k + " does not exist")
  781    coredata = [
  782        projectID,
  783        subjectID,
  784        date,
  785        imageUniqueID,
  786        modality,
  787        source_image_directory,
  788        output_image_directory,
  789        t1_filename,
  790        flair_filename, 
  791        perf_filename,
  792        pet3d_filename]
  793    mydata0 = coredata +  rsf_filenames + dti_filenames
  794    mydata = mydata0 + nm_filenames
  795    corecols = [
  796        'projectID',
  797        'subjectID',
  798        'date',
  799        'imageID',
  800        'modality',
  801        'sourcedir',
  802        'outputdir',
  803        'filename',
  804        'flairid',
  805        'perfid',
  806        'pet3did']
  807    mycols0 = corecols + [
  808        'rsfid1', 'rsfid2',
  809        'dtid1', 'dtid2','dtid3']
  810    nmext = [
  811        'nmid1', 'nmid2', 'nmid3', 'nmid4', 'nmid5',
  812        'nmid6', 'nmid7','nmid8', 'nmid9', 'nmid10' #, 'nmid11'
  813    ]
  814    mycols = mycols0 + nmext
  815    if not check_pd_construction( [mydata], mycols ) :
  816#        print( mydata )
  817#        print( len(mydata ))
  818#        print( mycols )
  819#        print( len(mycols ))
  820        raise ValueError( "Error in generate_mm_dataframe: len( mycols ) != len( mydata ) which indicates a bad input parameter to this function." )
  821    studycsv = pd.DataFrame([ mydata ], columns=mycols)
  822    return studycsv
  823
  824import pandas as pd
  825from os.path import exists
  826
  827def validate_filename(filename, valid_keywords, error_message):
  828    """
  829    Validate if the given filename contains any of the specified keywords.
  830
  831    Parameters:
  832    - filename (str): The filename to validate.
  833    - valid_keywords (list): A list of keywords to look for in the filename.
  834    - error_message (str): The error message to raise if validation fails.
  835
  836    Raises:
  837    - ValueError: If none of the keywords are found in the filename.
  838    """
  839    if filename is not None and not any(keyword in filename for keyword in valid_keywords):
  840        raise ValueError(error_message)
  841
  842def validate_modality(modality, valid_modalities):
  843    if modality not in valid_modalities:
  844        valid_modalities_str = ', '.join(valid_modalities)
  845        raise ValueError(f'Modality {modality} not a valid mm modality: {valid_modalities_str}')
  846
  847def extend_list_to_length(lst, target_length, fill_value=None):
  848    return lst + [fill_value] * (target_length - len(lst))
  849
  850def generate_mm_dataframe_gpt(
  851        projectID, subjectID, date, imageUniqueID, modality, 
  852        source_image_directory, output_image_directory, t1_filename, 
  853        flair_filename=[], rsf_filenames=[], dti_filenames=[], nm_filenames=[], perf_filename=[] ):
  854    """
  855    see help for generate_mm_dataframe - same as this
  856    """
  857    def check_pd_construction(data, columns):
  858        return all(len(row) == len(columns) for row in data)
  859
  860    flair_filename.sort()
  861    rsf_filenames.sort()
  862    dti_filenames.sort()
  863    nm_filenames.sort()
  864    perf_filename.sort()
  865
  866    valid_modalities = get_valid_modalities()  
  867
  868    if not isinstance(t1_filename, str):
  869        raise ValueError("t1_filename is not a string")
  870    if not exists(t1_filename):
  871        raise ValueError("t1_filename does not exist")
  872
  873    validate_modality(modality, valid_modalities)
  874
  875    if not exists(source_image_directory):
  876        raise ValueError("source_image_directory does not exist")
  877
  878    rsf_filenames = extend_list_to_length(rsf_filenames, 2)
  879    dti_filenames = extend_list_to_length(dti_filenames, 2)
  880    nm_filenames = extend_list_to_length(nm_filenames, 11)
  881
  882    validate_filename(t1_filename, ["T1w"], "T1w is not in t1 filename " + t1_filename)
  883
  884    if flair_filename:
  885        flair_filename = flair_filename[0] if isinstance(flair_filename, list) else flair_filename
  886        validate_filename(flair_filename, ["lair"], "flair is not in flair filename " + flair_filename)
  887
  888    if perf_filename:
  889        perf_filename = perf_filename[0] if isinstance(perf_filename, list) else perf_filename
  890        validate_filename(perf_filename, ["perf"], "perf_filename is not a valid perfusion (perf) filename")
  891
  892    for k in nm_filenames:
  893        if k: validate_filename(k, ["NM"], "NM is not in NM filename " + k)
  894
  895    for k in dti_filenames:
  896        if k: validate_filename(k, ["DTI","dwi"], "DTI or dwi is not in DTI filename " + k)
  897
  898    for k in rsf_filenames:
  899        if k: validate_filename(k, ["fMRI","func"], "rsfMRI or func is not in rsfMRI filename " + k)
  900
  901    allfns = [t1_filename, flair_filename] + nm_filenames + dti_filenames + rsf_filenames + [perf_filename]
  902    for k in allfns:
  903        if k and not exists(k):
  904            raise ValueError("image " + k + " does not exist")
  905
  906    coredata = [projectID, subjectID, date, imageUniqueID, modality,
  907                source_image_directory, output_image_directory, t1_filename, 
  908                flair_filename, perf_filename]
  909    mydata0 = coredata + rsf_filenames + dti_filenames
  910    mydata = mydata0 + nm_filenames
  911
  912    corecols = ['projectID', 'subjectID', 'date', 'imageID', 'modality',
  913                'sourcedir', 'outputdir', 'filename', 'flairid', 'perfid']
  914    mycols0 = corecols + ['rsfid1', 'rsfid2', 'dtid1', 'dtid2']
  915    nmext = ['nmid1', 'nmid2', 'nmid3', 'nmid4', 'nmid5',
  916             'nmid6', 'nmid7', 'nmid8', 'nmid9', 'nmid10', 'nmid11']
  917    mycols = mycols0 + nmext
  918
  919    if not check_pd_construction([mydata], mycols):
  920        print( mydata )
  921        print( mycols )
  922        raise ValueError("Error in generate_mm_dataframe: len(mycols) != len(mydata), indicating bad input parameters.")
  923
  924    studycsv = pd.DataFrame([mydata], columns=mycols)
  925    return studycsv
  926
  927
  928
  929def filter_columns_by_nan_percentage(df, max_nan_percentage=50.0):
  930    """
  931    Filter columns in a DataFrame based on a threshold for the percentage of NaN values.
  932
  933    Parameters
  934    ----------
  935    df : pandas.DataFrame
  936        The input DataFrame from which columns are to be filtered.
  937    max_nan_percentage : float, optional
  938        The maximum allowed percentage of NaN values in a column. Columns with a higher
  939        percentage of NaN values than this threshold will be removed from the DataFrame.
  940        The default is 50.0, which means columns with more than 50% NaN values will be removed.
  941
  942    Returns
  943    -------
  944    pandas.DataFrame
  945        A DataFrame with columns filtered based on the NaN values percentage criterion.
  946
  947    Examples
  948    --------
  949    >>> import pandas as pd
  950    >>> data = {'A': [1, 2, None, 4], 'B': [None, 2, 3, None], 'C': [1, 2, 3, 4]}
  951    >>> df = pd.DataFrame(data)
  952    >>> filtered_df = filter_columns_by_nan_percentage(df, 50.0)
  953    >>> print(filtered_df)
  954
  955    Notes
  956    -----
  957    The function calculates the percentage of NaN values in each column and filters out
  958    those columns where the percentage exceeds the `max_nan_percentage` threshold.
  959    """
  960    # Calculate the percentage of NaN values for each column
  961    nan_percentage = df.isnull().mean() * 100
  962
  963    # Filter columns where the percentage of NaN values is less than or equal to the threshold
  964    columns_to_keep = nan_percentage[nan_percentage <= max_nan_percentage].index
  965
  966    # Return the filtered DataFrame
  967    return df[columns_to_keep]
  968
  969
  970
  971def parse_nrg_filename( x, separator='-' ):
  972    """
  973    split a NRG filename into its named parts
  974    """
  975    temp = x.split( separator )
  976    if len(temp) != 5:
  977        raise ValueError(x + " not a valid NRG filename")
  978    return {
  979        'project':temp[0],
  980        'subjectID':temp[1],
  981        'date':temp[2],
  982        'modality':temp[3],
  983        'imageID':temp[4]
  984    }
  985
  986
  987
  988def nrg_2_bids( nrg_filename ):
  989    """
  990    Convert an NRG filename to BIDS path/filename.
  991
  992    Parameters:
  993    nrg_filename (str): The NRG filename to convert.
  994
  995    Returns:
  996    str: The BIDS path/filename.
  997    """
  998
  999    # Split the NRG filename into its components
 1000    nrg_dirname, nrg_basename = os.path.split(nrg_filename)
 1001    nrg_suffix = '.' + nrg_basename.split('.',1)[-1]
 1002    nrg_basename = nrg_basename.replace(nrg_suffix, '') # remove ext
 1003    nrg_parts = nrg_basename.split('-')
 1004    nrg_subject_id = nrg_parts[1]
 1005    nrg_modality = nrg_parts[3]
 1006    nrg_repeat= nrg_parts[4]
 1007
 1008    # Build the BIDS path/filename
 1009    bids_dirname = os.path.join(nrg_dirname, 'bids')
 1010    bids_subject = f'sub-{nrg_subject_id}'
 1011    bids_session = f'ses-{nrg_repeat}'
 1012
 1013    valid_modalities = get_valid_modalities()
 1014    if nrg_modality is not None:
 1015        if not nrg_modality in valid_modalities:
 1016            raise ValueError('nrg_modality ' + str(nrg_modality) + " not a valid mm modality:  " + get_valid_modalities(asString=True))
 1017
 1018    if nrg_modality == 'T1w' :
 1019        bids_modality_folder = 'anat'
 1020        bids_modality_filename = 'T1w'
 1021
 1022    if nrg_modality == 'T2Flair' :
 1023        bids_modality_folder = 'anat'
 1024        bids_modality_filename = 'flair'
 1025
 1026    if nrg_modality == 'NM2DMT' :
 1027        bids_modality_folder = 'anat'
 1028        bids_modality_filename = 'nm2dmt'
 1029
 1030    if nrg_modality == 'DTI' or nrg_modality == 'DTI_RL' or nrg_modality == 'DTI_LR' :
 1031        bids_modality_folder = 'dwi'
 1032        bids_modality_filename = 'dwi'
 1033
 1034    if nrg_modality == 'rsfMRI' or nrg_modality == 'rsfMRI_RL' or nrg_modality == 'rsfMRI_LR' :
 1035        bids_modality_folder = 'func'
 1036        bids_modality_filename = 'func'
 1037
 1038    if nrg_modality == 'perf'  :
 1039        bids_modality_folder = 'perf'
 1040        bids_modality_filename = 'perf'
 1041
 1042    bids_suffix = nrg_suffix[1:]
 1043    bids_filename = f'{bids_subject}_{bids_session}_{bids_modality_filename}.{bids_suffix}'
 1044
 1045    # Return bids filepath/filename
 1046    return os.path.join(bids_dirname, bids_subject, bids_session, bids_modality_folder, bids_filename)
 1047
 1048
 1049def bids_2_nrg( bids_filename, project_name, date, nrg_modality=None ):
 1050    """
 1051    Convert a BIDS filename to NRG path/filename.
 1052
 1053    Parameters:
 1054    bids_filename (str): The BIDS filename to convert
 1055    project_name (str) : Name of project (i.e. PPMI)
 1056    date (str) : Date of image acquisition
 1057
 1058
 1059    Returns:
 1060    str: The NRG path/filename.
 1061    """
 1062
 1063    bids_dirname, bids_basename = os.path.split(bids_filename)
 1064    bids_suffix = '.'+ bids_basename.split('.',1)[-1]
 1065    bids_basename = bids_basename.replace(bids_suffix, '') # remove ext
 1066    bids_parts = bids_basename.split('_')
 1067    nrg_subject_id = bids_parts[0].replace('sub-','')
 1068    nrg_image_id = bids_parts[1].replace('ses-', '')
 1069    bids_modality = bids_parts[2]
 1070    valid_modalities = get_valid_modalities()
 1071    if nrg_modality is not None:
 1072        if not nrg_modality in valid_modalities:
 1073            raise ValueError('nrg_modality ' + str(nrg_modality) + " not a valid mm modality: " + get_valid_modalities(asString=True))
 1074
 1075    if bids_modality == 'anat' and nrg_modality is None :
 1076        nrg_modality = 'T1w'
 1077
 1078    if bids_modality == 'dwi' and nrg_modality is None  :
 1079        nrg_modality = 'DTI'
 1080
 1081    if bids_modality == 'func' and nrg_modality is None  :
 1082        nrg_modality = 'rsfMRI'
 1083
 1084    if bids_modality == 'perf' and nrg_modality is None  :
 1085        nrg_modality = 'perf'
 1086
 1087    nrg_suffix = bids_suffix[1:]
 1088    nrg_filename = f'{project_name}-{nrg_subject_id}-{date}-{nrg_modality}-{nrg_image_id}.{nrg_suffix}'
 1089
 1090    return os.path.join(project_name, nrg_subject_id, date, nrg_modality, nrg_image_id,nrg_filename)
 1091
 1092def collect_blind_qc_by_modality( modality_path, set_index_to_fn=True ):
 1093    """
 1094    Collects blind QC data from multiple CSV files with the same modality.
 1095
 1096    Args:
 1097
 1098    modality_path (str): The path to the folder containing the CSV files.
 1099
 1100    set_index_to_fn: boolean
 1101
 1102    Returns:
 1103    Pandas DataFrame: A DataFrame containing all the blind QC data from the CSV files.
 1104    """
 1105    import glob as glob
 1106    fns = glob.glob( modality_path )
 1107    fns.sort()
 1108    jdf = pd.DataFrame()
 1109    for k in range(len(fns)):
 1110        temp=pd.read_csv(fns[k])
 1111        if not 'filename' in temp.keys():
 1112            temp['filename']=fns[k]
 1113        jdf=pd.concat( [jdf,temp], axis=0, ignore_index=False )
 1114    if set_index_to_fn:
 1115        jdf.reset_index(drop=True)
 1116        if "Unnamed: 0" in jdf.columns:
 1117            holder=jdf.pop( "Unnamed: 0" )
 1118        jdf.set_index('filename')
 1119    return jdf
 1120
 1121
 1122def outlierness_by_modality( qcdf, uid='filename', outlier_columns = ['noise', 'snr', 'cnr', 'psnr', 'ssim', 'mi','reflection_err', 'EVR', 'msk_vol'], verbose=False ):
 1123    """
 1124    Calculates outlierness scores for each modality in a dataframe based on given outlier columns using antspyt1w.loop_outlierness() and LOF.  LOF appears to be more conservative.  This function will impute missing columns with the mean.
 1125
 1126    Args:
 1127    - qcdf: (Pandas DataFrame) Dataframe containing columns with outlier information for each modality.
 1128    - uid: (str) Unique identifier for a subject. Default is 'filename'.
 1129    - outlier_columns: (list) List of columns containing outlier information. Default is ['noise', 'snr', 'cnr', 'psnr', 'ssim', 'mi', 'reflection_err', 'EVR', 'msk_vol'].
 1130    - verbose: (bool) If True, prints information for each modality. Default is False.
 1131
 1132    Returns:
 1133    - qcdf: (Pandas DataFrame) Updated dataframe with outlierness scores for each modality in the 'ol_loop' and 'ol_lof' column.  Higher values near 1 are more outlying.
 1134
 1135    Raises:
 1136    - ValueError: If uid is not present in the dataframe.
 1137
 1138    Example:
 1139    >>> df = pd.read_csv('data.csv')
 1140    >>> outlierness_by_modality(df)
 1141    """
 1142    from PyNomaly import loop
 1143    from sklearn.neighbors import LocalOutlierFactor
 1144    qcdfout = qcdf.copy()
 1145    pd.set_option('future.no_silent_downcasting', True)
 1146    qcdfout.replace([np.inf, -np.inf], np.nan, inplace=True)
 1147    if uid not in qcdfout.keys():
 1148        raise ValueError( str(uid) + " not in dataframe")
 1149    if 'ol_loop' not in qcdfout.keys():
 1150        qcdfout['ol_loop']=math.nan
 1151    if 'ol_lof' not in qcdfout.keys():
 1152        qcdfout['ol_lof']=math.nan
 1153    didit=False
 1154    for mod in get_valid_modalities( qc=True ):
 1155        didit=True
 1156        lof = LocalOutlierFactor()
 1157        locsel = qcdfout["modality"] == mod
 1158        rr = qcdfout[locsel][outlier_columns]
 1159        column_means = rr.mean()
 1160        rr.fillna(column_means, inplace=True)
 1161        if rr.shape[0] > 1:
 1162            if verbose:
 1163                print("calc: " + mod + " outlierness " )
 1164            myneigh = np.min( [24, int(np.round(rr.shape[0]*0.5)) ] )
 1165            temp = antspyt1w.loop_outlierness(rr.astype(float), standardize=True, extent=3, n_neighbors=myneigh, cluster_labels=None)
 1166            qcdfout.loc[locsel,'ol_loop']=temp.astype('float64')
 1167            yhat = lof.fit_predict(rr)
 1168            temp = lof.negative_outlier_factor_*(-1.0)
 1169            temp = temp - temp.min()
 1170            yhat[ yhat == 1] = 0
 1171            yhat[ yhat == -1] = 1 # these are outliers
 1172            qcdfout.loc[locsel,'ol_lof_decision']=yhat
 1173            qcdfout.loc[locsel,'ol_lof']=temp/temp.max()
 1174    if verbose:
 1175        print( didit )
 1176    return qcdfout
 1177
 1178
 1179def nrg_format_path( projectID, subjectID, date, modality, imageID, separator='-' ):
 1180    """
 1181    create the NRG path on disk given the project, subject id, date, modality and image id
 1182
 1183    Arguments
 1184    ---------
 1185
 1186    projectID : string for the project e.g. PPMI
 1187
 1188    subjectID : string uniquely identifying the subject e.g. 0001
 1189
 1190    date : string for the date usually 20550228 ie YYYYMMDD format
 1191
 1192    modality : string should be one of T1w, T2Flair, rsfMRI, NM2DMT and DTI ... rsfMRI and DTI may also be DTI_LR, DTI_RL, rsfMRI_LR and rsfMRI_RL where the RL / LR relates to phase encoding direction (even if it is AP/PA)
 1193
 1194    imageID : string uniquely identifying the specific image
 1195
 1196    separator : default to -
 1197
 1198    Returns
 1199    -------
 1200    the path where one would write the image on disk
 1201
 1202    """
 1203    thedirectory = os.path.join( str(projectID), str(subjectID), str(date), str(modality), str(imageID) )
 1204    thefilename = str(projectID) + separator + str(subjectID) + separator + str(date) + separator + str(modality) + separator + str(imageID)
 1205    return os.path.join( thedirectory, thefilename )
 1206
 1207
 1208def get_first_item_as_string(df, column_name):
 1209    """
 1210    Check if the first item in the specified column of the DataFrame is a string.
 1211    If it is not a string, attempt to convert it to an integer and then to a string.
 1212
 1213    Parameters:
 1214    df (pd.DataFrame): The DataFrame to operate on.
 1215    column_name (str): The name of the column to check.
 1216
 1217    Returns:
 1218    str: The first item in the specified column, guaranteed to be returned as a string.
 1219    """
 1220    if isinstance(df[column_name].iloc[0], str):
 1221        return df[column_name].iloc[0]
 1222    else:
 1223        try:
 1224            return str(int(df[column_name].iloc[0]))
 1225        except ValueError:
 1226            raise ValueError("The value cannot be converted to an integer.")
 1227
 1228
 1229def study_dataframe_from_matched_dataframe( matched_dataframe, rootdir, outputdir, verbose=False ):
 1230    """
 1231    converts the output of antspymm.match_modalities dataframe (one row) to that needed for a study-driving dataframe for input to mm_csv
 1232
 1233    matched_dataframe : output of antspymm.match_modalities
 1234
 1235    rootdir : location for the input data root folder (in e.g. NRG format)
 1236
 1237    outputdir : location for the output data
 1238
 1239    verbose : boolean
 1240    """
 1241    iext='.nii.gz'
 1242    from os.path import exists
 1243    musthavecols = ['projectID', 'subjectID','date','imageID','filename']
 1244    for k in range(len(musthavecols)):
 1245        if not musthavecols[k] in matched_dataframe.keys():
 1246            raise ValueError('matched_dataframe is missing column ' + musthavecols[k] + ' in study_dataframe_from_qc_dataframe' )
 1247    csvrow=matched_dataframe.dropna(axis=1)
 1248    pid=get_first_item_as_string( csvrow, 'projectID'  )
 1249    sid=get_first_item_as_string( csvrow, 'subjectID'  ) # str(csvrow['subjectID'].iloc[0] )
 1250    dt=get_first_item_as_string( csvrow, 'date'  )  # str(csvrow['date'].iloc[0])
 1251    iid=get_first_item_as_string( csvrow, 'imageID'  ) # str(csvrow['imageID'].iloc[0])
 1252    nrgt1fn=os.path.join( rootdir, pid, sid, dt, 'T1w', iid, str(csvrow['filename'].iloc[0]+iext) )
 1253    if not exists( nrgt1fn ) and iid == '0':
 1254        iid='000'
 1255        nrgt1fn=os.path.join( rootdir, pid, sid, dt, 'T1w', iid, str(csvrow['filename'].iloc[0]+iext) )
 1256    if not exists( nrgt1fn ):
 1257        raise ValueError("T1 " + nrgt1fn + " does not exist in study_dataframe_from_qc_dataframe")
 1258    flList=[]
 1259    dtList=[]
 1260    rsfList=[]
 1261    nmList=[]
 1262    perfList=[]
 1263    if 'flairfn' in csvrow.keys():
 1264        flid=get_first_item_as_string( csvrow, 'flairid' )
 1265        nrgt2fn=os.path.join( rootdir, pid, sid, dt, 'T2Flair', flid, str(csvrow['flairfn'].iloc[0]+iext) )
 1266        if not exists( nrgt2fn ) and flid == '0':
 1267            flid='000'
 1268            nrgt2fn=os.path.join( rootdir, pid, sid, dt, 'T2Flair', flid, str(csvrow['flairfn'].iloc[0]+iext) )
 1269        if verbose:
 1270            print("Trying " + nrgt2fn )
 1271        if exists( nrgt2fn ):
 1272            if verbose:
 1273                print("success" )
 1274            flList.append( nrgt2fn )
 1275    if 'perffn' in csvrow.keys():
 1276        flid=get_first_item_as_string( csvrow, 'perfid' )
 1277        nrgt2fn=os.path.join( rootdir, pid, sid, dt, 'perf', flid, str(csvrow['perffn'].iloc[0]+iext) )
 1278        if not exists( nrgt2fn ) and flid == '0':
 1279            flid='000'
 1280            nrgt2fn=os.path.join( rootdir, pid, sid, dt, 'perf', flid, str(csvrow['perffn'].iloc[0]+iext) )
 1281        if verbose:
 1282            print("Trying " + nrgt2fn )
 1283        if exists( nrgt2fn ):
 1284            if verbose:
 1285                print("success" )
 1286            perfList.append( nrgt2fn )
 1287    if 'dtfn1' in csvrow.keys():
 1288        dtid=get_first_item_as_string( csvrow, 'dtid1' )
 1289        dtfn1=glob.glob(os.path.join( rootdir, pid, sid, dt, 'DTI*', dtid, str(csvrow['dtfn1'].iloc[0]+iext) ))
 1290        if len( dtfn1) == 0 :
 1291            dtid = '000'
 1292            dtfn1=glob.glob(os.path.join( rootdir, pid, sid, dt, 'DTI*', dtid, str(csvrow['dtfn1'].iloc[0]+iext) ))
 1293        dtfn1=dtfn1[0]
 1294        if exists( dtfn1 ):
 1295            dtList.append( dtfn1 )
 1296    if 'dtfn2' in csvrow.keys():
 1297        dtid=get_first_item_as_string( csvrow, 'dtid2' )
 1298        dtfn2=glob.glob(os.path.join(rootdir, pid, sid, dt, 'DTI*', dtid, str(csvrow['dtfn2'].iloc[0]+iext) ))
 1299        if len( dtfn2) == 0 :
 1300            dtid = '000'
 1301            dtfn2=glob.glob(os.path.join( rootdir, pid, sid, dt, 'DTI*', dtid, str(csvrow['dtfn2'].iloc[0]+iext) ))
 1302        dtfn2=dtfn2[0]
 1303        if exists( dtfn2 ):
 1304            dtList.append( dtfn2 )
 1305    if 'dtfn3' in csvrow.keys():
 1306        dtid=get_first_item_as_string( csvrow, 'dtid3' )
 1307        dtfn3=glob.glob(os.path.join(rootdir, pid, sid, dt, 'DTI*', dtid, str(csvrow['dtfn3'].iloc[0]+iext) ))
 1308        if len( dtfn3) == 0 :
 1309            dtid = '000'
 1310            dtfn3=glob.glob(os.path.join( rootdir, pid, sid, dt, 'DTI*', dtid, str(csvrow['dtfn3'].iloc[0]+iext) ))
 1311        dtfn3=dtfn3[0]
 1312        if exists( dtfn3 ):
 1313            dtList.append( dtfn3 )
 1314    if 'rsffn1' in csvrow.keys():
 1315        rsid=get_first_item_as_string( csvrow, 'rsfid1' )
 1316        rsfn1=glob.glob(os.path.join( rootdir, pid, sid, dt, 'rsfMRI*', rsid, str(csvrow['rsffn1'].iloc[0]+iext) ))
 1317        if len( rsfn1 ) == 0 :
 1318            rsid = '000'
 1319            rsfn1=glob.glob(os.path.join( rootdir, pid, sid, dt, 'rsfMRI*', rsid, str(csvrow['rsffn1'].iloc[0]+iext) ))
 1320        rsfn1=rsfn1[0]
 1321        if exists( rsfn1 ):
 1322            rsfList.append( rsfn1 )
 1323    if 'rsffn2' in csvrow.keys():
 1324        rsid=get_first_item_as_string( csvrow, 'rsfid2' )
 1325        rsfn2=glob.glob(os.path.join( rootdir, pid, sid, dt, 'rsfMRI*', rsid, str(csvrow['rsffn2'].iloc[0]+iext) ))[0]
 1326        if len( rsfn2 ) == 0 :
 1327            rsid = '000'
 1328            rsfn2=glob.glob(os.path.join( rootdir, pid, sid, dt, 'rsfMRI*', rsid, str(csvrow['rsffn2'].iloc[0]+iext) ))
 1329        rsfn2=rsfn2[0]
 1330        if exists( rsfn2 ):
 1331            rsfList.append( rsfn2 )
 1332    for j in range(11):
 1333        keyname="nmfn"+str(j)
 1334        keynameid="nmid"+str(j)
 1335        if keyname in csvrow.keys() and keynameid in csvrow.keys():
 1336            nmid=get_first_item_as_string( csvrow, keynameid )
 1337            nmsearchpath=os.path.join( rootdir, pid, sid, dt, 'NM2DMT', nmid, "*"+nmid+iext)
 1338            nmfn=glob.glob( nmsearchpath )
 1339            nmfn=nmfn[0]
 1340            if exists( nmfn ):
 1341                nmList.append( nmfn )
 1342    if verbose:
 1343        print("assembled the image lists mapping to ....")
 1344        print(nrgt1fn)
 1345        print("NM")
 1346        print(nmList)
 1347        print("FLAIR")
 1348        print(flList)
 1349        print("DTI")
 1350        print(dtList)
 1351        print("rsfMRI")
 1352        print(rsfList)
 1353        print("perf")
 1354        print(perfList)
 1355    studycsv = generate_mm_dataframe(
 1356        pid,
 1357        sid,
 1358        dt,
 1359        iid, # the T1 id
 1360        'T1w',
 1361        rootdir,
 1362        outputdir,
 1363        t1_filename=nrgt1fn,
 1364        flair_filename=flList,
 1365        dti_filenames=dtList,
 1366        rsf_filenames=rsfList,
 1367        nm_filenames=nmList,
 1368        perf_filename=perfList)
 1369    return studycsv.dropna(axis=1)
 1370
 1371def highest_quality_repeat(mxdfin, idvar, visitvar, qualityvar):
 1372    """
 1373    This function returns a subset of the input dataframe that retains only the rows
 1374    that correspond to the highest quality observation for each combination of ID and visit.
 1375
 1376    Parameters:
 1377    ----------
 1378    mxdfin: pandas.DataFrame
 1379        The input dataframe.
 1380    idvar: str
 1381        The name of the column that contains the ID variable.
 1382    visitvar: str
 1383        The name of the column that contains the visit variable.
 1384    qualityvar: str
 1385        The name of the column that contains the quality variable.
 1386
 1387    Returns:
 1388    -------
 1389    pandas.DataFrame
 1390        A subset of the input dataframe that retains only the rows that correspond
 1391        to the highest quality observation for each combination of ID and visit.
 1392    """
 1393    if visitvar not in mxdfin.columns:
 1394        raise ValueError("visitvar not in dataframe")
 1395    if idvar not in mxdfin.columns:
 1396        raise ValueError("idvar not in dataframe")
 1397    if qualityvar not in mxdfin.columns:
 1398        raise ValueError("qualityvar not in dataframe")
 1399
 1400    mxdfin[qualityvar] = mxdfin[qualityvar].astype(float)
 1401
 1402    vizzes = mxdfin[visitvar].unique()
 1403    uids = mxdfin[idvar].unique()
 1404    useit = np.zeros(mxdfin.shape[0], dtype=bool)
 1405
 1406    for u in uids:
 1407        losel = mxdfin[idvar] == u
 1408        vizzesloc = mxdfin[losel][visitvar].unique()
 1409
 1410        for v in vizzesloc:
 1411            losel = (mxdfin[idvar] == u) & (mxdfin[visitvar] == v)
 1412            mysnr = mxdfin.loc[losel, qualityvar]
 1413            myw = np.where(losel)[0]
 1414
 1415            if len(myw) > 1:
 1416                if any(~np.isnan(mysnr)):
 1417                    useit[myw[np.argmax(mysnr)]] = True
 1418                else:
 1419                    useit[myw] = True
 1420            else:
 1421                useit[myw] = True
 1422
 1423    return mxdfin[useit]
 1424
 1425
 1426def match_modalities( qc_dataframe, unique_identifier='filename', outlier_column='ol_loop', mysep='-', verbose=False ):
 1427    """
 1428    Find the best multiple modality dataset at each time point
 1429
 1430    :param qc_dataframe: quality control data frame with
 1431    :param unique_identifier : the unique NRG filename for each image
 1432    :param outlier_column: outlierness score used to identify the best image (pair) at a given date
 1433    :param mysep (str, optional): the separator used in the image file names. Defaults to '-'.
 1434    :param verbose: boolean
 1435    :return: filtered matched modality data frame
 1436    """
 1437    import pandas as pd
 1438    import numpy as np
 1439    qc_dataframe['filename']=qc_dataframe['filename'].astype(str)
 1440    qc_dataframe['ol_loop']=qc_dataframe['ol_loop'].astype(float)
 1441    qc_dataframe['ol_lof']=qc_dataframe['ol_lof'].astype(float)
 1442    qc_dataframe['ol_lof_decision']=qc_dataframe['ol_lof_decision'].astype(float)
 1443    mmdf = best_mmm( qc_dataframe, 'T1w', outlier_column=outlier_column )['filt']
 1444    fldf = best_mmm( qc_dataframe, 'T2Flair', outlier_column=outlier_column )['filt']
 1445    nmdf = best_mmm( qc_dataframe, 'NM2DMT', outlier_column=outlier_column )['filt']
 1446    rsdf = best_mmm( qc_dataframe, 'rsfMRI', outlier_column=outlier_column )['filt']
 1447    dtdf = best_mmm( qc_dataframe, 'DTI', outlier_column=outlier_column )['filt']
 1448    mmdf['flairid'] = None
 1449    mmdf['flairfn'] = None
 1450    mmdf['flairloop'] = None
 1451    mmdf['flairlof'] = None
 1452    mmdf['dtid1'] = None
 1453    mmdf['dtfn1'] = None
 1454    mmdf['dtntimepoints1'] = 0
 1455    mmdf['dtloop1'] = math.nan
 1456    mmdf['dtlof1'] = math.nan
 1457    mmdf['dtid2'] = None
 1458    mmdf['dtfn2'] = None
 1459    mmdf['dtntimepoints2'] = 0
 1460    mmdf['dtloop2'] = math.nan
 1461    mmdf['dtlof2'] = math.nan
 1462    mmdf['rsfid1'] = None
 1463    mmdf['rsffn1'] = None
 1464    mmdf['rsfntimepoints1'] = 0
 1465    mmdf['rsfloop1'] = math.nan
 1466    mmdf['rsflof1'] = math.nan
 1467    mmdf['rsfid2'] = None
 1468    mmdf['rsffn2'] = None
 1469    mmdf['rsfntimepoints2'] = 0
 1470    mmdf['rsfloop2'] = math.nan
 1471    mmdf['rsflof2'] = math.nan
 1472    for k in range(1,11):
 1473        myid='nmid'+str(k)
 1474        mmdf[myid] = None
 1475        myid='nmfn'+str(k)
 1476        mmdf[myid] = None
 1477        myid='nmloop'+str(k)
 1478        mmdf[myid] = math.nan
 1479        myid='nmlof'+str(k)
 1480        mmdf[myid] = math.nan
 1481    if verbose:
 1482        print( mmdf.shape )
 1483    for k in range(mmdf.shape[0]):
 1484        if verbose:
 1485            if k % 100 == 0:
 1486                progger = str( k ) # np.round( k / mmdf.shape[0] * 100 ) )
 1487                print( progger, end ="...", flush=True)
 1488        if dtdf is not None:
 1489            locsel = (dtdf["subjectIDdate"] == mmdf["subjectIDdate"].iloc[k])
 1490            if sum(locsel) == 1:
 1491                mmdf.iloc[k, mmdf.columns.get_loc("dtid1")] = dtdf["imageID"][locsel].values[0]
 1492                mmdf.iloc[k, mmdf.columns.get_loc("dtfn1")] = dtdf[unique_identifier][locsel].values[0]
 1493                mmdf.iloc[k, mmdf.columns.get_loc("dtloop1")] = dtdf[outlier_column][locsel].values[0]
 1494                mmdf.iloc[k, mmdf.columns.get_loc("dtlof1")] = float(dtdf['ol_lof_decision'][locsel].values[0])
 1495                mmdf.iloc[k, mmdf.columns.get_loc("dtntimepoints1")] = float(dtdf['dimt'][locsel].values[0])
 1496            elif sum(locsel) > 1:
 1497                locdf = dtdf[locsel]
 1498                dedupe = locdf[["snr","cnr"]].duplicated()
 1499                locdf = locdf[~dedupe]
 1500                if locdf.shape[0] > 1:
 1501                    locdf = locdf.sort_values(outlier_column).iloc[:2]
 1502                mmdf.iloc[k, mmdf.columns.get_loc("dtid1")] = locdf["imageID"].values[0]
 1503                mmdf.iloc[k, mmdf.columns.get_loc("dtfn1")] = locdf[unique_identifier].values[0]
 1504                mmdf.iloc[k, mmdf.columns.get_loc("dtloop1")] = locdf[outlier_column].values[0]
 1505                mmdf.iloc[k, mmdf.columns.get_loc("dtlof1")] = float(locdf['ol_lof_decision'][locsel].values[0])
 1506                mmdf.iloc[k, mmdf.columns.get_loc("dtntimepoints1")] = float(dtdf['dimt'][locsel].values[0])
 1507                if locdf.shape[0] > 1:
 1508                    mmdf.iloc[k, mmdf.columns.get_loc("dtid2")] = locdf["imageID"].values[1]
 1509                    mmdf.iloc[k, mmdf.columns.get_loc("dtfn2")] = locdf[unique_identifier].values[1]
 1510                    mmdf.iloc[k, mmdf.columns.get_loc("dtloop2")] = locdf[outlier_column].values[1]
 1511                    mmdf.iloc[k, mmdf.columns.get_loc("dtlof2")] = float(locdf['ol_lof_decision'][locsel].values[1])
 1512                    mmdf.iloc[k, mmdf.columns.get_loc("dtntimepoints2")] = float(dtdf['dimt'][locsel].values[1])
 1513        if rsdf is not None:
 1514            locsel = (rsdf["subjectIDdate"] == mmdf["subjectIDdate"].iloc[k])
 1515            if sum(locsel) == 1:
 1516                mmdf.iloc[k, mmdf.columns.get_loc("rsfid1")] = rsdf["imageID"][locsel].values[0]
 1517                mmdf.iloc[k, mmdf.columns.get_loc("rsffn1")] = rsdf[unique_identifier][locsel].values[0]
 1518                mmdf.iloc[k, mmdf.columns.get_loc("rsfloop1")] = rsdf[outlier_column][locsel].values[0]
 1519                mmdf.iloc[k, mmdf.columns.get_loc("rsflof1")] = float(rsdf['ol_lof_decision'].values[0])
 1520                mmdf.iloc[k, mmdf.columns.get_loc("rsfntimepoints1")] = float(rsdf['dimt'][locsel].values[0])
 1521            elif sum(locsel) > 1:
 1522                locdf = rsdf[locsel]
 1523                dedupe = locdf[["snr","cnr"]].duplicated()
 1524                locdf = locdf[~dedupe]
 1525                if locdf.shape[0] > 1:
 1526                    locdf = locdf.sort_values(outlier_column).iloc[:2]
 1527                mmdf.iloc[k, mmdf.columns.get_loc("rsfid1")] = locdf["imageID"].values[0]
 1528                mmdf.iloc[k, mmdf.columns.get_loc("rsffn1")] = locdf[unique_identifier].values[0]
 1529                mmdf.iloc[k, mmdf.columns.get_loc("rsfloop1")] = locdf[outlier_column].values[0]
 1530                mmdf.iloc[k, mmdf.columns.get_loc("rsflof1")] = float(locdf['ol_lof_decision'].values[0])
 1531                mmdf.iloc[k, mmdf.columns.get_loc("rsfntimepoints1")] = float(locdf['dimt'][locsel].values[0])
 1532                if locdf.shape[0] > 1:
 1533                    mmdf.iloc[k, mmdf.columns.get_loc("rsfid2")] = locdf["imageID"].values[1]
 1534                    mmdf.iloc[k, mmdf.columns.get_loc("rsffn2")] = locdf[unique_identifier].values[1]
 1535                    mmdf.iloc[k, mmdf.columns.get_loc("rsfloop2")] = locdf[outlier_column].values[1]
 1536                    mmdf.iloc[k, mmdf.columns.get_loc("rsflof2")] = float(locdf['ol_lof_decision'].values[1])
 1537                    mmdf.iloc[k, mmdf.columns.get_loc("rsfntimepoints2")] = float(locdf['dimt'][locsel].values[1])
 1538
 1539        if fldf is not None:
 1540            locsel = fldf['subjectIDdate'] == mmdf['subjectIDdate'].iloc[k]
 1541            if locsel.sum() == 1:
 1542                mmdf.iloc[k, mmdf.columns.get_loc("flairid")] = fldf['imageID'][locsel].values[0]
 1543                mmdf.iloc[k, mmdf.columns.get_loc("flairfn")] = fldf[unique_identifier][locsel].values[0]
 1544                mmdf.iloc[k, mmdf.columns.get_loc("flairloop")] = fldf[outlier_column][locsel].values[0]
 1545                mmdf.iloc[k, mmdf.columns.get_loc("flairlof")] = float(fldf['ol_lof_decision'][locsel].values[0])
 1546            elif sum(locsel) > 1:
 1547                locdf = fldf[locsel]
 1548                dedupe = locdf[["snr","cnr"]].duplicated()
 1549                locdf = locdf[~dedupe]
 1550                if locdf.shape[0] > 1:
 1551                    locdf = locdf.sort_values(outlier_column).iloc[:2]
 1552                mmdf.iloc[k, mmdf.columns.get_loc("flairid")] = locdf["imageID"].values[0]
 1553                mmdf.iloc[k, mmdf.columns.get_loc("flairfn")] = locdf[unique_identifier].values[0]
 1554                mmdf.iloc[k, mmdf.columns.get_loc("flairloop")] = locdf[outlier_column].values[0]
 1555                mmdf.iloc[k, mmdf.columns.get_loc("flairlof")] = float(locdf['ol_lof_decision'].values[0])
 1556
 1557        if nmdf is not None:
 1558            locsel = nmdf['subjectIDdate'] == mmdf['subjectIDdate'].iloc[k]
 1559            if locsel.sum() > 0:
 1560                locdf = nmdf[locsel]
 1561                for i in range(np.min( [10,locdf.shape[0]])):
 1562                    nmid = "nmid"+str(i+1)
 1563                    mmdf.loc[k,nmid] = locdf['imageID'].iloc[i]
 1564                    nmfn = "nmfn"+str(i+1)
 1565                    mmdf.loc[k,nmfn] = locdf['imageID'].iloc[i]
 1566                    nmloop = "nmloop"+str(i+1)
 1567                    mmdf.loc[k,nmloop] = locdf[outlier_column].iloc[i]
 1568                    nmloop = "nmlof"+str(i+1)
 1569                    mmdf.loc[k,nmloop] = float(locdf['ol_lof_decision'].iloc[i])
 1570
 1571    mmdf['rsf_total_timepoints']=mmdf['rsfntimepoints1']+mmdf['rsfntimepoints2']
 1572    mmdf['dt_total_timepoints']=mmdf['dtntimepoints1']+mmdf['dtntimepoints2']
 1573    return mmdf
 1574
 1575
 1576def add_repeat_column(df, groupby_column):
 1577    """
 1578    Adds a 'repeat' column to the DataFrame that counts occurrences of each unique value
 1579    in the specified 'groupby_column'. The count increments from 1 for each identical entry.
 1580    
 1581    Parameters:
 1582    - df: pandas DataFrame.
 1583    - groupby_column: The name of the column to group by and count repeats.
 1584    
 1585    Returns:
 1586    - Modified pandas DataFrame with an added 'repeat' column.
 1587    """
 1588    # Validate if the groupby_column exists in the DataFrame
 1589    if groupby_column not in df.columns:
 1590        raise ValueError(f"Column '{groupby_column}' does not exist in the DataFrame.")
 1591    
 1592    # Count the occurrences of each unique value in the specified column and increment from 1
 1593    df['repeat'] = df.groupby(groupby_column).cumcount() + 1
 1594    
 1595    return df
 1596
 1597def best_mmm( mmdf, wmod, mysep='-', outlier_column='ol_loop', verbose=False):
 1598    """
 1599    Selects the best repeats per modality.
 1600
 1601    Args:
 1602    wmod (str): the modality of the image ( 'T1w', 'T2Flair', 'NM2DMT' 'rsfMRI', 'DTI')
 1603
 1604    mysep (str, optional): the separator used in the image file names. Defaults to '-'.
 1605
 1606    outlier_name : column name for outlier score
 1607
 1608    verbose (bool, optional): default True
 1609
 1610    Returns:
 1611
 1612    list: a list containing two metadata dataframes - raw and filt. raw contains all the metadata for the selected modality and filt contains the metadata filtered for highest quality repeats.
 1613
 1614    """
 1615#    mmdf = mmdf.astype(str)
 1616    mmdf[outlier_column]=mmdf[outlier_column].astype(float)
 1617    msel = mmdf['modality'] == wmod
 1618    if wmod == 'rsfMRI':
 1619        msel1 = mmdf['modality'] == 'rsfMRI'
 1620        msel2 = mmdf['modality'] == 'rsfMRI_LR'
 1621        msel3 = mmdf['modality'] == 'rsfMRI_RL'
 1622        msel = msel1 | msel2
 1623        msel = msel | msel3
 1624    if wmod == 'DTI':
 1625        msel1 = mmdf['modality'] == 'DTI'
 1626        msel2 = mmdf['modality'] == 'DTI_LR'
 1627        msel3 = mmdf['modality'] == 'DTI_RL'
 1628        msel4 = mmdf['modality'] == 'DTIdwi'
 1629        msel5 = mmdf['modality'] == 'DTIb0'
 1630        msel = msel1 | msel2 | msel3 | msel4 | msel5
 1631    if sum(msel) == 0:
 1632        return {'raw': None, 'filt': None}
 1633    metasub = mmdf[msel].copy()
 1634
 1635    if verbose:
 1636        print(f"{wmod} {(metasub.shape[0])} pre")
 1637
 1638    metasub['subjectID']=None
 1639    metasub['date']=None
 1640    metasub['subjectIDdate']=None
 1641    metasub['imageID']=None
 1642    metasub['negol']=math.nan
 1643    for k in metasub.index:
 1644        temp = metasub.loc[k, 'filename'].split( mysep )
 1645        metasub.loc[k,'subjectID'] = str( temp[1] )
 1646        metasub.loc[k,'date'] = str( temp[2] )
 1647        metasub.loc[k,'subjectIDdate'] = str( temp[1] + mysep + temp[2] )
 1648        metasub.loc[k,'imageID'] = str( temp[4])
 1649
 1650
 1651    if 'ol_' in outlier_column:
 1652        metasub['negol'] = metasub[outlier_column].max() - metasub[outlier_column]
 1653    else:
 1654        metasub['negol'] = metasub[outlier_column]
 1655    if 'date' not in metasub.keys():
 1656        metasub['date']=None
 1657    metasubq = add_repeat_column( metasub, 'subjectIDdate' )
 1658    metasubq = highest_quality_repeat(metasubq, 'filename', 'date', 'negol')
 1659
 1660    if verbose:
 1661        print(f"{wmod} {metasubq.shape[0]} post")
 1662
 1663#    metasub = metasub.astype(str)
 1664#    metasubq = metasubq.astype(str)
 1665    metasub[outlier_column]=metasub[outlier_column].astype(float)
 1666    metasubq[outlier_column]=metasubq[outlier_column].astype(float)
 1667    return {'raw': metasub, 'filt': metasubq}
 1668
 1669def mm_read( x, standardize_intensity=False, modality='' ):
 1670    """
 1671    read an image from a filename - same as ants.image_read (for now)
 1672
 1673    standardize_intensity : boolean ; if True will set negative values to zero and normalize into the range of zero to one
 1674
 1675    modality : not used
 1676    """
 1677    if x is None:
 1678        raise ValueError( " None passed to function antspymm.mm_read." )
 1679    if not isinstance(x,str):
 1680        raise ValueError( " Non-string passed to function antspymm.mm_read." )
 1681    if not os.path.exists( x ):
 1682        raise ValueError( " file " + fni + " does not exist." )
 1683    img = ants.image_read( x, reorient=False )
 1684    if standardize_intensity:
 1685        img[img<0.0]=0.0
 1686        img=ants.iMath(img,'Normalize')
 1687    if modality == "T1w" and img.dimension == 4:
 1688        print("WARNING: input image is 4D - we attempt a hack fix that works in some odd cases of PPMI data - please check this image: " + x, flush=True )
 1689        i1=ants.slice_image(img,3,0)
 1690        i2=ants.slice_image(img,3,1)
 1691        kk=np.concatenate( [i1.numpy(),i2.numpy()], axis=2 )
 1692        kk=ants.from_numpy(kk)
 1693        img=ants.copy_image_info(i1,kk)
 1694    return img
 1695
 1696def mm_read_to_3d( x, slice=None, modality='' ):
 1697    """
 1698    read an image from a filename - and return as 3d or None if that is not possible
 1699    """
 1700    img = ants.image_read( x, reorient=False )
 1701    if img.dimension <= 3:
 1702        return img
 1703    elif img.dimension == 4:
 1704        nslices = img.shape[3]
 1705        if slice is None:
 1706            sl = np.round( nslices * 0.5 )
 1707        else:
 1708            sl = slice
 1709        if sl > nslices:
 1710            sl = nslices-1
 1711        return ants.slice_image( img, axis=3, idx=int(sl) )
 1712    elif img.dimension > 4:
 1713        return img
 1714    return None
 1715
 1716def timeseries_n3(x):
 1717    """
 1718    Perform N3 bias field correction on a time-series image dataset using ANTsPy library.
 1719
 1720    This function processes a multi-dimensional image dataset, where the last dimension
 1721    represents different time points. It applies N3 bias field correction to each time point 
 1722    individually to correct intensity non-uniformity.
 1723
 1724    Parameters:
 1725    x (ndarray): A multi-dimensional array where the last dimension represents time points. 
 1726                 Each 'slice' along this dimension is a separate image to be corrected.
 1727
 1728    Returns:
 1729    ndarray: A multi-dimensional array of the same shape as x, with N3 bias field correction 
 1730             applied to each time slice.
 1731
 1732    The function works as follows:
 1733    - Initializes an empty list `mimg` to store the corrected images.
 1734    - Determines the number of time points in the input image series.
 1735    - Iterates over each time point, extracting the image slice and applying N3 bias 
 1736      field correction.
 1737    - The corrected images are then appended to the `mimg` list.
 1738    - Finally, the list of corrected images is converted back into a multi-dimensional 
 1739      array and returned.
 1740
 1741    Example:
 1742    corrected_images = timeseries_n3(image_data)
 1743    """
 1744    mimg = []
 1745    n = len(x.shape) - 1
 1746    for kk in range(x.shape[n]):
 1747        temp = ants.slice_image(x, axis=n, idx=kk)
 1748        temp = ants.n3_bias_field_correction(temp, downsample_factor=2)
 1749        mimg.append(temp)
 1750    return ants.list_to_ndimage(x, mimg)
 1751
 1752def image_write_with_thumbnail( x,  fn, y=None, thumb=True ):
 1753    """
 1754    will write the image and (optionally) a png thumbnail with (optional) overlay/underlay
 1755    """
 1756    ants.image_write( x, fn )
 1757    if not thumb or x.components > 1:
 1758        return
 1759    thumb_fn=re.sub(".nii.gz","_3dthumb.png",fn)
 1760    if thumb and x.dimension == 3:
 1761        if y is None:
 1762            try:
 1763                ants.plot_ortho( x, crop=True, filename=thumb_fn, flat=True, xyz_lines=False, orient_labels=False, xyz_pad=0 )
 1764            except:
 1765                pass
 1766        else:
 1767            try:
 1768                ants.plot_ortho( y, x, crop=True, filename=thumb_fn, flat=True, xyz_lines=False, orient_labels=False, xyz_pad=0 )
 1769            except:
 1770                pass
 1771    if thumb and x.dimension == 4:
 1772        thumb_fn=re.sub(".nii.gz","_4dthumb.png",fn)
 1773        nslices = x.shape[3]
 1774        sl = np.round( nslices * 0.5 )
 1775        if sl > nslices:
 1776            sl = nslices-1
 1777        xview = ants.slice_image( x, axis=3, idx=int(sl) )
 1778        if y is None:
 1779            try:
 1780                ants.plot_ortho( xview, crop=True, filename=thumb_fn, flat=True, xyz_lines=False, orient_labels=False, xyz_pad=0 )
 1781            except:
 1782                pass
 1783        else:
 1784            if y.dimension == 3:
 1785                try:
 1786                    ants.plot_ortho(y, xview, crop=True, filename=thumb_fn, flat=True, xyz_lines=False, orient_labels=False, xyz_pad=0 )
 1787                except:
 1788                    pass
 1789    return
 1790
 1791def convert_np_in_dict(data_dict):
 1792    """
 1793    Convert values in the dictionary from nupmy float or int to regular float or int.
 1794
 1795    :param data_dict: A dictionary with values of various types.
 1796    :return: Dictionary with numpy values converted.
 1797    """
 1798    converted_dict = {}
 1799    for key, value in data_dict.items():
 1800        if isinstance(value, (np.float32, np.float64)):
 1801            converted_dict[key] = float(value)
 1802        elif isinstance(value, (np.int8,  np.uint8, np.int16,  np.uint16, np.int32,  np.uint32, np.int64,  np.uint64)):
 1803            converted_dict[key] = int(value)
 1804        else:
 1805            converted_dict[key] = value
 1806    return converted_dict
 1807
 1808def mc_resample_image_to_target( x , y, interp_type='linear' ):
 1809    """
 1810    multichannel version of resample_image_to_target
 1811    """
 1812    xx=ants.split_channels( x )
 1813    yy=ants.split_channels( y )[0]
 1814    newl=[]
 1815    for k in range(len(xx)):
 1816        newl.append(  ants.resample_image_to_target( xx[k], yy, interp_type=interp_type ) )
 1817    return ants.merge_channels( newl )
 1818
 1819def nrg_filelist_to_dataframe( filename_list, myseparator="-" ):
 1820    """
 1821    convert a list of files in nrg format to a dataframe
 1822
 1823    Arguments
 1824    ---------
 1825    filename_list : globbed list of files
 1826
 1827    myseparator : string separator between nrg parts
 1828
 1829    Returns
 1830    -------
 1831
 1832    df : pandas data frame
 1833
 1834    """
 1835    def getmtime(x):
 1836        x= dt.datetime.fromtimestamp(os.path.getmtime(x)).strftime("%Y-%m-%d %H:%M:%d")
 1837        return x
 1838    df=pd.DataFrame(columns=['filename','file_last_mod_t','else','sid','visitdate','modality','uid'])
 1839    df.set_index('filename')
 1840    df['filename'] = pd.Series([file for file in filename_list ])
 1841    # I applied a time modified file to df['file_last_mod_t'] by getmtime function
 1842    df['file_last_mod_t'] = df['filename'].apply(lambda x: getmtime(x))
 1843    for k in range(df.shape[0]):
 1844        locfn=df['filename'].iloc[k]
 1845        splitter=os.path.basename(locfn).split( myseparator )
 1846        df['sid'].iloc[k]=splitter[1]
 1847        df['visitdate'].iloc[k]=splitter[2]
 1848        df['modality'].iloc[k]=splitter[3]
 1849        temp = os.path.splitext(splitter[4])[0]
 1850        df['uid'].iloc[k]=os.path.splitext(temp)[0]
 1851    return df
 1852
 1853
 1854def merge_timeseries_data( img_LR, img_RL, allow_resample=True ):
 1855    """
 1856    merge time series data into space of reference_image
 1857
 1858    img_LR : image
 1859
 1860    img_RL : image
 1861
 1862    allow_resample : boolean
 1863
 1864    """
 1865    # concatenate the images into the reference space
 1866    mimg=[]
 1867    for kk in range( img_LR.shape[3] ):
 1868        temp = ants.slice_image( img_LR, axis=3, idx=kk )
 1869        mimg.append( temp )
 1870    for kk in range( img_RL.shape[3] ):
 1871        temp = ants.slice_image( img_RL, axis=3, idx=kk )
 1872        if kk == 0:
 1873            insamespace = ants.image_physical_space_consistency( temp, mimg[0] )
 1874        if allow_resample and not insamespace :
 1875            temp = ants.resample_image_to_target( temp, mimg[0] )
 1876        mimg.append( temp )
 1877    return ants.list_to_ndimage( img_LR, mimg )
 1878
 1879def copy_spatial_metadata_from_3d_to_4d(spatial_img, timeseries_img):
 1880    """
 1881    Copy spatial metadata (origin, spacing, direction) from a 3D image to the
 1882    spatial dimensions (first 3) of a 4D image, preserving the 4th dimension's metadata.
 1883
 1884    Parameters
 1885    ----------
 1886    spatial_img : ants.ANTsImage
 1887        A 3D ANTsImage with the desired spatial metadata.
 1888    timeseries_img : ants.ANTsImage
 1889        A 4D ANTsImage to update.
 1890
 1891    Returns
 1892    -------
 1893    ants.ANTsImage
 1894        A 4D ANTsImage with updated spatial metadata.
 1895    """
 1896    if spatial_img.dimension != 3:
 1897        raise ValueError("spatial_img must be a 3D ANTsImage.")
 1898    if timeseries_img.dimension != 4:
 1899        raise ValueError("timeseries_img must be a 4D ANTsImage.")
 1900    # Get 3D metadata
 1901    spatial_origin = list(spatial_img.origin)
 1902    spatial_spacing = list(spatial_img.spacing)
 1903    spatial_direction = spatial_img.direction  # 3x3
 1904    # Get original 4D metadata
 1905    ts_spacing = list(timeseries_img.spacing)
 1906    ts_origin = list(timeseries_img.origin)
 1907    ts_direction = timeseries_img.direction  # 4x4
 1908    # Replace only the first 3 entries for origin and spacing
 1909    new_origin = spatial_origin + [ts_origin[3]]
 1910    new_spacing = spatial_spacing + [ts_spacing[3]]
 1911    # Replace top-left 3x3 block of direction matrix, preserve last row/column
 1912    new_direction = ts_direction.copy()
 1913    new_direction[:3, :3] = spatial_direction
 1914    # Create updated image
 1915    updated_img = ants.from_numpy(
 1916        timeseries_img.numpy(),
 1917        origin=new_origin,
 1918        spacing=new_spacing,
 1919        direction=new_direction
 1920    )
 1921    return updated_img
 1922
 1923def timeseries_transform(transform, image, reference, interpolation='linear'):
 1924    """
 1925    Apply a spatial transform to each 3D volume in a 4D time series image.
 1926
 1927    Parameters
 1928    ----------
 1929    transform : ants transform object
 1930        Path(s) to ANTs-compatible transform(s) to apply.
 1931    image : ants.ANTsImage
 1932        4D input image with shape (X, Y, Z, T).
 1933    reference : ants.ANTsImage
 1934        Reference image to match in space.
 1935    interpolation : str
 1936        Interpolation method: 'linear', 'nearestNeighbor', etc.
 1937
 1938    Returns
 1939    -------
 1940    ants.ANTsImage
 1941        4D transformed image.
 1942    """
 1943    if image.dimension != 4:
 1944        raise ValueError("Input image must be 4D (X, Y, Z, T).")
 1945    n_volumes = image.shape[3]
 1946    transformed_volumes = []
 1947    for t in range(n_volumes):
 1948        vol = ants.slice_image( image, 3, t )
 1949        transformed = ants.apply_ants_transform_to_image(
 1950            transform=transform,
 1951            image=vol,
 1952            reference=reference,
 1953            interpolation=interpolation
 1954        )
 1955        transformed_volumes.append(transformed.numpy())
 1956    # Stack along time axis and convert to ANTsImage
 1957    transformed_array = np.stack(transformed_volumes, axis=-1)
 1958    out_image = ants.from_numpy(transformed_array)
 1959    out_image = ants.copy_image_info(image, out_image)
 1960    out_image = copy_spatial_metadata_from_3d_to_4d(reference, out_image)
 1961    return out_image
 1962
 1963def timeseries_reg(
 1964    image,
 1965    avg_b0,
 1966    type_of_transform='antsRegistrationSyNRepro[r]',
 1967    total_sigma=1.0,
 1968    fdOffset=2.0,
 1969    trim = 0,
 1970    output_directory=None,
 1971    return_numpy_motion_parameters=False,
 1972    verbose=False, **kwargs
 1973):
 1974    """
 1975    Correct time-series data for motion.
 1976
 1977    Arguments
 1978    ---------
 1979    image: antsImage, usually ND where D=4.
 1980
 1981    avg_b0: Fixed image b0 image
 1982
 1983    type_of_transform : string
 1984            A linear or non-linear registration type. Mutual information metric and rigid transformation by default.
 1985            See ants registration for details.
 1986
 1987    fdOffset: offset value to use in framewise displacement calculation
 1988
 1989    trim : integer - trim this many images off the front of the time series
 1990
 1991    output_directory : string
 1992            output will be placed in this directory plus a numeric extension.
 1993
 1994    return_numpy_motion_parameters : boolean
 1995
 1996    verbose: boolean
 1997
 1998    kwargs: keyword args
 1999            extra arguments - these extra arguments will control the details of registration that is performed. see ants registration for more.
 2000
 2001    Returns
 2002    -------
 2003    dict containing follow key/value pairs:
 2004        `motion_corrected`: Moving image warped to space of fixed image.
 2005        `motion_parameters`: transforms for each image in the time series.
 2006        `FD`: Framewise displacement generalized for arbitrary transformations.
 2007
 2008    Notes
 2009    -----
 2010    Control extra arguments via kwargs. see ants.registration for details.
 2011
 2012    Example
 2013    -------
 2014    >>> import ants
 2015    """
 2016    idim = image.dimension
 2017    ishape = image.shape
 2018    nTimePoints = ishape[idim - 1]
 2019    FD = np.zeros(nTimePoints)
 2020    if type_of_transform is None:
 2021        return {
 2022            "motion_corrected": image,
 2023            "motion_parameters": None,
 2024            "FD": FD
 2025        }
 2026
 2027    remove_it=False
 2028    if output_directory is None:
 2029        remove_it=True
 2030        output_directory = tempfile.mkdtemp()
 2031    output_directory_w = output_directory + "/ts_reg/"
 2032    os.makedirs(output_directory_w,exist_ok=True)
 2033    ofnG = tempfile.NamedTemporaryFile(delete=False,suffix='global_deformation',dir=output_directory_w).name
 2034    ofnL = tempfile.NamedTemporaryFile(delete=False,suffix='local_deformation',dir=output_directory_w).name
 2035    if verbose:
 2036        print('bold motcorr with ' + type_of_transform)
 2037        print(output_directory_w)
 2038        print(ofnG)
 2039        print(ofnL)
 2040        print("remove_it " + str( remove_it ) )
 2041
 2042    # get a local deformation from slice to local avg space
 2043    motion_parameters = list()
 2044    motion_corrected = list()
 2045    mask = ants.get_mask( avg_b0 )
 2046    centerOfMass = mask.get_center_of_mass()
 2047    npts = pow(2, idim - 1)
 2048    pointOffsets = np.zeros((npts, idim - 1))
 2049    myrad = np.ones(idim - 1).astype(int).tolist()
 2050    mask1vals = np.zeros(int(mask.sum()))
 2051    mask1vals[round(len(mask1vals) / 2)] = 1
 2052    mask1 = ants.make_image(mask, mask1vals)
 2053    myoffsets = ants.get_neighborhood_in_mask(
 2054        mask1, mask1, radius=myrad, spatial_info=True
 2055    )["offsets"]
 2056    mycols = list("xy")
 2057    if idim - 1 == 3:
 2058        mycols = list("xyz")
 2059    useinds = list()
 2060    for k in range(myoffsets.shape[0]):
 2061        if abs(myoffsets[k, :]).sum() == (idim - 2):
 2062            useinds.append(k)
 2063        myoffsets[k, :] = myoffsets[k, :] * fdOffset / 2.0 + centerOfMass
 2064    fdpts = pd.DataFrame(data=myoffsets[useinds, :], columns=mycols)
 2065    if verbose:
 2066        print("Progress:")
 2067    counter = round( nTimePoints / 10 ) + 1
 2068    for k in range( nTimePoints):
 2069        if verbose and ( ( k % counter ) ==  0 ) or ( k == (nTimePoints-1) ):
 2070            myperc = round( k / nTimePoints * 100)
 2071            print(myperc, end="%.", flush=True)
 2072        temp = ants.slice_image(image, axis=idim - 1, idx=k)
 2073        temp = ants.iMath(temp, "Normalize")
 2074        txprefix = ofnL+str(k % 2).zfill(4)+"_"
 2075        if temp.numpy().var() > 0:
 2076            myrig = ants.registration(
 2077                    avg_b0, temp,
 2078                    type_of_transform='antsRegistrationSyNRepro[r]',
 2079                    outprefix=txprefix
 2080                )
 2081            if type_of_transform == 'SyN':
 2082                myreg = ants.registration(
 2083                    avg_b0, temp,
 2084                    type_of_transform='SyNOnly',
 2085                    total_sigma=total_sigma,
 2086                    initial_transform=myrig['fwdtransforms'][0],
 2087                    outprefix=txprefix,
 2088                    **kwargs
 2089                )
 2090            else:
 2091                myreg = myrig
 2092            fdptsTxI = ants.apply_transforms_to_points(
 2093                idim - 1, fdpts, myrig["fwdtransforms"]
 2094            )
 2095            if k > 0 and motion_parameters[k - 1] != "NA":
 2096                fdptsTxIminus1 = ants.apply_transforms_to_points(
 2097                    idim - 1, fdpts, motion_parameters[k - 1]
 2098                )
 2099            else:
 2100                fdptsTxIminus1 = fdptsTxI
 2101            # take the absolute value, then the mean across columns, then the sum
 2102            FD[k] = (fdptsTxIminus1 - fdptsTxI).abs().mean().sum()
 2103            motion_parameters.append(myreg["fwdtransforms"])
 2104        else:
 2105            motion_parameters.append("NA")
 2106
 2107        temp = ants.slice_image(image, axis=idim - 1, idx=k)
 2108        if temp.numpy().var() > 0:
 2109            img1w = ants.apply_transforms( avg_b0,
 2110                temp,
 2111                motion_parameters[k] )
 2112            motion_corrected.append(img1w)
 2113        else:
 2114            motion_corrected.append(avg_b0)
 2115
 2116    motion_parameters = motion_parameters[trim:len(motion_parameters)]
 2117    if return_numpy_motion_parameters:
 2118        motion_parameters = read_ants_transforms_to_numpy( motion_parameters )
 2119
 2120    if remove_it:
 2121        import shutil
 2122        shutil.rmtree(output_directory, ignore_errors=True )
 2123
 2124    if verbose:
 2125        print("Done")
 2126    d4siz = list(avg_b0.shape)
 2127    d4siz.append( 2 )
 2128    spc = list(ants.get_spacing( avg_b0 ))
 2129    spc.append( ants.get_spacing(image)[3] )
 2130    mydir = ants.get_direction( avg_b0 )
 2131    mydir4d = ants.get_direction( image )
 2132    mydir4d[0:3,0:3]=mydir
 2133    myorg = list(ants.get_origin( avg_b0 ))
 2134    myorg.append( 0.0 )
 2135    avg_b0_4d = ants.make_image(d4siz,0,spacing=spc,origin=myorg,direction=mydir4d)
 2136    return {
 2137        "motion_corrected": ants.list_to_ndimage(avg_b0_4d, motion_corrected[trim:len(motion_corrected)]),
 2138        "motion_parameters": motion_parameters,
 2139        "FD": FD[trim:len(FD)]
 2140    }
 2141
 2142
 2143def merge_dwi_data( img_LRdwp, bval_LR, bvec_LR, img_RLdwp, bval_RL, bvec_RL ):
 2144    """
 2145    merge motion and distortion corrected data if possible
 2146
 2147    img_LRdwp : image
 2148
 2149    bval_LR : array
 2150
 2151    bvec_LR : array
 2152
 2153    img_RLdwp : image
 2154
 2155    bval_RL : array
 2156
 2157    bvec_RL : array
 2158
 2159    """
 2160    import warnings
 2161    insamespace = ants.image_physical_space_consistency( img_LRdwp, img_RLdwp )
 2162    if not insamespace :
 2163        warnings.warn('not insamespace ... corrected image pair should occupy the same physical space; returning only the 1st set and wont join these data.')
 2164        return img_LRdwp, bval_LR, bvec_LR
 2165    
 2166    bval_LR = np.concatenate([bval_LR,bval_RL])
 2167    bvec_LR = np.concatenate([bvec_LR,bvec_RL])
 2168    # concatenate the images
 2169    mimg=[]
 2170    for kk in range( img_LRdwp.shape[3] ):
 2171            mimg.append( ants.slice_image( img_LRdwp, axis=3, idx=kk ) )
 2172    for kk in range( img_RLdwp.shape[3] ):
 2173            mimg.append( ants.slice_image( img_RLdwp, axis=3, idx=kk ) )
 2174    img_LRdwp = ants.list_to_ndimage( img_LRdwp, mimg )
 2175    return img_LRdwp, bval_LR, bvec_LR
 2176
 2177def bvec_reorientation( motion_parameters, bvecs, rebase=None ):
 2178    if motion_parameters is None:
 2179        return bvecs
 2180    n = len(motion_parameters)
 2181    if n < 1:
 2182        return bvecs
 2183    from scipy.linalg import inv, polar
 2184    from dipy.core.gradients import reorient_bvecs
 2185    dipymoco = np.zeros( [n,3,3] )
 2186    for myidx in range(n):
 2187        if myidx < bvecs.shape[0]:
 2188            dipymoco[myidx,:,:] = np.eye( 3 )
 2189            if motion_parameters[myidx] != 'NA':
 2190                temp = motion_parameters[myidx]
 2191                if len(temp) == 4 :
 2192                    temp1=temp[3] # FIXME should be composite of index 1 and 3
 2193                    temp2=temp[1] # FIXME should be composite of index 1 and 3
 2194                    txparam1 = ants.read_transform(temp1)
 2195                    txparam1 = ants.get_ants_transform_parameters(txparam1)[0:9].reshape( [3,3])
 2196                    txparam2 = ants.read_transform(temp2)
 2197                    txparam2 = ants.get_ants_transform_parameters(txparam2)[0:9].reshape( [3,3])
 2198                    Rinv = inv( np.dot( txparam2, txparam1 ) )
 2199                elif len(temp) == 2 :
 2200                    temp=temp[1] # FIXME should be composite of index 1 and 3
 2201                    txparam = ants.read_transform(temp)
 2202                    txparam = ants.get_ants_transform_parameters(txparam)[0:9].reshape( [3,3])
 2203                    Rinv = inv( txparam )
 2204                elif len(temp) == 3 :
 2205                    temp1=temp[2] # FIXME should be composite of index 1 and 3
 2206                    temp2=temp[1] # FIXME should be composite of index 1 and 3
 2207                    txparam1 = ants.read_transform(temp1)
 2208                    txparam1 = ants.get_ants_transform_parameters(txparam1)[0:9].reshape( [3,3])
 2209                    txparam2 = ants.read_transform(temp2)
 2210                    txparam2 = ants.get_ants_transform_parameters(txparam2)[0:9].reshape( [3,3])
 2211                    Rinv = inv( np.dot( txparam2, txparam1 ) )
 2212                else:
 2213                    temp=temp[0]
 2214                    txparam = ants.read_transform(temp)
 2215                    txparam = ants.get_ants_transform_parameters(txparam)[0:9].reshape( [3,3])
 2216                    Rinv = inv( txparam )
 2217                bvecs[myidx,:] = np.dot( Rinv, bvecs[myidx,:] )
 2218                if rebase is not None:
 2219                    # FIXME - should combine these operations
 2220                    bvecs[myidx,:] = np.dot( rebase, bvecs[myidx,:] )
 2221    return bvecs
 2222
 2223
 2224def distortion_correct_bvecs(bvecs, def_grad, A_img, A_ref):
 2225    """
 2226    Vectorized computation of voxel-wise distortion corrected b-vectors.
 2227
 2228    Parameters
 2229    ----------
 2230    bvecs : ndarray (N, 3)
 2231    def_grad : ndarray (X, Y, Z, 3, 3) containing rotations derived from the deformation gradient
 2232    A_img : ndarray (3, 3) direction matrix of the fixed image (target undistorted space)
 2233    A_ref : ndarray (3, 3) direction matrix of the moving image (being corrected)
 2234
 2235    Returns
 2236    -------
 2237    bvecs_5d : ndarray (X, Y, Z, N, 3)
 2238    """
 2239    X, Y, Z = def_grad.shape[:3]
 2240    N = bvecs.shape[0]
 2241    # Combined rotation: R_voxel = A_ref.T @ A_img @ def_grad
 2242    A = A_ref.T @ A_img
 2243    R_voxel = np.einsum('ij,xyzjk->xyzik', A, def_grad)  # (X, Y, Z, 3, 3)
 2244    # Apply R_voxel.T @ bvecs
 2245    # First, reshape R_voxel: (X*Y*Z, 3, 3)
 2246    R_voxel_reshaped = R_voxel.reshape(-1, 3, 3)
 2247    # Rotate all bvecs for each voxel
 2248    # Output: (X*Y*Z, N, 3)
 2249    rotated = np.einsum('vij,nj->vni', R_voxel_reshaped, bvecs)
 2250    # Normalize
 2251    norms = np.linalg.norm(rotated, axis=2, keepdims=True)
 2252    rotated /= np.clip(norms, 1e-8, None)
 2253    # Reshape back to (X, Y, Z, N, 3)
 2254    bvecs_5d = rotated.reshape(X, Y, Z, N, 3)
 2255    return bvecs_5d    
 2256
 2257def get_dti( reference_image, tensormodel, upper_triangular=True, return_image=False ):
 2258    """
 2259    extract DTI data from a dipy tensormodel
 2260
 2261    reference_image : antsImage defining physical space (3D)
 2262
 2263    tensormodel : from dipy e.g. the variable myoutx['dtrecon_LR_dewarp']['tensormodel'] if myoutx is produced my joint_dti_recon
 2264
 2265    upper_triangular: boolean otherwise use lower triangular coding
 2266
 2267    return_image : boolean return the ANTsImage form of DTI otherwise return an array
 2268
 2269    Returns
 2270    -------
 2271    either an ANTsImage (dim=X.Y.Z with 6 component voxels, upper triangular form)
 2272        or a 5D NumPy array (dim=X.Y.Z.3.3)
 2273
 2274    Notes
 2275    -----
 2276    DiPy returns lower triangular form but ANTs expects upper triangular.
 2277        Here, we default to the ANTs standard but could generalize in the future 
 2278        because not much here depends on ANTs standards of tensor data.
 2279        ANTs xx,xy,xz,yy,yz,zz
 2280        DiPy Dxx, Dxy, Dyy, Dxz, Dyz, Dzz
 2281
 2282    """
 2283    # make the DTI - see 
 2284    # https://dipy.org/documentation/1.7.0/examples_built/07_reconstruction/reconst_dti/#sphx-glr-examples-built-07-reconstruction-reconst-dti-py
 2285    # By default, in DIPY, values are ordered as (Dxx, Dxy, Dyy, Dxz, Dyz, Dzz)
 2286    # in ANTs - we have: [xx,xy,xz,yy,yz,zz]
 2287    reoind = np.array([0,1,3,2,4,5]) # arrays are faster than lists
 2288    import dipy.reconst.dti as dti
 2289    dtiut = dti.lower_triangular(tensormodel.quadratic_form)
 2290    it = np.ndindex( reference_image.shape )
 2291    yyind=2
 2292    xzind=3
 2293    if upper_triangular:
 2294        yyind=3
 2295        xzind=2
 2296        for i in it: # convert to upper triangular
 2297            dtiut[i] = dtiut[i][ reoind ] # do we care if this is doing extra work?
 2298    if return_image:
 2299        dtiAnts = ants.from_numpy(dtiut,has_components=True)
 2300        ants.copy_image_info( reference_image, dtiAnts )
 2301        return dtiAnts
 2302    # copy these data into a tensor 
 2303    dtinp = np.zeros(reference_image.shape + (3,3), dtype=float)  
 2304    dtix = np.zeros((3,3), dtype=float)  
 2305    it = np.ndindex( reference_image.shape )
 2306    for i in it:
 2307        dtivec = dtiut[i] # in ANTs - we have: [xx,xy,xz,yy,yz,zz]
 2308        dtix[0,0]=dtivec[0]
 2309        dtix[1,1]=dtivec[yyind] # 2 for LT
 2310        dtix[2,2]=dtivec[5] 
 2311        dtix[0,1]=dtix[1,0]=dtivec[1]
 2312        dtix[0,2]=dtix[2,0]=dtivec[xzind] # 3 for LT
 2313        dtix[1,2]=dtix[2,1]=dtivec[4]
 2314        dtinp[i]=dtix
 2315    return dtinp
 2316
 2317def triangular_to_tensor( image, upper_triangular=True ):
 2318    """
 2319    convert triangular tensor image to a full tensor form (in numpy)
 2320
 2321    image : antsImage holding dti in either upper or lower triangular format 
 2322
 2323    upper_triangular: boolean
 2324
 2325    Note
 2326    --------
 2327    see get_dti function for more details
 2328    """
 2329    reoind = np.array([0,1,3,2,4,5]) # arrays are faster than lists
 2330    it = np.ndindex( image.shape )
 2331    yyind=2
 2332    xzind=3
 2333    if upper_triangular:
 2334        yyind=3
 2335        xzind=2
 2336    # copy these data into a tensor 
 2337    dtinp = np.zeros(image.shape + (3,3), dtype=float)
 2338    dtix = np.zeros((3,3), dtype=float)
 2339    it = np.ndindex( image.shape )
 2340    dtiut = image.numpy()
 2341    for i in it:
 2342        dtivec = dtiut[i] # in ANTs - we have: [xx,xy,xz,yy,yz,zz]
 2343        dtix[0,0]=dtivec[0]
 2344        dtix[1,1]=dtivec[yyind] # 2 for LT
 2345        dtix[2,2]=dtivec[5] 
 2346        dtix[0,1]=dtix[1,0]=dtivec[1]
 2347        dtix[0,2]=dtix[2,0]=dtivec[xzind] # 3 for LT
 2348        dtix[1,2]=dtix[2,1]=dtivec[4]
 2349        dtinp[i]=dtix
 2350    return dtinp
 2351
 2352
 2353def dti_numpy_to_image( reference_image, tensorarray, upper_triangular=True):
 2354    """
 2355    convert numpy DTI data to antsImage
 2356
 2357    reference_image : antsImage defining physical space (3D)
 2358
 2359    tensorarray : numpy array X,Y,Z,3,3 shape
 2360
 2361    upper_triangular: boolean otherwise use lower triangular coding
 2362
 2363    Returns
 2364    -------
 2365    ANTsImage
 2366
 2367    Notes
 2368    -----
 2369    DiPy returns lower triangular form but ANTs expects upper triangular.
 2370        Here, we default to the ANTs standard but could generalize in the future 
 2371        because not much here depends on ANTs standards of tensor data.
 2372        ANTs xx,xy,xz,yy,yz,zz
 2373        DiPy Dxx, Dxy, Dyy, Dxz, Dyz, Dzz
 2374
 2375    """
 2376    dtiut = np.zeros(reference_image.shape + (6,), dtype=float)  
 2377    dtivec = np.zeros(6, dtype=float)  
 2378    it = np.ndindex( reference_image.shape )
 2379    yyind=2
 2380    xzind=3
 2381    if upper_triangular:
 2382        yyind=3
 2383        xzind=2
 2384    for i in it:
 2385        dtix = tensorarray[i] # in ANTs - we have: [xx,xy,xz,yy,yz,zz]
 2386        dtivec[0]=dtix[0,0]
 2387        dtivec[yyind]=dtix[1,1] # 2 for LT
 2388        dtivec[5]=dtix[2,2]
 2389        dtivec[1]=dtix[0,1]
 2390        dtivec[xzind]=dtix[2,0] # 3 for LT
 2391        dtivec[4]=dtix[1,2]
 2392        dtiut[i]=dtivec
 2393    dtiAnts = ants.from_numpy( dtiut, has_components=True )
 2394    ants.copy_image_info( reference_image, dtiAnts )
 2395    return dtiAnts
 2396
 2397
 2398def deformation_gradient_optimized(warp_image, to_rotation=False, to_inverse_rotation=False):
 2399    """
 2400    Compute the deformation gradient tensor from a displacement (warp) field image.
 2401
 2402    This function computes the **deformation gradient** `F = ∂φ/∂x` where `φ(x) = x + u(x)` is the mapping
 2403    induced by the displacement field `u(x)` stored in `warp_image`.
 2404
 2405    The returned tensor field has shape `(x, y, z, dim, dim)` (for 3D), where each matrix represents 
 2406    the **Jacobian** of the transformation at that voxel. The gradient is computed in the physical space 
 2407    of the image using spacing and direction metadata.
 2408
 2409    Optionally, the deformation gradient can be projected onto the space of pure rotations using the polar
 2410    decomposition (via SVD). This is useful for applications like reorientation of tensors (e.g., DTI).
 2411
 2412    Parameters
 2413    ----------
 2414    warp_image : ants.ANTsImage
 2415        A vector-valued ANTsImage encoding the warp/displacement field. It must have `dim` components
 2416        (e.g., shape `(x, y, z, 3)` for 3D) representing the displacements in each spatial direction.
 2417        
 2418    to_rotation : bool, optional
 2419        If True, the deformation gradient will be replaced with its **nearest rotation matrix**
 2420        using the polar decomposition (`F → R`, where `F = R U`).
 2421        
 2422    to_inverse_rotation : bool, optional
 2423        If True, the deformation gradient will be replaced with the **inverse of the rotation**
 2424        (`F → R.T`), which is often needed for transforming tensors **back** to their original frame.
 2425
 2426    Returns
 2427    -------
 2428    F : np.ndarray
 2429        A NumPy array of shape `(x, y, z, dim, dim)` (or `(dim1, dim2, ..., dim, dim)` in general),
 2430        representing the deformation gradient tensor field at each voxel.
 2431
 2432    Raises
 2433    ------
 2434    RuntimeError
 2435        If `warp_image` is not an `ants.ANTsImage`.
 2436
 2437    Notes
 2438    -----
 2439    - The function computes gradients in physical space using the spacing of the image and applies 
 2440      the image direction matrix (`tdir`) to properly orient gradients.
 2441    - The gradient is added to the identity matrix to yield the deformation gradient `F = I + ∂u/∂x`.
 2442    - The polar decomposition ensures `F` is replaced with the closest rotation matrix (orthogonal, det=1).
 2443    - This is a **vectorized pure NumPy implementation**, intended for performance and simplicity.
 2444
 2445    Examples
 2446    --------
 2447    >>> warp = ants.create_warp_image(reference_image, displacement_field)
 2448    >>> F = deformation_gradient_optimized(warp)
 2449    >>> R = deformation_gradient_optimized(warp, to_rotation=True)
 2450    >>> Rinv = deformation_gradient_optimized(warp, to_inverse_rotation=True)
 2451    """
 2452    if not ants.is_image(warp_image):
 2453        raise RuntimeError("antsimage is required")
 2454    dim = warp_image.dimension
 2455    tshp = warp_image.shape
 2456    tdir = warp_image.direction
 2457    spc = warp_image.spacing
 2458    warpnp = warp_image.numpy()
 2459    gradient_list = [np.gradient(warpnp[..., k], *spc, axis=range(dim)) for k in range(dim)]
 2460    # This correctly calculates J.T, where dg[..., i, j] = d(u_j)/d(x_i)
 2461    dg = np.stack([np.stack(grad_k, axis=-1) for grad_k in gradient_list], axis=-1)
 2462    dg = (tdir @ dg).swapaxes(-1, -2)
 2463    dg += np.eye(dim)
 2464    if to_rotation or to_inverse_rotation:
 2465        U, s, Vh = np.linalg.svd(dg)
 2466        Z = U @ Vh
 2467        dets = np.linalg.det(Z)
 2468        reflection_mask = dets < 0
 2469        Vh[reflection_mask, -1, :] *= -1
 2470        Z[reflection_mask] = U[reflection_mask] @ Vh[reflection_mask]
 2471        dg = Z
 2472        if to_inverse_rotation:
 2473            dg = np.transpose(dg, axes=(*range(dg.ndim - 2), dg.ndim - 1, dg.ndim - 2))
 2474    new_shape = tshp + (dim,dim)
 2475    return np.reshape(dg, new_shape)
 2476
 2477
 2478def transform_and_reorient_dti( fixed, moving_dti, composite_transform, verbose=False, **kwargs):
 2479    """
 2480    Applies a transformation to a DTI image using an ANTs composite transform,
 2481    including local tensor reorientation via the Finite Strain method.
 2482
 2483    This function expects:
 2484    - Input `moving_dti` to be a 6-component ANTsImage (upper triangular format).
 2485    - `composite_transform` to point to an ANTs-readable transform file,
 2486      which maps points from `fixed` space to `moving` space.
 2487
 2488    Args:
 2489        fixed (ants.ANTsImage): The reference space image (defines the output grid).
 2490        moving_dti (ants.ANTsImage): The input DTI (6-component), to be transformed.
 2491        composite_transform (str): File path to an ANTs transform
 2492                                   (e.g., from `ants.read_transform` or a written composite transform).
 2493        verbose (bool): Whether to print verbose output during execution.
 2494        **kwargs: Additional keyword arguments passed to `ants.apply_transforms`.
 2495
 2496    Returns:
 2497        ants.ANTsImage: The transformed and reoriented DTI image in the `fixed` space,
 2498                        in 6-component upper triangular format.
 2499    """
 2500    if moving_dti.dimension != 3:
 2501        raise ValueError('moving_dti must be 3-dimensional.')
 2502    if moving_dti.components != 6:
 2503        raise ValueError('moving_dti must have 6 components (upper triangular format).')
 2504
 2505    if verbose:
 2506        print("1. Spatially transforming DTI scalar components from moving to fixed space...")
 2507
 2508    # ants.apply_transforms resamples the *values* of each DTI component from 'moving_dti'
 2509    # onto the grid of 'fixed'.
 2510    # The output 'dtiw' will have the same spatial metadata (spacing, origin, direction) as 'fixed'.
 2511    # However, the tensor values contained within it are still oriented as they were in
 2512    # 'moving_dti's original image space, not 'fixed' image space, and certainly not yet reoriented
 2513    # by the local deformation.
 2514    dtsplit = moving_dti.split_channels()
 2515    dtiw_channels = []
 2516    for k in range(len(dtsplit)):
 2517        dtiw_channels.append( ants.apply_transforms( fixed, dtsplit[k], composite_transform, **kwargs ) )
 2518    dtiw = ants.merge_channels(dtiw_channels)
 2519    
 2520    if verbose:
 2521        print(f"   DTI scalar components resampled to fixed grid. Result shape: {dtiw.shape}")
 2522        print("2. Computing local rotation field from composite transform...")
 2523    
 2524    # Read the composite transform as an image (assumed to be a displacement field).
 2525    # The 'deformation_gradient_optimized' function is assumed to be 100% correct,
 2526    # meaning it returns the appropriate local rotation matrix field (R_moving_to_fixed)
 2527    # in (spatial_dims..., 3, 3 ) format when called with these flags.
 2528    wtx = ants.image_read(composite_transform)
 2529    R_moving_to_fixed_forward = deformation_gradient_optimized(
 2530        wtx,
 2531        to_rotation=False,      # This means the *deformation gradient* F=I+J is computed first.
 2532        to_inverse_rotation=True # This requests the inverse of the rotation part of F.
 2533    )
 2534
 2535    if verbose:
 2536        print(f"   Local reorientation matrices (R_moving_to_fixed_forward) computed. Shape: {R_moving_to_fixed_forward.shape}")
 2537        print("3. Converting 6-component DTI to full 3x3 tensors for vectorized reorientation...")
 2538    
 2539    # Convert `dtiw` (resampled, but still in moving-image-space orientation)
 2540    # from 6-components to full 3x3 tensor representation.
 2541    # dtiw2tensor_np will have shape (spatial_dims..., 3, 3).
 2542    dtiw2tensor_np = triangular_to_tensor(dtiw)
 2543    
 2544    if verbose:
 2545        print("4. Applying vectorized tensor reorientation (Finite Strain Method)...")
 2546    
 2547    # --- Vectorized Tensor Reorientation ---
 2548    # This replaces the entire `for i in it:` loop and its contents with efficient NumPy operations.
 2549
 2550    # Step 4.1: Rebase tensors from `moving_dti.direction` coordinate system to World Coordinates.
 2551    # D_world_moving_orient = moving_dti.direction @ D_moving_image_frame @ moving_dti.direction.T
 2552    # This transforms the tensor's components from being relative to `moving_dti`'s image axes
 2553    # (where they are currently defined) into absolute World (physical) coordinates.
 2554    D_world_moving_orient = np.einsum(
 2555        'ab, ...bc, cd -> ...ad',
 2556        moving_dti.direction,            # 3x3 matrix (moving_image_axes -> world_axes)
 2557        dtiw2tensor_np,                  # (spatial_dims..., 3, 3)
 2558        moving_dti.direction.T           # 3x3 matrix (world_axes -> moving_image_axes) - inverse of moving_dti.direction
 2559    )
 2560
 2561    # Step 4.2: Apply local rotation in World Coordinates (Finite Strain Reorientation).
 2562    # D_reoriented_world = R_moving_to_fixed_forward @ D_world_moving_orient @ (R_moving_to_fixed_forward).T
 2563    # This is the core reorientation step, transforming the tensor's orientation from
 2564    # the original `moving` space to the new `fixed` space, all within world coordinates.
 2565    D_world_fixed_orient = np.einsum(
 2566        '...ab, ...bc, ...cd -> ...ad',
 2567        R_moving_to_fixed_forward,      # (spatial_dims..., 3, 3) - local rotation
 2568        D_world_moving_orient,          # (spatial_dims..., 3, 3) - tensor in world space, moving_orient
 2569        np.swapaxes(R_moving_to_fixed_forward, -1, -2) # (spatial_dims..., 3, 3) - transpose of local rotation
 2570    )
 2571
 2572    # Step 4.3: Rebase reoriented tensors from World Coordinates to `fixed.direction` coordinate system.
 2573    # D_final_fixed_image_frame = (fixed.direction).T @ D_world_fixed_orient @ fixed.direction
 2574    # This transforms the tensor's components from absolute World (physical) coordinates
 2575    # back into `fixed.direction`'s image coordinate system.
 2576    final_dti_tensors_numpy = np.einsum(
 2577        'ba, ...bc, cd -> ...ad',
 2578        fixed.direction,                # Using `fixed.direction` here, but 'ba' indices specify to use its transpose.
 2579        D_world_fixed_orient,           # (spatial_dims..., 3, 3)
 2580        fixed.direction                 # 3x3 matrix (world_axes -> fixed_image_axes)
 2581    )
 2582
 2583    if verbose:
 2584        print("   Vectorized tensor reorientation complete.")
 2585
 2586    if verbose:
 2587        print("5. Converting reoriented full tensors back to 6-component ANTsImage...")
 2588    
 2589    # Convert the final (spatial_dims..., 3, 3) NumPy array of tensors back into a
 2590    # 6-component ANTsImage with the correct spatial metadata from `fixed`.
 2591    final_dti_image = dti_numpy_to_image(fixed, final_dti_tensors_numpy)
 2592    
 2593    if verbose:
 2594        print(f"Done. Final reoriented DTI image in fixed space generated. Shape: {final_dti_image.shape}")
 2595
 2596    return final_dti_image
 2597
 2598def dti_reg(
 2599    image,
 2600    avg_b0,
 2601    avg_dwi,
 2602    bvals=None,
 2603    bvecs=None,
 2604    b0_idx=None,
 2605    type_of_transform="antsRegistrationSyNRepro[r]",
 2606    total_sigma=3.0,
 2607    fdOffset=2.0,
 2608    mask_csf=False,
 2609    brain_mask_eroded=None,
 2610    output_directory=None,
 2611    verbose=False, **kwargs
 2612):
 2613    """
 2614    Correct time-series data for motion - with optional deformation.
 2615
 2616    Arguments
 2617    ---------
 2618        image: antsImage, usually ND where D=4.
 2619
 2620        avg_b0: Fixed image b0 image
 2621
 2622        avg_dwi: Fixed dwi same space as b0 image
 2623
 2624        bvals: bvalues (file or array)
 2625
 2626        bvecs: bvecs (file or array)
 2627
 2628        b0_idx: indices of b0
 2629
 2630        type_of_transform : string
 2631            A linear or non-linear registration type. Mutual information metric and rigid transformation by default.
 2632            See ants registration for details.
 2633
 2634        fdOffset: offset value to use in framewise displacement calculation
 2635
 2636        mask_csf: boolean
 2637
 2638        brain_mask_eroded: optional mask that will trigger mixed interpolation
 2639
 2640        output_directory : string
 2641            output will be placed in this directory plus a numeric extension.
 2642
 2643        verbose: boolean
 2644
 2645        kwargs: keyword args
 2646            extra arguments - these extra arguments will control the details of registration that is performed. see ants registration for more.
 2647
 2648    Returns
 2649    -------
 2650    dict containing follow key/value pairs:
 2651        `motion_corrected`: Moving image warped to space of fixed image.
 2652        `motion_parameters`: transforms for each image in the time series.
 2653        `FD`: Framewise displacement generalized for arbitrary transformations.
 2654
 2655    Notes
 2656    -----
 2657    Control extra arguments via kwargs. see ants.registration for details.
 2658
 2659    Example
 2660    -------
 2661    >>> import ants
 2662    """
 2663
 2664    idim = image.dimension
 2665    ishape = image.shape
 2666    nTimePoints = ishape[idim - 1]
 2667    FD = np.zeros(nTimePoints)
 2668    if bvals is not None and bvecs is not None:
 2669        if isinstance(bvecs, str):
 2670            bvals, bvecs = read_bvals_bvecs( bvals , bvecs  )
 2671        else: # assume we already read them
 2672            bvals = bvals.copy()
 2673            bvecs = bvecs.copy()
 2674    if type_of_transform is None:
 2675        return {
 2676            "motion_corrected": image,
 2677            "motion_parameters": None,
 2678            "FD": FD,
 2679            'bvals':bvals,
 2680            'bvecs':bvecs
 2681        }
 2682
 2683    from scipy.linalg import inv, polar
 2684    from dipy.core.gradients import reorient_bvecs
 2685
 2686    remove_it=False
 2687    if output_directory is None:
 2688        remove_it=True
 2689        output_directory = tempfile.mkdtemp()
 2690    output_directory_w = output_directory + "/dti_reg/"
 2691    os.makedirs(output_directory_w,exist_ok=True)
 2692    ofnG = tempfile.NamedTemporaryFile(delete=False,suffix='global_deformation',dir=output_directory_w).name
 2693    ofnL = tempfile.NamedTemporaryFile(delete=False,suffix='local_deformation',dir=output_directory_w).name
 2694    if verbose:
 2695        print(output_directory_w)
 2696        print(ofnG)
 2697        print(ofnL)
 2698        print("remove_it " + str( remove_it ) )
 2699
 2700    if b0_idx is None:
 2701        # b0_idx = segment_timeseries_by_meanvalue( image )['highermeans']
 2702        b0_idx = segment_timeseries_by_bvalue( bvals )['lowbvals']
 2703
 2704    # first get a local deformation from slice to local avg space
 2705    # then get a global deformation from avg to ref space
 2706    ab0, adw = get_average_dwi_b0( image )
 2707    # mask is used to roughly locate middle of brain
 2708    mask = ants.threshold_image( ants.iMath(adw,'Normalize'), 0.1, 1.0 )
 2709    if brain_mask_eroded is None:
 2710        brain_mask_eroded = mask * 0 + 1
 2711    motion_parameters = list()
 2712    motion_corrected = list()
 2713    centerOfMass = mask.get_center_of_mass()
 2714    npts = pow(2, idim - 1)
 2715    pointOffsets = np.zeros((npts, idim - 1))
 2716    myrad = np.ones(idim - 1).astype(int).tolist()
 2717    mask1vals = np.zeros(int(mask.sum()))
 2718    mask1vals[round(len(mask1vals) / 2)] = 1
 2719    mask1 = ants.make_image(mask, mask1vals)
 2720    myoffsets = ants.get_neighborhood_in_mask(
 2721        mask1, mask1, radius=myrad, spatial_info=True
 2722    )["offsets"]
 2723    mycols = list("xy")
 2724    if idim - 1 == 3:
 2725        mycols = list("xyz")
 2726    useinds = list()
 2727    for k in range(myoffsets.shape[0]):
 2728        if abs(myoffsets[k, :]).sum() == (idim - 2):
 2729            useinds.append(k)
 2730        myoffsets[k, :] = myoffsets[k, :] * fdOffset / 2.0 + centerOfMass
 2731    fdpts = pd.DataFrame(data=myoffsets[useinds, :], columns=mycols)
 2732
 2733
 2734    if verbose:
 2735        print("begin global distortion correction")
 2736    # initrig = tra_initializer(avg_b0, ab0, max_rotation=60, transform=['rigid'], verbose=verbose)
 2737    if mask_csf:
 2738        bcsf = ants.threshold_image( avg_b0,"Otsu",2).threshold_image(1,1).morphology("open",1).iMath("GetLargestComponent")
 2739    else:
 2740        bcsf = ab0 * 0 + 1
 2741
 2742    initrig = ants.registration( avg_b0, ab0,'antsRegistrationSyNRepro[r]',outprefix=ofnG)
 2743    deftx = ants.registration( avg_dwi, adw, 'SyNOnly',
 2744        syn_metric='CC', syn_sampling=2,
 2745        reg_iterations=[50,50,20],
 2746        multivariate_extras=[ [ "CC", avg_b0, ab0, 1, 2 ]],
 2747        initial_transform=initrig['fwdtransforms'][0],
 2748        outprefix=ofnG
 2749        )['fwdtransforms']
 2750    if verbose:
 2751        print("end global distortion correction")
 2752
 2753    if verbose:
 2754        print("Progress:")
 2755    counter = round( nTimePoints / 10 ) + 1
 2756    for k in range(nTimePoints):
 2757        if verbose and nTimePoints > 0 and ( ( k % counter ) ==  0 ) or ( k == (nTimePoints-1) ):
 2758            myperc = round( k / nTimePoints * 100)
 2759            print(myperc, end="%.", flush=True)
 2760        if k in b0_idx:
 2761            fixed=ants.image_clone( ab0 )
 2762        else:
 2763            fixed=ants.image_clone( adw )
 2764        temp = ants.slice_image(image, axis=idim - 1, idx=k)
 2765        temp = ants.iMath(temp, "Normalize")
 2766        txprefix = ofnL+str(k).zfill(4)+"rig_"
 2767        txprefix2 = ofnL+str(k % 2).zfill(4)+"def_"
 2768        if temp.numpy().var() > 0:
 2769            myrig = ants.registration(
 2770                    fixed, temp,
 2771                    type_of_transform='antsRegistrationSyNRepro[r]',
 2772                    outprefix=txprefix,
 2773                    **kwargs
 2774                )
 2775            if type_of_transform == 'SyN':
 2776                myreg = ants.registration(
 2777                    fixed, temp,
 2778                    type_of_transform='SyNOnly',
 2779                    total_sigma=total_sigma, grad_step=0.1,
 2780                    initial_transform=myrig['fwdtransforms'][0],
 2781                    outprefix=txprefix2,
 2782                    **kwargs
 2783                )
 2784            else:
 2785                myreg = myrig
 2786            fdptsTxI = ants.apply_transforms_to_points(
 2787                idim - 1, fdpts, myrig["fwdtransforms"]
 2788            )
 2789            if k > 0 and motion_parameters[k - 1] != "NA":
 2790                fdptsTxIminus1 = ants.apply_transforms_to_points(
 2791                    idim - 1, fdpts, motion_parameters[k - 1]
 2792                )
 2793            else:
 2794                fdptsTxIminus1 = fdptsTxI
 2795            # take the absolute value, then the mean across columns, then the sum
 2796            FD[k] = (fdptsTxIminus1 - fdptsTxI).abs().mean().sum()
 2797            motion_parameters.append(myreg["fwdtransforms"])
 2798        else:
 2799            motion_parameters.append("NA")
 2800
 2801        temp = ants.slice_image(image, axis=idim - 1, idx=k)
 2802        if k in b0_idx:
 2803            fixed=ants.image_clone( ab0 )
 2804        else:
 2805            fixed=ants.image_clone( adw )
 2806        if temp.numpy().var() > 0:
 2807            motion_parameters[k]=deftx+motion_parameters[k]
 2808            img1w = apply_transforms_mixed_interpolation( avg_dwi,
 2809                ants.slice_image(image, axis=idim - 1, idx=k),
 2810                motion_parameters[k], mask=brain_mask_eroded )
 2811            motion_corrected.append(img1w)
 2812        else:
 2813            motion_corrected.append(fixed)
 2814
 2815    if verbose:
 2816        print("Reorient bvecs")
 2817    if bvecs is not None:
 2818        #    direction = target->GetDirection().GetTranspose() * img_mov->GetDirection().GetVnlMatrix();
 2819        rebase = np.dot( np.transpose( avg_b0.direction  ), ab0.direction )
 2820        bvecs = bvec_reorientation( motion_parameters, bvecs, rebase )
 2821
 2822    if remove_it:
 2823        import shutil
 2824        shutil.rmtree(output_directory, ignore_errors=True )
 2825
 2826    if verbose:
 2827        print("Done")
 2828    d4siz = list(avg_b0.shape)
 2829    d4siz.append( 2 )
 2830    spc = list(ants.get_spacing( avg_b0 ))
 2831    spc.append( 1.0 )
 2832    mydir = ants.get_direction( avg_b0 )
 2833    mydir4d = ants.get_direction( image )
 2834    mydir4d[0:3,0:3]=mydir
 2835    myorg = list(ants.get_origin( avg_b0 ))
 2836    myorg.append( 0.0 )
 2837    avg_b0_4d = ants.make_image(d4siz,0,spacing=spc,origin=myorg,direction=mydir4d)
 2838    return {
 2839        "motion_corrected": ants.list_to_ndimage(avg_b0_4d, motion_corrected),
 2840        "motion_parameters": motion_parameters,
 2841        "FD": FD,
 2842        'bvals':bvals,
 2843        'bvecs':bvecs
 2844    }
 2845
 2846
 2847def mc_reg(
 2848    image,
 2849    fixed=None,
 2850    type_of_transform="antsRegistrationSyNRepro[r]",
 2851    mask=None,
 2852    total_sigma=3.0,
 2853    fdOffset=2.0,
 2854    output_directory=None,
 2855    verbose=False, **kwargs
 2856):
 2857    """
 2858    Correct time-series data for motion - with deformation.
 2859
 2860    Arguments
 2861    ---------
 2862        image: antsImage, usually ND where D=4.
 2863
 2864        fixed: Fixed image to register all timepoints to.  If not provided,
 2865            mean image is used.
 2866
 2867        type_of_transform : string
 2868            A linear or non-linear registration type. Mutual information metric and rigid transformation by default.
 2869            See ants registration for details.
 2870
 2871        fdOffset: offset value to use in framewise displacement calculation
 2872
 2873        output_directory : string
 2874            output will be named with this prefix plus a numeric extension.
 2875
 2876        verbose: boolean
 2877
 2878        kwargs: keyword args
 2879            extra arguments - these extra arguments will control the details of registration that is performed. see ants registration for more.
 2880
 2881    Returns
 2882    -------
 2883    dict containing follow key/value pairs:
 2884        `motion_corrected`: Moving image warped to space of fixed image.
 2885        `motion_parameters`: transforms for each image in the time series.
 2886        `FD`: Framewise displacement generalized for arbitrary transformations.
 2887
 2888    Notes
 2889    -----
 2890    Control extra arguments via kwargs. see ants.registration for details.
 2891
 2892    Example
 2893    -------
 2894    >>> import ants
 2895    >>> fi = ants.image_read(ants.get_ants_data('ch2'))
 2896    >>> mytx = ants.motion_correction( fi )
 2897    """
 2898    remove_it=False
 2899    if output_directory is None:
 2900        remove_it=True
 2901        output_directory = tempfile.mkdtemp()
 2902    output_directory_w = output_directory + "/mc_reg/"
 2903    os.makedirs(output_directory_w,exist_ok=True)
 2904    ofnG = tempfile.NamedTemporaryFile(delete=False,suffix='global_deformation',dir=output_directory_w).name
 2905    ofnL = tempfile.NamedTemporaryFile(delete=False,suffix='local_deformation',dir=output_directory_w).name
 2906    if verbose:
 2907        print(output_directory_w)
 2908        print(ofnG)
 2909        print(ofnL)
 2910
 2911    idim = image.dimension
 2912    ishape = image.shape
 2913    nTimePoints = ishape[idim - 1]
 2914    if fixed is None:
 2915        fixed = ants.get_average_of_timeseries( image )
 2916    if mask is None:
 2917        mask = ants.get_mask(fixed)
 2918    FD = np.zeros(nTimePoints)
 2919    motion_parameters = list()
 2920    motion_corrected = list()
 2921    centerOfMass = mask.get_center_of_mass()
 2922    npts = pow(2, idim - 1)
 2923    pointOffsets = np.zeros((npts, idim - 1))
 2924    myrad = np.ones(idim - 1).astype(int).tolist()
 2925    mask1vals = np.zeros(int(mask.sum()))
 2926    mask1vals[round(len(mask1vals) / 2)] = 1
 2927    mask1 = ants.make_image(mask, mask1vals)
 2928    myoffsets = ants.get_neighborhood_in_mask(
 2929        mask1, mask1, radius=myrad, spatial_info=True
 2930    )["offsets"]
 2931    mycols = list("xy")
 2932    if idim - 1 == 3:
 2933        mycols = list("xyz")
 2934    useinds = list()
 2935    for k in range(myoffsets.shape[0]):
 2936        if abs(myoffsets[k, :]).sum() == (idim - 2):
 2937            useinds.append(k)
 2938        myoffsets[k, :] = myoffsets[k, :] * fdOffset / 2.0 + centerOfMass
 2939    fdpts = pd.DataFrame(data=myoffsets[useinds, :], columns=mycols)
 2940    if verbose:
 2941        print("Progress:")
 2942    counter = 0
 2943    for k in range(nTimePoints):
 2944        mycount = round(k / nTimePoints * 100)
 2945        if verbose and mycount == counter:
 2946            counter = counter + 10
 2947            print(mycount, end="%.", flush=True)
 2948        temp = ants.slice_image(image, axis=idim - 1, idx=k)
 2949        temp = ants.iMath(temp, "Normalize")
 2950        if temp.numpy().var() > 0:
 2951            myrig = ants.registration(
 2952                    fixed, temp,
 2953                    type_of_transform='antsRegistrationSyNRepro[r]',
 2954                    outprefix=ofnL+str(k).zfill(4)+"_",
 2955                    **kwargs
 2956                )
 2957            if type_of_transform == 'SyN':
 2958                myreg = ants.registration(
 2959                    fixed, temp,
 2960                    type_of_transform='SyNOnly',
 2961                    total_sigma=total_sigma,
 2962                    initial_transform=myrig['fwdtransforms'][0],
 2963                    outprefix=ofnL+str(k).zfill(4)+"_",
 2964                    **kwargs
 2965                )
 2966            else:
 2967                myreg = myrig
 2968            fdptsTxI = ants.apply_transforms_to_points(
 2969                idim - 1, fdpts, myreg["fwdtransforms"]
 2970            )
 2971            if k > 0 and motion_parameters[k - 1] != "NA":
 2972                fdptsTxIminus1 = ants.apply_transforms_to_points(
 2973                    idim - 1, fdpts, motion_parameters[k - 1]
 2974                )
 2975            else:
 2976                fdptsTxIminus1 = fdptsTxI
 2977            # take the absolute value, then the mean across columns, then the sum
 2978            FD[k] = (fdptsTxIminus1 - fdptsTxI).abs().mean().sum()
 2979            motion_parameters.append(myreg["fwdtransforms"])
 2980            img1w = ants.apply_transforms( fixed,
 2981                ants.slice_image(image, axis=idim - 1, idx=k),
 2982                myreg["fwdtransforms"] )
 2983            motion_corrected.append(img1w)
 2984        else:
 2985            motion_parameters.append("NA")
 2986            motion_corrected.append(temp)
 2987
 2988    if remove_it:
 2989        import shutil
 2990        shutil.rmtree(output_directory, ignore_errors=True )
 2991
 2992    if verbose:
 2993        print("Done")
 2994    return {
 2995        "motion_corrected": ants.list_to_ndimage(image, motion_corrected),
 2996        "motion_parameters": motion_parameters,
 2997        "FD": FD,
 2998    }
 2999
 3000def map_scalar_to_labels(dataframe, label_image_template):
 3001    """
 3002    Map scalar values from a DataFrame to associated integer image labels.
 3003
 3004    Parameters:
 3005    - dataframe (pd.DataFrame): A Pandas DataFrame containing a label column and scalar_value column.
 3006    - label_image_template (ants.ANTsImage): ANTs image with (at least some of) the same values as labels.
 3007
 3008    Returns:
 3009    - ants.ANTsImage: A label image with scalar values mapped to associated integer labels.
 3010    """
 3011
 3012    # Create an empty label image with the same geometry as the template
 3013    mapped_label_image = label_image_template.clone() * 0.0
 3014
 3015    # Loop through DataFrame and map scalar values to labels
 3016    for index, row in dataframe.iterrows():
 3017        label = int(row['label'])  # Assuming the DataFrame has a 'label' column
 3018        scalar_value = row['scalar_value']  # Replace with your column name
 3019        mapped_label_image[label_image_template == label] = scalar_value
 3020
 3021    return mapped_label_image
 3022
 3023
 3024def template_figure_with_overlay(scalar_label_df, prefix, outputfilename=None, template='cit168', xyz=None, mask_dilation=25, padding=12, verbose=True):
 3025    """
 3026    Process and visualize images with mapped scalar values.
 3027
 3028    Parameters:
 3029    - scalar_label_df (pd.DataFrame): A Pandas DataFrame containing scalar values and labels.
 3030    - prefix (str): The prefix for input image files.
 3031    - template (str, optional): Template for selecting image data (default is 'cit168').
 3032    - xyz (str, optional): The integer index of the slices to display.
 3033    - mask_dilation (int, optional): Dilation factor for creating a mask (default is 25).
 3034    - padding (int, optional): Padding value for the mapped images (default is 12).
 3035    - verbose (bool, optional): Enable verbose mode for printing (default is True).
 3036
 3037    Example Usage:
 3038    >>> scalar_label_df = pd.DataFrame({'label': [1, 2, 3], 'scalar_value': [0.5, 0.8, 1.2]})
 3039    >>> prefix = '../PPMI_template0_'
 3040    >>> process_and_visualize_images(scalar_label_df, prefix, template='cit168', xyz=None, mask_dilation=25, padding=12, verbose=True)
 3041    """
 3042
 3043    # Template image paths
 3044    template_paths = {
 3045        'cit168': 'cit168lab.nii.gz',
 3046        'bf': 'bf.nii.gz',
 3047        'cerebellum': 'cerebellum.nii.gz',
 3048        'mtl': 'mtl.nii.gz',
 3049        'ctx': 'dkt_cortex.nii.gz',
 3050        'jhuwm': 'JHU_wm.nii.gz'
 3051    }
 3052
 3053    if template not in template_paths:
 3054        print( "Valid options:")
 3055        print( template_paths )
 3056        raise ValueError(f"Template option '{template}' does not exist.")
 3057
 3058    template_image_path = template_paths[template]
 3059    template_image = ants.image_read(f'{prefix}{template_image_path}')
 3060
 3061    # Load image data
 3062    edgeimg = ants.image_read(f'{prefix}edge.nii.gz')
 3063    dktimg = ants.image_read(f'{prefix}dkt_parcellation.nii.gz')
 3064    segimg = ants.image_read(f'{prefix}tissue_segmentation.nii.gz')
 3065    ttl = ''
 3066
 3067    # Load and process the template image
 3068    ventricles = ants.threshold_image(dktimg, 4, 4) + ants.threshold_image(dktimg, 43, 43)
 3069    seggm = ants.mask_image(segimg, segimg, [2, 4], binarize=False)
 3070    edgeimg = edgeimg.clone()
 3071    edgeimg[edgeimg == 0] = ventricles[edgeimg == 0]
 3072    segwm = ants.threshold_image(segimg, 3, 4).morphology("open", 1)
 3073
 3074    # Define cropping mask
 3075    cmask = ants.threshold_image(template_image, 1, 1.e9).iMath("MD", mask_dilation)
 3076
 3077    mapped_image = map_scalar_to_labels(scalar_label_df, template_image)
 3078    tcrop = ants.crop_image(template_image, cmask)
 3079    toviz = ants.crop_image(mapped_image, cmask)
 3080    seggm = ants.crop_image(edgeimg, cmask)
 3081       
 3082    # Map scalar values to labels and visualize
 3083    toviz = ants.pad_image(toviz, pad_width=(padding, padding, padding))
 3084    seggm = ants.pad_image(seggm, pad_width=(padding, padding, padding))
 3085    tcrop = ants.pad_image(tcrop, pad_width=(padding, padding, padding))
 3086
 3087    if xyz is None:
 3088        if template == 'cit168':
 3089            xyz=[140, 89, 94]
 3090        elif template == 'bf':
 3091            xyz=[114,92,76]
 3092        elif template == 'cerebellum':
 3093            xyz=[169, 128, 137]
 3094        elif template == 'mtl':
 3095            xyz=[154, 112, 113]
 3096        elif template == 'ctx':
 3097            xyz=[233, 190, 174]
 3098        elif template == 'jhuwm':
 3099            xyz=[146, 133, 182]
 3100
 3101    if verbose:
 3102        print("plot xyz for " + template )
 3103        print( xyz )
 3104        
 3105    if outputfilename is None:
 3106        temp = ants.plot_ortho( seggm, overlay=toviz, crop=False,
 3107                        xyz=xyz, cbar_length=0.2, cbar_vertical=False,
 3108                        flat=True, xyz_lines=False, resample=False, orient_labels=False,
 3109                        title=ttl, titlefontsize=12, title_dy=-0.02, textfontcolor='red', 
 3110                        cbar=True, allow_xyz_change=False)
 3111    else:
 3112        temp = ants.plot_ortho( seggm, overlay=toviz, crop=False,
 3113                    xyz=xyz, cbar_length=0.2, cbar_vertical=False,
 3114                    flat=True, xyz_lines=False, resample=False, orient_labels=False,
 3115                    title=ttl, titlefontsize=12, title_dy=-0.02, textfontcolor='red', 
 3116                    cbar=True, allow_xyz_change=False, filename=outputfilename )
 3117    seggm = temp['image']
 3118    toviz = temp['overlay']
 3119    return { "underlay": seggm, 'overlay': toviz, 'seg': tcrop  }
 3120
 3121def get_data( name=None, force_download=False, version=26, target_extension='.csv' ):
 3122    """
 3123    Get ANTsPyMM data filename
 3124
 3125    The first time this is called, it will download data to ~/.antspymm.
 3126    After, it will just read data from disk.  The ~/.antspymm may need to
 3127    be periodically deleted in order to ensure data is current.
 3128
 3129    Arguments
 3130    ---------
 3131    name : string
 3132        name of data tag to retrieve
 3133        Options:
 3134            - 'all'
 3135
 3136    force_download: boolean
 3137
 3138    version: version of data to download (integer)
 3139
 3140    Returns
 3141    -------
 3142    string
 3143        filepath of selected data
 3144
 3145    Example
 3146    -------
 3147    >>> import antspymm
 3148    >>> antspymm.get_data()
 3149    """
 3150    os.makedirs(DATA_PATH, exist_ok=True)
 3151
 3152    def mv_subfolder_files(folder, verbose=False):
 3153        """
 3154        Move files from subfolders to the parent folder.
 3155
 3156        Parameters
 3157        ----------
 3158        folder : str
 3159            Path to the folder.
 3160        verbose : bool, optional
 3161            Print information about the files and folders being processed (default is False).
 3162
 3163        Returns
 3164        -------
 3165        None
 3166        """
 3167        import os
 3168        import shutil
 3169        for root, dirs, files in os.walk(folder):
 3170            if verbose:
 3171                print(f"Processing directory: {root}")
 3172                print(f"Subdirectories: {dirs}")
 3173                print(f"Files: {files}")
 3174            
 3175            for file in files:
 3176                if root != folder:
 3177                    if verbose:
 3178                        print(f"Moving file: {file} from {root} to {folder}")
 3179                    shutil.move(os.path.join(root, file), folder)
 3180            
 3181            for dir in dirs:
 3182                if root != folder:
 3183                    if verbose:
 3184                        print(f"Removing directory: {dir} from {root}")
 3185                    shutil.rmtree(os.path.join(root, dir))
 3186
 3187    def download_data( version ):
 3188        url = "https://figshare.com/ndownloader/articles/16912366/versions/" + str(version)
 3189        target_file_name = "16912366.zip"
 3190        target_file_name_path = tf.keras.utils.get_file(target_file_name, url,
 3191            cache_subdir=DATA_PATH, extract = True )
 3192        mv_subfolder_files( os.path.expanduser("~/.antspymm"), False )
 3193        os.remove( DATA_PATH + target_file_name )
 3194
 3195    if force_download:
 3196        download_data( version = version )
 3197
 3198
 3199    files = []
 3200    for fname in os.listdir(DATA_PATH):
 3201        if ( fname.endswith(target_extension) ) :
 3202            fname = os.path.join(DATA_PATH, fname)
 3203            files.append(fname)
 3204
 3205    if len( files ) == 0 :
 3206        download_data( version = version )
 3207        for fname in os.listdir(DATA_PATH):
 3208            if ( fname.endswith(target_extension) ) :
 3209                fname = os.path.join(DATA_PATH, fname)
 3210                files.append(fname)
 3211
 3212
 3213    if name == 'all':
 3214        return files
 3215
 3216    datapath = None
 3217
 3218    for fname in os.listdir(DATA_PATH):
 3219        mystem = (Path(fname).resolve().stem)
 3220        mystem = (Path(mystem).resolve().stem)
 3221        mystem = (Path(mystem).resolve().stem)
 3222        if ( name == mystem and fname.endswith(target_extension) ) :
 3223            datapath = os.path.join(DATA_PATH, fname)
 3224
 3225    return datapath
 3226
 3227
 3228def get_models( version=3, force_download=True ):
 3229    """
 3230    Get ANTsPyMM data models
 3231
 3232    force_download: boolean
 3233
 3234    Returns
 3235    -------
 3236    None
 3237
 3238    """
 3239    os.makedirs(DATA_PATH, exist_ok=True)
 3240
 3241    def download_data( version ):
 3242        url = "https://figshare.com/ndownloader/articles/21718412/versions/"+str(version)
 3243        target_file_name = "21718412.zip"
 3244        target_file_name_path = tf.keras.utils.get_file(target_file_name, url,
 3245            cache_subdir=DATA_PATH, extract = True )
 3246        os.remove( DATA_PATH + target_file_name )
 3247
 3248    if force_download:
 3249        download_data( version = version )
 3250    return
 3251
 3252
 3253
 3254def dewarp_imageset( image_list, initial_template=None,
 3255    iterations=None, padding=0, target_idx=[0], **kwargs ):
 3256    """
 3257    Dewarp a set of images
 3258
 3259    Makes simplifying heuristic decisions about how to transform an image set
 3260    into an unbiased reference space.  Will handle plenty of decisions
 3261    automatically so beware.  Computes an average shape space for the images
 3262    and transforms them to that space.
 3263
 3264    Arguments
 3265    ---------
 3266    image_list : list containing antsImages 2D, 3D or 4D
 3267
 3268    initial_template : optional
 3269
 3270    iterations : number of template building iterations
 3271
 3272    padding:  will pad the images by an integer amount to limit edge effects
 3273
 3274    target_idx : the target indices for the time series over which we should average;
 3275        a list of integer indices into the last axis of the input images.
 3276
 3277    kwargs : keyword args
 3278        arguments passed to ants registration - these must be set explicitly
 3279
 3280    Returns
 3281    -------
 3282    a dictionary with the mean image and the list of the transformed images as
 3283    well as motion correction parameters for each image in the input list
 3284
 3285    Example
 3286    -------
 3287    >>> import antspymm
 3288    """
 3289    outlist = []
 3290    avglist = []
 3291    if len(image_list[0].shape) > 3:
 3292        imagetype = 3
 3293        for k in range(len(image_list)):
 3294            for j in range(len(target_idx)):
 3295                avglist.append( ants.slice_image( image_list[k], axis=3, idx=target_idx[j] ) )
 3296    else:
 3297        imagetype = 0
 3298        avglist=image_list
 3299
 3300    pw=[]
 3301    for k in range(len(avglist[0].shape)):
 3302        pw.append( padding )
 3303    for k in range(len(avglist)):
 3304        avglist[k] = ants.pad_image( avglist[k], pad_width=pw  )
 3305
 3306    if initial_template is None:
 3307        initial_template = avglist[0] * 0
 3308        for k in range(len(avglist)):
 3309            initial_template = initial_template + avglist[k]/len(avglist)
 3310
 3311    if iterations is None:
 3312        iterations = 2
 3313
 3314    btp = ants.build_template(
 3315        initial_template=initial_template,
 3316        image_list=avglist,
 3317        gradient_step=0.5, blending_weight=0.8,
 3318        iterations=iterations, verbose=False, **kwargs )
 3319
 3320    # last - warp all images to this frame
 3321    mocoplist = []
 3322    mocofdlist = []
 3323    reglist = []
 3324    for k in range(len(image_list)):
 3325        if imagetype == 3:
 3326            moco0 = ants.motion_correction( image=image_list[k], fixed=btp, type_of_transform='antsRegistrationSyNRepro[r]' )
 3327            mocoplist.append( moco0['motion_parameters'] )
 3328            mocofdlist.append( moco0['FD'] )
 3329            locavg = ants.slice_image( moco0['motion_corrected'], axis=3, idx=0 ) * 0.0
 3330            for j in range(len(target_idx)):
 3331                locavg = locavg + ants.slice_image( moco0['motion_corrected'], axis=3, idx=target_idx[j] )
 3332            locavg = locavg * 1.0 / len(target_idx)
 3333        else:
 3334            locavg = image_list[k]
 3335        reg = ants.registration( btp, locavg, **kwargs )
 3336        reglist.append( reg )
 3337        if imagetype == 3:
 3338            myishape = image_list[k].shape
 3339            mytslength = myishape[ len(myishape) - 1 ]
 3340            mywarpedlist = []
 3341            for j in range(mytslength):
 3342                locimg = ants.slice_image( image_list[k], axis=3, idx = j )
 3343                mywarped = ants.apply_transforms( btp, locimg,
 3344                    reg['fwdtransforms'] + moco0['motion_parameters'][j], imagetype=0 )
 3345                mywarpedlist.append( mywarped )
 3346            mywarped = ants.list_to_ndimage( image_list[k], mywarpedlist )
 3347        else:
 3348            mywarped = ants.apply_transforms( btp, image_list[k], reg['fwdtransforms'], imagetype=imagetype )
 3349        outlist.append( mywarped )
 3350
 3351    return {
 3352        'dewarpedmean':btp,
 3353        'dewarped':outlist,
 3354        'deformable_registrations': reglist,
 3355        'FD': mocofdlist,
 3356        'motionparameters': mocoplist }
 3357
 3358
 3359def super_res_mcimage( image,
 3360    srmodel,
 3361    truncation=[0.0001,0.995],
 3362    poly_order='hist',
 3363    target_range=[0,1],
 3364    isotropic = False,
 3365    verbose=False ):
 3366    """
 3367    Super resolution on a timeseries or multi-channel image
 3368
 3369    Arguments
 3370    ---------
 3371    image : an antsImage
 3372
 3373    srmodel : a tensorflow fully convolutional model
 3374
 3375    truncation :  quantiles at which we truncate intensities to limit impact of outliers e.g. [0.005,0.995]
 3376
 3377    poly_order : if not None, will fit a global regression model to map
 3378        intensity back to original histogram space; if 'hist' will match
 3379        by histogram matching - ants.histogram_match_image
 3380
 3381    target_range : 2-element tuple
 3382        a tuple or array defining the (min, max) of the input image
 3383        (e.g., [-127.5, 127.5] or [0,1]).  Output images will be scaled back to original
 3384        intensity. This range should match the mapping used in the training
 3385        of the network.
 3386
 3387    isotropic : boolean
 3388
 3389    verbose : boolean
 3390
 3391    Returns
 3392    -------
 3393    super resolution version of the image
 3394
 3395    Example
 3396    -------
 3397    >>> import antspymm
 3398    """
 3399    idim = image.dimension
 3400    ishape = image.shape
 3401    nTimePoints = ishape[idim - 1]
 3402    mcsr = list()
 3403    for k in range(nTimePoints):
 3404        if verbose and (( k % 5 ) == 0 ):
 3405            mycount = round(k / nTimePoints * 100)
 3406            print(mycount, end="%.", flush=True)
 3407        temp = ants.slice_image( image, axis=idim - 1, idx=k )
 3408        temp = ants.iMath( temp, "TruncateIntensity", truncation[0], truncation[1] )
 3409        mysr = antspynet.apply_super_resolution_model_to_image( temp, srmodel,
 3410            target_range = target_range )
 3411        if poly_order is not None:
 3412            bilin = ants.resample_image_to_target( temp, mysr )
 3413            if poly_order == 'hist':
 3414                mysr = ants.histogram_match_image( mysr, bilin )
 3415            else:
 3416                mysr = ants.regression_match_image( mysr, bilin, poly_order = poly_order )
 3417        if isotropic:
 3418            mysr = down2iso( mysr )
 3419        if k == 0:
 3420            upshape = list()
 3421            for j in range(len(ishape)-1):
 3422                upshape.append( mysr.shape[j] )
 3423            upshape.append( ishape[ idim-1 ] )
 3424            if verbose:
 3425                print("SR will be of voxel size:" + str(upshape) )
 3426        mcsr.append( mysr )
 3427
 3428    upshape = list()
 3429    for j in range(len(ishape)-1):
 3430        upshape.append( mysr.shape[j] )
 3431    upshape.append( ishape[ idim-1 ] )
 3432    if verbose:
 3433        print("SR will be of voxel size:" + str(upshape) )
 3434
 3435    imageup = ants.resample_image( image, upshape, use_voxels = True )
 3436    if verbose:
 3437        print("Done")
 3438
 3439    return ants.list_to_ndimage( imageup, mcsr )
 3440
 3441
 3442def segment_timeseries_by_bvalue(bvals):
 3443    """
 3444    Segments a time series based on a threshold applied to b-values.
 3445    
 3446    This function categorizes indices of the given b-values array into two groups:
 3447    one for indices where b-values are above a near-zero threshold, and another
 3448    where b-values are at or below this threshold. The threshold is set to 1e-12.
 3449    
 3450    Parameters:
 3451    - bvals (numpy.ndarray): An array of b-values.
 3452
 3453    Returns:
 3454    - dict: A dictionary with two keys, 'largerbvals' and 'lowbvals', each containing
 3455      the indices of bvals where the b-values are above and at/below the threshold, respectively.
 3456    """
 3457    # Define the threshold
 3458    threshold = 1e-12
 3459    def find_min_value(data):
 3460        if isinstance(data, list):
 3461            return min(data)
 3462        elif isinstance(data, np.ndarray):
 3463            return np.min(data)
 3464        else:
 3465            raise TypeError("Input must be either a list or a numpy array")
 3466
 3467    # Get indices where b-values are greater than the threshold
 3468    lowermeans = list(np.where(bvals > threshold)[0])
 3469    
 3470    # Get indices where b-values are less than or equal to the threshold
 3471    highermeans = list(np.where(bvals <= threshold)[0])
 3472    
 3473    if len(highermeans) == 0:
 3474        minval = find_min_value( bvals )
 3475        lowermeans = list(np.where(bvals > minval )[0])
 3476        highermeans = list(np.where(bvals <= minval)[0])
 3477
 3478    return {
 3479        'largerbvals': lowermeans,
 3480        'lowbvals': highermeans
 3481    }
 3482
 3483def segment_timeseries_by_meanvalue( image, quantile = 0.995 ):
 3484    """
 3485    Identify indices of a time series where we assume there is a different mean
 3486    intensity over the volumes.  The indices of volumes with higher and lower
 3487    intensities is returned.  Can be used to automatically identify B0 volumes
 3488    in DWI timeseries.
 3489
 3490    Arguments
 3491    ---------
 3492    image : an antsImage holding B0 and DWI
 3493
 3494    quantile : a quantile for splitting the indices of the volume - should be greater than 0.5
 3495
 3496    Returns
 3497    -------
 3498    dictionary holding the two sets of indices
 3499
 3500    Example
 3501    -------
 3502    >>> import antspymm
 3503    """
 3504    ishape = image.shape
 3505    lastdim = len(ishape)-1
 3506    meanvalues = list()
 3507    for x in range(ishape[lastdim]):
 3508        meanvalues.append(  ants.slice_image( image, axis=lastdim, idx=x ).mean() )
 3509    myhiq = np.quantile( meanvalues, quantile )
 3510    myloq = np.quantile( meanvalues, 1.0 - quantile )
 3511    lowerindices = list()
 3512    higherindices = list()
 3513    for x in range(len(meanvalues)):
 3514        hiabs = abs( meanvalues[x] - myhiq )
 3515        loabs = abs( meanvalues[x] - myloq )
 3516        if hiabs < loabs:
 3517            higherindices.append(x)
 3518        else:
 3519            lowerindices.append(x)
 3520
 3521    return {
 3522    'lowermeans':lowerindices,
 3523    'highermeans':higherindices }
 3524
 3525
 3526def get_average_rsf( x, min_t=10, max_t=35 ):
 3527    """
 3528    automatically generates the average bold image with quick registration
 3529
 3530    returns:
 3531        avg_bold
 3532    """
 3533    output_directory = tempfile.mkdtemp()
 3534    ofn = output_directory + "/w"
 3535    bavg = ants.slice_image( x, axis=3, idx=0 ) * 0.0
 3536    oavg = ants.slice_image( x, axis=3, idx=0 )
 3537    if x.shape[3] <= min_t:
 3538        min_t=0
 3539    if x.shape[3] <= max_t:
 3540        max_t=x.shape[3]
 3541    for myidx in range(min_t,max_t):
 3542        b0 = ants.slice_image( x, axis=3, idx=myidx)
 3543        bavg = bavg + ants.registration(oavg,b0,'antsRegistrationSyNRepro[r]',outprefix=ofn)['warpedmovout']
 3544    bavg = ants.iMath( bavg, 'Normalize' )
 3545    oavg = ants.image_clone( bavg )
 3546    bavg = oavg * 0.0
 3547    for myidx in range(min_t,max_t):
 3548        b0 = ants.slice_image( x, axis=3, idx=myidx)
 3549        bavg = bavg + ants.registration(oavg,b0,'antsRegistrationSyNRepro[r]',outprefix=ofn)['warpedmovout']
 3550    import shutil
 3551    shutil.rmtree(output_directory, ignore_errors=True )
 3552    bavg = ants.iMath( bavg, 'Normalize' )
 3553    return bavg
 3554    # return ants.n4_bias_field_correction(bavg, mask=ants.get_mask( bavg ) )
 3555
 3556
 3557def get_average_dwi_b0( x, fixed_b0=None, fixed_dwi=None, fast=False ):
 3558    """
 3559    automatically generates the average b0 and dwi and outputs both;
 3560    maps dwi to b0 space at end.
 3561
 3562    x : input image
 3563
 3564    fixed_b0 : alernative reference space
 3565
 3566    fixed_dwi : alernative reference space
 3567
 3568    fast : boolean
 3569
 3570    returns:
 3571        avg_b0, avg_dwi
 3572    """
 3573    output_directory = tempfile.mkdtemp()
 3574    ofn = output_directory + "/w"
 3575    temp = segment_timeseries_by_meanvalue( x )
 3576    b0_idx = temp['highermeans']
 3577    non_b0_idx = temp['lowermeans']
 3578    if ( fixed_b0 is None and fixed_dwi is None ) or fast:
 3579        xavg = ants.slice_image( x, axis=3, idx=0 ) * 0.0
 3580        bavg = ants.slice_image( x, axis=3, idx=0 ) * 0.0
 3581        fixed_b0_use = ants.slice_image( x, axis=3, idx=b0_idx[0] )
 3582        fixed_dwi_use = ants.slice_image( x, axis=3, idx=non_b0_idx[0] )
 3583    else:
 3584        temp_b0 = ants.slice_image( x, axis=3, idx=b0_idx[0] )
 3585        temp_dwi = ants.slice_image( x, axis=3, idx=non_b0_idx[0] )
 3586        xavg = fixed_b0 * 0.0
 3587        bavg = fixed_b0 * 0.0
 3588        tempreg = ants.registration( fixed_b0, temp_b0,'antsRegistrationSyNRepro[r]')
 3589        fixed_b0_use = tempreg['warpedmovout']
 3590        fixed_dwi_use = ants.apply_transforms( fixed_b0, temp_dwi, tempreg['fwdtransforms'] )
 3591    for myidx in range(x.shape[3]):
 3592        b0 = ants.slice_image( x, axis=3, idx=myidx)
 3593        if not fast:
 3594            if not myidx in b0_idx:
 3595                xavg = xavg + ants.registration(fixed_dwi_use,b0,'antsRegistrationSyNRepro[r]',outprefix=ofn)['warpedmovout']
 3596            else:
 3597                bavg = bavg + ants.registration(fixed_b0_use,b0,'antsRegistrationSyNRepro[r]',outprefix=ofn)['warpedmovout']
 3598        else:
 3599            if not myidx in b0_idx:
 3600                xavg = xavg + b0
 3601            else:
 3602                bavg = bavg + b0
 3603    bavg = ants.iMath( bavg, 'Normalize' )
 3604    xavg = ants.iMath( xavg, 'Normalize' )
 3605    import shutil
 3606    shutil.rmtree(output_directory, ignore_errors=True )
 3607    avgb0=ants.n4_bias_field_correction(bavg)
 3608    avgdwi=ants.n4_bias_field_correction(xavg)
 3609    avgdwi=ants.registration( avgb0, avgdwi, 'antsRegistrationSyNRepro[r]' )['warpedmovout']
 3610    return avgb0, avgdwi
 3611
 3612def dti_template(
 3613    b_image_list=None,
 3614    w_image_list=None,
 3615    iterations=5,
 3616    gradient_step=0.5,
 3617    mask_csf=False,
 3618    average_both=True,
 3619    verbose=False
 3620):
 3621    """
 3622    two channel version of build_template
 3623
 3624    returns:
 3625        avg_b0, avg_dwi
 3626    """
 3627    output_directory = tempfile.mkdtemp()
 3628    mydeftx = tempfile.NamedTemporaryFile(delete=False,dir=output_directory).name
 3629    tmp = tempfile.NamedTemporaryFile(delete=False,dir=output_directory,suffix=".nii.gz")
 3630    wavgfn = tmp.name
 3631    tmp2 = tempfile.NamedTemporaryFile(delete=False,dir=output_directory)
 3632    comptx = tmp2.name
 3633    weights = np.repeat(1.0 / len(b_image_list), len(b_image_list))
 3634    weights = [x / sum(weights) for x in weights]
 3635    w_initial_template = w_image_list[0]
 3636    b_initial_template = b_image_list[0]
 3637    b_initial_template = ants.iMath(b_initial_template,"Normalize")
 3638    w_initial_template = ants.iMath(w_initial_template,"Normalize")
 3639    if mask_csf:
 3640        bcsf0 = ants.threshold_image( b_image_list[0],"Otsu",2).threshold_image(1,1).morphology("open",1).iMath("GetLargestComponent")
 3641        bcsf1 = ants.threshold_image( b_image_list[1],"Otsu",2).threshold_image(1,1).morphology("open",1).iMath("GetLargestComponent")
 3642    else:
 3643        bcsf0 = b_image_list[0] * 0 + 1
 3644        bcsf1 = b_image_list[1] * 0 + 1
 3645    bavg = b_initial_template.clone() * bcsf0
 3646    wavg = w_initial_template.clone() * bcsf0
 3647    bcsf = [ bcsf0, bcsf1 ]
 3648    for i in range(iterations):
 3649        for k in range(len(w_image_list)):
 3650            fimg=wavg
 3651            mimg=w_image_list[k] * bcsf[k]
 3652            fimg2=bavg
 3653            mimg2=b_image_list[k] * bcsf[k]
 3654            w1 = ants.registration(
 3655                fimg, mimg, type_of_transform='antsRegistrationSyNQuickRepro[s]',
 3656                    multivariate_extras= [ [ "CC", fimg2, mimg2, 1, 2 ]],
 3657                    outprefix=mydeftx,
 3658                    verbose=0 )
 3659            txname = ants.apply_transforms(wavg, wavg,
 3660                w1["fwdtransforms"], compose=comptx )
 3661            if k == 0:
 3662                txavg = ants.image_read(txname) * weights[k]
 3663                wavgnew = ants.apply_transforms( wavg,
 3664                    w_image_list[k] * bcsf[k], txname ).iMath("Normalize")
 3665                bavgnew = ants.apply_transforms( wavg,
 3666                    b_image_list[k] * bcsf[k], txname ).iMath("Normalize")
 3667            else:
 3668                txavg = txavg + ants.image_read(txname) * weights[k]
 3669                if i >= (iterations-2) and average_both:
 3670                    wavgnew = wavgnew+ants.apply_transforms( wavg,
 3671                        w_image_list[k] * bcsf[k], txname ).iMath("Normalize")
 3672                    bavgnew = bavgnew+ants.apply_transforms( wavg,
 3673                        b_image_list[k] * bcsf[k], txname ).iMath("Normalize")
 3674        if verbose:
 3675            print("iteration:",str(i),str(txavg.abs().mean()))
 3676        wscl = (-1.0) * gradient_step
 3677        txavg = txavg * wscl
 3678        ants.image_write( txavg, wavgfn )
 3679        wavg = ants.apply_transforms(wavg, wavgnew, wavgfn).iMath("Normalize")
 3680        bavg = ants.apply_transforms(bavg, bavgnew, wavgfn).iMath("Normalize")
 3681    import shutil
 3682    shutil.rmtree( output_directory, ignore_errors=True )
 3683    if verbose:
 3684        print("done")
 3685    return bavg, wavg
 3686
 3687def read_ants_transforms_to_numpy(transform_files ):
 3688    """
 3689    Read a list of ANTs transform files and convert them to a NumPy array.
 3690    The function filters out any files that are not .mat and will only  use
 3691    the first .mat in each entry of the list.
 3692
 3693    :param transform_files: List of lists of file paths to ANTs transform files.  
 3694    :return: NumPy array of the transforms.
 3695    """
 3696    extension = '.mat'
 3697    # Filter the list of lists
 3698    filtered_lists = [[string for string in sublist if string.endswith(extension)] 
 3699                    for sublist in transform_files]
 3700    transforms = []
 3701    for file in filtered_lists:
 3702        transform = ants.read_transform(file[0])
 3703        np_transform = np.array(ants.get_ants_transform_parameters(transform)[0:9])
 3704        transforms.append(np_transform)
 3705    return np.array(transforms)
 3706
 3707def t1_based_dwi_brain_extraction(
 3708    t1w_head,
 3709    t1w,
 3710    dwi,
 3711    b0_idx = None,
 3712    transform='antsRegistrationSyNRepro[r]',
 3713    deform=None,
 3714    verbose=False
 3715):
 3716    """
 3717    Map a t1-based brain extraction to b0 and return a mask and average b0
 3718
 3719    Arguments
 3720    ---------
 3721    t1w_head : an antsImage of the hole head
 3722
 3723    t1w : an antsImage probably but not necessarily T1-weighted
 3724
 3725    dwi : an antsImage holding B0 and DWI
 3726
 3727    b0_idx : the indices of the B0; if None, use segment_timeseries_by_meanvalue to guess
 3728
 3729    transform : string Rigid or other ants.registration tx type
 3730
 3731    deform : follow up transform with deformation
 3732
 3733    Returns
 3734    -------
 3735    dictionary holding the avg_b0 and its mask
 3736
 3737    Example
 3738    -------
 3739    >>> import antspymm
 3740    """
 3741    t1w_use = ants.iMath( t1w, "Normalize" )
 3742    t1bxt = ants.threshold_image( t1w_use, 0.05, 1 ).iMath("FillHoles")
 3743    if b0_idx is None:
 3744        b0_idx = segment_timeseries_by_meanvalue( dwi )['highermeans']
 3745    # first get the average b0
 3746    if len( b0_idx ) > 1:
 3747        b0_avg = ants.slice_image( dwi, axis=3, idx=b0_idx[0] ).iMath("Normalize")
 3748        for n in range(1,len(b0_idx)):
 3749            temp = ants.slice_image( dwi, axis=3, idx=b0_idx[n] )
 3750            reg = ants.registration( b0_avg, temp, 'antsRegistrationSyNRepro[r]' )
 3751            b0_avg = b0_avg + ants.iMath( reg['warpedmovout'], "Normalize")
 3752    else:
 3753        b0_avg = ants.slice_image( dwi, axis=3, idx=b0_idx[0] )
 3754    b0_avg = ants.iMath(b0_avg,"Normalize")
 3755    reg = tra_initializer( b0_avg, t1w, n_simulations=12,   verbose=verbose )
 3756    if deform is not None:
 3757        reg = ants.registration( b0_avg, t1w,
 3758            'SyNOnly',
 3759            total_sigma=0.5,
 3760            initial_transform=reg['fwdtransforms'][0],
 3761            verbose=False )
 3762    outmsk = ants.apply_transforms( b0_avg, t1bxt, reg['fwdtransforms'], interpolator='linear').threshold_image( 0.5, 1.0 )
 3763    return  {
 3764    'b0_avg':b0_avg,
 3765    'b0_mask':outmsk }
 3766
 3767def mc_denoise( x, ratio = 0.5 ):
 3768    """
 3769    ants denoising for timeseries (4D)
 3770
 3771    Arguments
 3772    ---------
 3773    x : an antsImage 4D
 3774
 3775    ratio : weight between 1 and 0 - lower weights bring result closer to initial image
 3776
 3777    Returns
 3778    -------
 3779    denoised time series
 3780
 3781    """
 3782    dwpimage = []
 3783    for myidx in range(x.shape[3]):
 3784        b0 = ants.slice_image( x, axis=3, idx=myidx)
 3785        dnzb0 = ants.denoise_image( b0, p=1,r=1,noise_model='Gaussian' )
 3786        dwpimage.append( dnzb0 * ratio + b0 * (1.0-ratio) )
 3787    return ants.list_to_ndimage( x, dwpimage )
 3788
 3789def tsnr( x, mask, indices=None ):
 3790    """
 3791    3D temporal snr image from a 4D time series image ... the matrix is normalized to range of 0,1
 3792
 3793    x: image
 3794
 3795    mask : mask
 3796
 3797    indices: indices to use
 3798
 3799    returns a 3D image
 3800    """
 3801    M = ants.timeseries_to_matrix( x, mask )
 3802    M = M - M.min()
 3803    M = M / M.max()
 3804    if indices is not None:
 3805        M=M[indices,:]
 3806    stdM = np.std(M, axis=0 )
 3807    stdM[np.isnan(stdM)] = 0
 3808    tt = round( 0.975*100 )
 3809    threshold_std = np.percentile( stdM, tt )
 3810    tsnrimage = ants.make_image( mask, stdM )
 3811    return tsnrimage
 3812
 3813def dvars( x,  mask, indices=None ):
 3814    """
 3815    dvars on a time series image ... the matrix is normalized to range of 0,1
 3816
 3817    x: image
 3818
 3819    mask : mask
 3820
 3821    indices: indices to use
 3822
 3823    returns an array
 3824    """
 3825    M = ants.timeseries_to_matrix( x, mask )
 3826    M = M - M.min()
 3827    M = M / M.max()
 3828    if indices is not None:
 3829        M=M[indices,:]
 3830    DVARS = np.zeros( M.shape[0] )
 3831    for i in range(1, M.shape[0] ):
 3832        vecdiff = M[i-1,:] - M[i,:]
 3833        DVARS[i] = np.sqrt( ( vecdiff * vecdiff ).mean() )
 3834    DVARS[0] = DVARS.mean()
 3835    return DVARS
 3836
 3837
 3838def slice_snr( x,  background_mask, foreground_mask, indices=None ):
 3839    """
 3840    slice-wise SNR on a time series image
 3841
 3842    x: image
 3843
 3844    background_mask : mask - maybe CSF or background or dilated brain mask minus original brain mask
 3845
 3846    foreground_mask : mask - maybe cortex or WM or brain mask
 3847
 3848    indices: indices to use
 3849
 3850    returns an array
 3851    """
 3852    xuse=ants.iMath(x,"Normalize")
 3853    MB = ants.timeseries_to_matrix( xuse, background_mask )
 3854    MF = ants.timeseries_to_matrix( xuse, foreground_mask )
 3855    if indices is not None:
 3856        MB=MB[indices,:]
 3857        MF=MF[indices,:]
 3858    ssnr = np.zeros( MB.shape[0] )
 3859    for i in range( MB.shape[0] ):
 3860        ssnr[i]=MF[i,:].mean()/MB[i,:].std()
 3861    ssnr[np.isnan(ssnr)] = 0
 3862    return ssnr
 3863
 3864
 3865def impute_fa( fa, md ):
 3866    """
 3867    impute bad values in dti, fa, md
 3868    """
 3869    def imputeit( x, fa ):
 3870        badfa=ants.threshold_image(fa,1,1)
 3871        if badfa.max() == 1:
 3872            temp=ants.image_clone(x)
 3873            temp[badfa==1]=0
 3874            temp=ants.iMath(temp,'GD',2)
 3875            x[ badfa==1 ]=temp[badfa==1]
 3876        return x
 3877    md=imputeit( md, fa )
 3878    fa=imputeit( ants.image_clone(fa), fa )
 3879    return fa, md
 3880
 3881def trim_dti_mask( fa, mask, param=4.0 ):
 3882    """
 3883    trim the dti mask to get rid of bright fa rim
 3884
 3885    this function erodes the famask by param amount then segments the rim into
 3886    bright and less bright parts.  the bright parts are trimmed from the mask
 3887    and the remaining edges are cleaned up a bit with closing.
 3888
 3889    param: closing radius unit is in physical space
 3890    """
 3891    spacing = ants.get_spacing(mask)
 3892    spacing_product = np.prod( spacing )
 3893    spcmin = min( spacing )
 3894    paramVox = int(np.round( param / spcmin ))
 3895    trim_mask = ants.image_clone( mask )
 3896    trim_mask = ants.iMath( trim_mask, "FillHoles" )
 3897    edgemask = trim_mask - ants.iMath( trim_mask, "ME", paramVox )
 3898    maxk=4
 3899    edgemask = ants.threshold_image( fa * edgemask, "Otsu", maxk )
 3900    edgemask = ants.threshold_image( edgemask, maxk-1, maxk )
 3901    trim_mask[edgemask >= 1 ]=0
 3902    trim_mask = ants.iMath(trim_mask,"ME",paramVox-1)
 3903    trim_mask = ants.iMath(trim_mask,'GetLargestComponent')
 3904    trim_mask = ants.iMath(trim_mask,"MD",paramVox-1)
 3905    return trim_mask
 3906
 3907
 3908
 3909def efficient_tensor_fit( gtab, fit_method, imagein, maskin, diffusion_model='DTI', 
 3910                         chunk_size=10, num_threads=1, verbose=True):
 3911    """
 3912    Efficient and optionally parallelized tensor reconstruction using DiPy.
 3913
 3914    Parameters
 3915    ----------
 3916    gtab : GradientTable
 3917        Dipy gradient table.
 3918    fit_method : str
 3919        Tensor fitting method (e.g. 'WLS', 'OLS', 'RESTORE').
 3920    imagein : ants.ANTsImage
 3921        4D diffusion-weighted image.
 3922    maskin : ants.ANTsImage
 3923        Binary brain mask image.
 3924    diffusion_model : string, optional
 3925        DTI, FreeWater, DKI.
 3926    chunk_size : int, optional
 3927        Number of slices (along z-axis) to process at once.
 3928    num_threads : int, optional
 3929        Number of threads to use (1 = single-threaded).
 3930    verbose : bool, optional
 3931        Print status updates.
 3932    
 3933    Returns
 3934    -------
 3935    tenfit : TensorFit or FreeWaterTensorFit
 3936        Fitted tensor model.
 3937    FA : ants.ANTsImage
 3938        Fractional anisotropy image.
 3939    MD : ants.ANTsImage
 3940        Mean diffusivity image.
 3941    RGB : ants.ANTsImage
 3942        RGB FA map.
 3943    """
 3944    assert imagein.dimension == 4, "Input image must be 4D"
 3945
 3946    import ants
 3947    import numpy as np
 3948    import dipy.reconst.dti as dti
 3949    import dipy.reconst.fwdti as fwdti
 3950    from dipy.reconst.dti import fractional_anisotropy
 3951    from dipy.reconst.dti import color_fa
 3952    from concurrent.futures import ThreadPoolExecutor, as_completed
 3953
 3954    img_data = imagein.numpy()
 3955    mask = maskin.numpy().astype(bool)
 3956    X, Y, Z, N = img_data.shape
 3957    if verbose:
 3958        print(f"Input shape: {img_data.shape}, Processing in chunks of {chunk_size} slices.")
 3959
 3960    model = fwdti.FreeWaterTensorModel(gtab) if diffusion_model == 'FreeWater' else dti.TensorModel(gtab, fit_method=fit_method)
 3961
 3962    def process_chunk(z_start):
 3963        z_end = min(Z, z_start + chunk_size)
 3964        local_data = img_data[:, :, z_start:z_end, :]
 3965        local_mask = mask[:, :, z_start:z_end]
 3966        masked_data = local_data * local_mask[..., None]
 3967        masked_data = np.nan_to_num(masked_data, nan=0)
 3968        fit = model.fit(masked_data)
 3969        FA_chunk = fractional_anisotropy(fit.evals)
 3970        FA_chunk[np.isnan(FA_chunk)] = 1
 3971        FA_chunk = np.clip(FA_chunk, 0, 1)
 3972        MD_chunk = dti.mean_diffusivity(fit.evals)
 3973        RGB_chunk = color_fa(FA_chunk, fit.evecs)
 3974        return z_start, z_end, FA_chunk, MD_chunk, RGB_chunk
 3975
 3976    FA_vol = np.zeros((X, Y, Z), dtype=np.float32)
 3977    MD_vol = np.zeros((X, Y, Z), dtype=np.float32)
 3978    RGB_vol = np.zeros((X, Y, Z, 3), dtype=np.float32)
 3979
 3980    chunks = range(0, Z, chunk_size)
 3981    if num_threads > 1:
 3982        with ThreadPoolExecutor(max_workers=num_threads) as executor:
 3983            futures = {executor.submit(process_chunk, z): z for z in chunks}
 3984            for f in as_completed(futures):
 3985                z_start, z_end, FA_chunk, MD_chunk, RGB_chunk = f.result()
 3986                FA_vol[:, :, z_start:z_end] = FA_chunk
 3987                MD_vol[:, :, z_start:z_end] = MD_chunk
 3988                RGB_vol[:, :, z_start:z_end, :] = RGB_chunk
 3989    else:
 3990        for z in chunks:
 3991            z_start, z_end, FA_chunk, MD_chunk, RGB_chunk = process_chunk(z)
 3992            FA_vol[:, :, z_start:z_end] = FA_chunk
 3993            MD_vol[:, :, z_start:z_end] = MD_chunk
 3994            RGB_vol[:, :, z_start:z_end, :] = RGB_chunk
 3995
 3996    b0 = ants.slice_image(imagein, axis=3, idx=0)
 3997    FA = ants.copy_image_info(b0, ants.from_numpy(FA_vol))
 3998    MD = ants.copy_image_info(b0, ants.from_numpy(MD_vol))
 3999    RGB_channels = [ants.copy_image_info(b0, ants.from_numpy(RGB_vol[..., i])) for i in range(3)]
 4000    RGB = ants.merge_channels(RGB_channels)
 4001
 4002    return model.fit(img_data * mask[..., None]), FA, MD, RGB
 4003
 4004
 4005
 4006def efficient_dwi_fit(gtab, diffusion_model, imagein, maskin,
 4007                      model_params=None, bvals_to_use=None,
 4008                      chunk_size=1024, num_threads=1, verbose=True):
 4009    """
 4010    Efficient and optionally parallelized diffusion model reconstruction using DiPy.
 4011
 4012    Parameters
 4013    ----------
 4014    gtab : GradientTable
 4015        DiPy gradient table.
 4016    diffusion_model : str
 4017        One of ['DTI', 'FreeWater', 'DKI'].
 4018    imagein : ants.ANTsImage
 4019        4D diffusion-weighted image.
 4020    maskin : ants.ANTsImage
 4021        Binary brain mask image.
 4022    model_params : dict, optional
 4023        Additional parameters passed to model constructors.
 4024    bvals_to_use : list of int, optional
 4025        Subset of b-values to use for the fit (e.g., [0, 1000, 2000]).
 4026    chunk_size : int, optional
 4027        Maximum number of voxels per chunk (default 1024).
 4028    num_threads : int, optional
 4029        Number of parallel threads.
 4030    verbose : bool, optional
 4031        Whether to print status messages.
 4032
 4033    Returns
 4034    -------
 4035    fit : dipy ModelFit
 4036        The fitted model object.
 4037    FA : ants.ANTsImage or None
 4038        Fractional anisotropy image (if applicable).
 4039    MD : ants.ANTsImage or None
 4040        Mean diffusivity image (if applicable).
 4041    RGB : ants.ANTsImage or None
 4042        Color FA image (if applicable).
 4043    """
 4044    import ants
 4045    import numpy as np
 4046    import dipy.reconst.dti as dti
 4047    import dipy.reconst.fwdti as fwdti
 4048    import dipy.reconst.dki as dki
 4049    from dipy.core.gradients import gradient_table
 4050    from dipy.reconst.dti import fractional_anisotropy, color_fa, mean_diffusivity
 4051    from concurrent.futures import ThreadPoolExecutor, as_completed
 4052
 4053    assert imagein.dimension == 4, "Input image must be 4D"
 4054    model_params = model_params or {}
 4055
 4056    img_data = imagein.numpy()
 4057    mask = maskin.numpy().astype(bool)
 4058    X, Y, Z, N = img_data.shape
 4059    inplane_size = X * Y
 4060
 4061    # Convert chunk_size from voxel count to number of slices
 4062    slices_per_chunk = max(1, chunk_size // inplane_size)
 4063
 4064    if verbose:
 4065        print(f"[INFO] Image shape: {img_data.shape}")
 4066        print(f"[INFO] Using model: {diffusion_model}")
 4067        print(f"[INFO] Max voxels per chunk: {chunk_size} (→ {slices_per_chunk} slices) | Threads: {num_threads}")
 4068
 4069    if bvals_to_use is not None:
 4070        bvals_to_use = set(bvals_to_use)
 4071        sel = np.isin(gtab.bvals, list(bvals_to_use))
 4072        img_data = img_data[..., sel]
 4073        gtab = gradient_table(gtab.bvals[sel], bvecs=gtab.bvecs[sel])
 4074        if verbose:
 4075            print(f"[INFO] Selected b-values: {sorted(bvals_to_use)}")
 4076            print(f"[INFO] Selected volumes: {sel.sum()} / {N}")
 4077
 4078    def get_model(name, gtab, **params):
 4079        if name == 'DTI':
 4080            return dti.TensorModel(gtab, **params)
 4081        elif name == 'FreeWater':
 4082            return fwdti.FreeWaterTensorModel(gtab)
 4083        elif name == 'DKI':
 4084            return dki.DiffusionKurtosisModel(gtab, **params)
 4085        else:
 4086            raise ValueError(f"Unsupported model: {name}")
 4087
 4088    model = get_model(diffusion_model, gtab, **model_params)
 4089
 4090    FA_vol = np.zeros((X, Y, Z), dtype=np.float32)
 4091    MD_vol = np.zeros((X, Y, Z), dtype=np.float32)
 4092    RGB_vol = np.zeros((X, Y, Z, 3), dtype=np.float32)
 4093    has_tensor_metrics = diffusion_model in ['DTI', 'FreeWater']
 4094
 4095    def process_chunk(z_start):
 4096        z_end = min(Z, z_start + slices_per_chunk)
 4097        local_data = img_data[:, :, z_start:z_end, :]
 4098        local_mask = mask[:, :, z_start:z_end]
 4099        masked_data = local_data * local_mask[..., None]
 4100        masked_data = np.nan_to_num(masked_data, nan=0)
 4101        fit = model.fit(masked_data)
 4102        if has_tensor_metrics and hasattr(fit, 'evals') and hasattr(fit, 'evecs'):
 4103            FA = fractional_anisotropy(fit.evals)
 4104            FA[np.isnan(FA)] = 1
 4105            FA = np.clip(FA, 0, 1)
 4106            MD = mean_diffusivity(fit.evals)
 4107            RGB = color_fa(FA, fit.evecs)
 4108            return z_start, z_end, FA, MD, RGB
 4109        return z_start, z_end, None, None, None
 4110
 4111    chunks = range(0, Z, slices_per_chunk)
 4112    if num_threads > 1:
 4113        with ThreadPoolExecutor(max_workers=num_threads) as executor:
 4114            futures = {executor.submit(process_chunk, z): z for z in chunks}
 4115            for f in as_completed(futures):
 4116                z_start, z_end, FA, MD, RGB = f.result()
 4117                if FA is not None:
 4118                    FA_vol[:, :, z_start:z_end] = FA
 4119                    MD_vol[:, :, z_start:z_end] = MD
 4120                    RGB_vol[:, :, z_start:z_end, :] = RGB
 4121    else:
 4122        for z in chunks:
 4123            z_start, z_end, FA, MD, RGB = process_chunk(z)
 4124            if FA is not None:
 4125                FA_vol[:, :, z_start:z_end] = FA
 4126                MD_vol[:, :, z_start:z_end] = MD
 4127                RGB_vol[:, :, z_start:z_end, :] = RGB
 4128
 4129    b0 = ants.slice_image(imagein, axis=3, idx=0)
 4130    FA_img = ants.copy_image_info(b0, ants.from_numpy(FA_vol)) if has_tensor_metrics else None
 4131    MD_img = ants.copy_image_info(b0, ants.from_numpy(MD_vol)) if has_tensor_metrics else None
 4132    RGB_img = (ants.merge_channels([
 4133        ants.copy_image_info(b0, ants.from_numpy(RGB_vol[..., i])) for i in range(3)
 4134    ]) if has_tensor_metrics else None)
 4135
 4136    full_fit = model.fit(img_data * mask[..., None])
 4137    return full_fit, FA_img, MD_img, RGB_img
 4138
 4139
 4140def efficient_dwi_fit_voxelwise(imagein, maskin, bvals, bvecs_5d, model_params=None,
 4141                                bvals_to_use=None, num_threads=1, verbose=True):
 4142    """
 4143    Voxel-wise diffusion model fitting with individual b-vectors per voxel.
 4144
 4145    Parameters
 4146    ----------
 4147    imagein : ants.ANTsImage
 4148        4D DWI image (X, Y, Z, N).
 4149    maskin : ants.ANTsImage
 4150        3D binary mask.
 4151    bvals : (N,) array-like
 4152        Common b-values across volumes.
 4153    bvecs_5d : (X, Y, Z, N, 3) ndarray
 4154        Voxel-specific b-vectors.
 4155    model_params : dict
 4156        Extra arguments for model.
 4157    bvals_to_use : list[int]
 4158        Subset of b-values to include.
 4159    num_threads : int
 4160        Number of threads to use.
 4161    verbose : bool
 4162        Whether to print status.
 4163
 4164    Returns
 4165    -------
 4166    FA_img : ants.ANTsImage
 4167        Fractional anisotropy.
 4168    MD_img : ants.ANTsImage
 4169        Mean diffusivity.
 4170    RGB_img : ants.ANTsImage
 4171        RGB FA image.
 4172    """
 4173    import numpy as np
 4174    import ants
 4175    import dipy.reconst.dti as dti
 4176    from dipy.core.gradients import gradient_table
 4177    from dipy.reconst.dti import fractional_anisotropy, color_fa, mean_diffusivity
 4178    from concurrent.futures import ThreadPoolExecutor
 4179    from tqdm import tqdm
 4180
 4181    model_params = model_params or {}
 4182    img = imagein.numpy()
 4183    mask = maskin.numpy().astype(bool)
 4184    X, Y, Z, N = img.shape
 4185
 4186    if bvals_to_use is not None:
 4187        sel = np.isin(bvals, bvals_to_use)
 4188        img = img[..., sel]
 4189        bvals = bvals[sel]
 4190        bvecs_5d = bvecs_5d[..., sel, :]
 4191
 4192    FA = np.zeros((X, Y, Z), dtype=np.float32)
 4193    MD = np.zeros((X, Y, Z), dtype=np.float32)
 4194    RGB = np.zeros((X, Y, Z, 3), dtype=np.float32)
 4195
 4196    def fit_voxel(ix, iy, iz):
 4197        if not mask[ix, iy, iz]:
 4198            return
 4199        sig = img[ix, iy, iz, :]
 4200        if np.all(sig == 0):
 4201            return
 4202        bv = bvecs_5d[ix, iy, iz, :, :]
 4203        gtab = gradient_table(bvals, bv)
 4204        try:
 4205            model = dti.TensorModel(gtab, **model_params)
 4206            fit = model.fit(sig)
 4207            evals = fit.evals
 4208            evecs = fit.evecs
 4209            FA[ix, iy, iz] = fractional_anisotropy(evals)
 4210            MD[ix, iy, iz] = mean_diffusivity(evals)
 4211            RGB[ix, iy, iz, :] = color_fa(FA[ix, iy, iz], evecs)
 4212        except Exception as e:
 4213            if verbose:
 4214                print(f"Voxel ({ix},{iy},{iz}) fit failed: {e}")
 4215
 4216    coords = np.argwhere(mask)
 4217    if verbose:
 4218        print(f"[INFO] Fitting {len(coords)} voxels using {num_threads} threads...")
 4219
 4220    if num_threads > 1:
 4221        with ThreadPoolExecutor(max_workers=num_threads) as executor:
 4222            list(tqdm(executor.map(lambda c: fit_voxel(*c), coords), total=len(coords)))
 4223    else:
 4224        for c in tqdm(coords):
 4225            fit_voxel(*c)
 4226
 4227    ref = ants.slice_image(imagein, axis=3, idx=0)
 4228    return (
 4229        ants.copy_image_info(ref, ants.from_numpy(FA)),
 4230        ants.copy_image_info(ref, ants.from_numpy(MD)),
 4231        ants.merge_channels([ants.copy_image_info(ref, ants.from_numpy(RGB[..., i])) for i in range(3)])
 4232    )
 4233
 4234
 4235def generate_voxelwise_bvecs(global_bvecs, voxel_rotations, transpose=False):
 4236    """
 4237    Generate voxel-wise b-vectors from a global bvec and voxel-wise rotation field.
 4238
 4239    Parameters
 4240    ----------
 4241    global_bvecs : ndarray of shape (N, 3)
 4242        Global diffusion gradient directions.
 4243    voxel_rotations : ndarray of shape (X, Y, Z, 3, 3)
 4244        3x3 rotation matrix for each voxel (can come from Jacobian of deformation field).
 4245    transpose : bool, optional
 4246        If True, transpose the rotation matrices before applying them to the b-vectors.
 4247
 4248
 4249    Returns
 4250    -------
 4251    bvecs_5d : ndarray of shape (X, Y, Z, N, 3)
 4252        Voxel-specific b-vectors.
 4253    """
 4254    X, Y, Z, _, _ = voxel_rotations.shape
 4255    N = global_bvecs.shape[0]
 4256    bvecs_5d = np.zeros((X, Y, Z, N, 3), dtype=np.float32)
 4257
 4258    for n in range(N):
 4259        bvec = global_bvecs[n]
 4260        for i in range(X):
 4261            for j in range(Y):
 4262                for k in range(Z):
 4263                    R = voxel_rotations[i, j, k]
 4264                    if transpose:
 4265                        R = R.T  # Use transpose if needed
 4266                    bvecs_5d[i, j, k, n, :] = R @ bvec
 4267    return bvecs_5d
 4268
 4269def dipy_dti_recon(
 4270    image,
 4271    bvalsfn,
 4272    bvecsfn,
 4273    mask = None,
 4274    b0_idx = None,
 4275    mask_dilation = 2,
 4276    mask_closing = 5,
 4277    fit_method='WLS',
 4278    trim_the_mask=2.0,
 4279    diffusion_model='DTI',
 4280    verbose=False ):
 4281    """
 4282    DiPy DTI reconstruction - building on the DiPy basic DTI example
 4283
 4284    Arguments
 4285    ---------
 4286    image : an antsImage holding B0 and DWI
 4287
 4288    bvalsfn : bvalues  obtained by dipy read_bvals_bvecs or the values themselves
 4289
 4290    bvecsfn : bvectors obtained by dipy read_bvals_bvecs or the values themselves
 4291
 4292    mask : brain mask for the DWI/DTI reconstruction; if it is not in the same
 4293        space as the image, we will resample directly to the image space.  This
 4294        could lead to problems if the inputs are really incorrect.
 4295
 4296    b0_idx : the indices of the B0; if None, use segment_timeseries_by_bvalue
 4297
 4298    mask_dilation : integer zero or more dilates the brain mask
 4299
 4300    mask_closing : integer zero or more closes the brain mask
 4301
 4302    fit_method : string one of WLS LS NLLS or restore - see import dipy.reconst.dti as dti and help(dti.TensorModel) ... if None, will not reconstruct DTI.
 4303
 4304    trim_the_mask : float >=0 post-hoc method for trimming the mask
 4305
 4306    diffusion_model : string
 4307        DTI, FreeWater, DKI
 4308
 4309    verbose : boolean
 4310
 4311    Returns
 4312    -------
 4313    dictionary holding the tensorfit, MD, FA and RGB images and motion parameters (optional)
 4314
 4315    NOTE -- see dipy reorient_bvecs(gtab, affines, atol=1e-2)
 4316
 4317    NOTE -- if the bvec.shape[0] is smaller than the image.shape[3], we neglect
 4318        the tailing image volumes.
 4319
 4320    Example
 4321    -------
 4322    >>> import antspymm
 4323    """
 4324
 4325    import dipy.reconst.fwdti as fwdti
 4326
 4327    if isinstance(bvecsfn, str):
 4328        bvals, bvecs = read_bvals_bvecs( bvalsfn , bvecsfn   )
 4329    else: # assume we already read them
 4330        bvals = bvalsfn.copy()
 4331        bvecs = bvecsfn.copy()
 4332
 4333    if bvals.max() < 1.0:
 4334        raise ValueError("DTI recon error: maximum bvalues are too small.")
 4335
 4336    b0_idx = segment_timeseries_by_bvalue( bvals )['lowbvals']
 4337
 4338    b0 = ants.slice_image( image, axis=3, idx=b0_idx[0] )
 4339    bxtmod='bold'
 4340    bxtmod='t2'
 4341    constant_mask=False
 4342    if verbose:
 4343        print( np.unique( bvals ), flush=True )
 4344    if mask is not None:
 4345        if verbose:
 4346            print("use set bxt in dipy_dti_recon", flush=True)
 4347        constant_mask=True
 4348        mask = ants.resample_image_to_target( mask, b0, interp_type='nearestNeighbor')
 4349    else:
 4350        if verbose:
 4351            print("use deep learning bxt in dipy_dti_recon")
 4352        mask = antspynet.brain_extraction( b0, bxtmod ).threshold_image(0.5,1).iMath("GetLargestComponent").morphology("close",2).iMath("FillHoles")
 4353    if mask_closing > 0 and not constant_mask :
 4354        mask = ants.morphology( mask, "close", mask_closing ) # good
 4355    maskdil = ants.iMath( mask, "MD", mask_dilation )
 4356
 4357    if verbose:
 4358        print("recon dti.TensorModel",flush=True)
 4359
 4360    bvecs = repair_bvecs( bvecs )
 4361    gtab = gradient_table(bvals, bvecs=bvecs, atol=2.0 )
 4362    mynt=1
 4363    threads_env = os.environ.get("ITK_GLOBAL_DEFAULT_NUMBER_OF_THREADS")
 4364    if threads_env is not None:
 4365        mynt = int(threads_env)
 4366    tenfit, FA, MD1, RGB = efficient_dwi_fit( gtab, diffusion_model, image, maskdil,
 4367                                             num_threads=mynt )
 4368    if verbose:
 4369        print("recon dti.TensorModel done",flush=True)
 4370
 4371    # change the brain mask based on high FA values
 4372    if trim_the_mask > 0 and fit_method is not None:
 4373        mask = trim_dti_mask( FA, mask, trim_the_mask )
 4374        tenfit, FA, MD1, RGB = efficient_dwi_fit( gtab, diffusion_model, image, maskdil,
 4375                                             num_threads=mynt )
 4376
 4377    return {
 4378        'tensormodel' : tenfit,
 4379        'MD' : MD1 ,
 4380        'FA' : FA ,
 4381        'RGB' : RGB,
 4382        'dwi_mask':mask,
 4383        'bvals':bvals,
 4384        'bvecs':bvecs
 4385        }
 4386
 4387
 4388def concat_dewarp(
 4389        refimg,
 4390        originalDWI,
 4391        physSpaceDWI,
 4392        dwpTx,
 4393        motion_parameters,
 4394        motion_correct=True,
 4395        verbose=False ):
 4396    """
 4397    Apply concatentated motion correction and dewarping transforms to timeseries image.
 4398
 4399    Arguments
 4400    ---------
 4401
 4402    refimg : an antsImage defining the reference domain (3D)
 4403
 4404    originalDWI : the antsImage in original (not interpolated space) (4D)
 4405
 4406    physSpaceDWI : ants antsImage defining the physical space of the mapping (4D)
 4407
 4408    dwpTx : dewarping transform
 4409
 4410    motion_parameters : previously computed list of motion parameters
 4411
 4412    motion_correct : boolean
 4413
 4414    verbose : boolean
 4415
 4416    """
 4417    # apply the dewarping tx to the original dwi and reconstruct again
 4418    # NOTE: refimg must be in the same space for this to work correctly
 4419    # due to the use of ants.list_to_ndimage( originalDWI, dwpimage )
 4420    dwpimage = []
 4421    for myidx in range(originalDWI.shape[3]):
 4422        b0 = ants.slice_image( originalDWI, axis=3, idx=myidx)
 4423        concatx = dwpTx.copy()
 4424        if motion_correct:
 4425            concatx = concatx + motion_parameters[myidx]
 4426        if verbose and myidx == 0:
 4427            print("dwp parameters")
 4428            print( dwpTx )
 4429            print("Motion parameters")
 4430            print( motion_parameters[myidx] )
 4431            print("concat parameters")
 4432            print(concatx)
 4433        warpedb0 = ants.apply_transforms( refimg, b0, concatx,
 4434            interpolator='nearestNeighbor' )
 4435        dwpimage.append( warpedb0 )
 4436    return ants.list_to_ndimage( physSpaceDWI, dwpimage )
 4437
 4438
 4439def joint_dti_recon(
 4440    img_LR,
 4441    bval_LR,
 4442    bvec_LR,
 4443    jhu_atlas,
 4444    jhu_labels,
 4445    reference_B0,
 4446    reference_DWI,
 4447    srmodel = None,
 4448    img_RL = None,
 4449    bval_RL = None,
 4450    bvec_RL = None,
 4451    t1w = None,
 4452    brain_mask = None,
 4453    motion_correct = None,
 4454    dewarp_modality = 'FA',
 4455    denoise=False,
 4456    fit_method='WLS',
 4457    impute = False,
 4458    censor = True,
 4459    diffusion_model = 'DTI',
 4460    verbose = False ):
 4461    """
 4462    1. pass in subject data and 1mm JHU atlas/labels
 4463    2. perform initial LR, RL reconstruction (2nd is optional) and motion correction (optional)
 4464    3. dewarp the images using dewarp_modality or T1w
 4465    4. apply dewarping to the original data
 4466        ===> may want to apply SR at this step
 4467    5. reconstruct DTI again
 4468    6. label images and do registration
 4469    7. return relevant outputs
 4470
 4471    NOTE: RL images are optional; should pass t1w in this case.
 4472
 4473    Arguments
 4474    ---------
 4475
 4476    img_LR : an antsImage holding B0 and DWI LR acquisition
 4477
 4478    bval_LR : bvalue filename LR
 4479
 4480    bvec_LR : bvector filename LR
 4481
 4482    jhu_atlas : atlas FA image
 4483
 4484    jhu_labels : atlas labels
 4485
 4486    reference_B0 : the "target" B0 image space
 4487
 4488    reference_DWI : the "target" DW image space
 4489
 4490    srmodel : optional h5 (tensorflow) model
 4491
 4492    img_RL : an antsImage holding B0 and DWI RL acquisition
 4493
 4494    bval_RL : bvalue filename RL
 4495
 4496    bvec_RL : bvector filename RL
 4497
 4498    t1w : antsimage t1w neuroimage (brain-extracted)
 4499
 4500    brain_mask : mask for the DWI - just 3D - provided brain mask should be in reference_B0 space
 4501
 4502    motion_correct : None Rigid or SyN
 4503
 4504    dewarp_modality : string average_dwi, average_b0, MD or FA
 4505
 4506    denoise: boolean
 4507
 4508    fit_method : string one of WLS LS NLLS or restore - see import dipy.reconst.dti as dti and help(dti.TensorModel)
 4509
 4510    impute : boolean
 4511
 4512    censor : boolean
 4513
 4514    diffusion_model : string
 4515        DTI, FreeWater, DKI
 4516
 4517    verbose : boolean
 4518
 4519    Returns
 4520    -------
 4521    dictionary holding the mean_fa, its summary statistics via JHU labels,
 4522        the JHU registration, the JHU labels, the dewarping dictionary and the
 4523        dti reconstruction dictionaries.
 4524
 4525    Example
 4526    -------
 4527    >>> import antspymm
 4528    """
 4529
 4530    if verbose:
 4531        print("Recon DTI on OR images ...")
 4532
 4533    def fix_dwi_shape( img, bvalfn, bvecfn ):
 4534        if isinstance(bvecfn, str):
 4535            bvals, bvecs = read_bvals_bvecs( bvalfn , bvecfn   )
 4536        else:
 4537            bvals = bvalfn
 4538            bvecs = bvecfn
 4539        if bvecs.shape[0] < img.shape[3]:
 4540            imgout = ants.from_numpy( img[:,:,:,0:bvecs.shape[0]] )
 4541            imgout = ants.copy_image_info( img, imgout )
 4542            return( imgout )
 4543        else:
 4544            return( img )
 4545
 4546    img_LR = fix_dwi_shape( img_LR, bval_LR, bvec_LR )
 4547    if denoise :
 4548        img_LR = mc_denoise( img_LR )
 4549    if img_RL is not None:
 4550        img_RL = fix_dwi_shape( img_RL, bval_RL, bvec_RL )
 4551        if denoise :
 4552            img_RL = mc_denoise( img_RL )
 4553
 4554    brainmaske = None
 4555    if brain_mask is not None:
 4556        maskInRightSpace = ants.image_physical_space_consistency( brain_mask, reference_B0 )
 4557        if not maskInRightSpace :
 4558            raise ValueError('not maskInRightSpace ... provided brain mask should be in reference_B0 space')
 4559        brainmaske = ants.iMath( brain_mask, "ME", 2 )
 4560
 4561    if img_RL is not None :
 4562        if verbose:
 4563            print("img_RL correction")
 4564        reg_RL = dti_reg(
 4565            img_RL,
 4566            avg_b0=reference_B0,
 4567            avg_dwi=reference_DWI,
 4568            bvals=bval_RL,
 4569            bvecs=bvec_RL,
 4570            type_of_transform=motion_correct,
 4571            brain_mask_eroded=brainmaske,
 4572            verbose=True )
 4573    else:
 4574        reg_RL=None
 4575
 4576
 4577    if verbose:
 4578        print("img_LR correction")
 4579    reg_LR = dti_reg(
 4580            img_LR,
 4581            avg_b0=reference_B0,
 4582            avg_dwi=reference_DWI,
 4583            bvals=bval_LR,
 4584            bvecs=bvec_LR,
 4585            type_of_transform=motion_correct,
 4586            brain_mask_eroded=brainmaske,
 4587            verbose=True )
 4588
 4589    ts_LR_avg = None
 4590    ts_RL_avg = None
 4591    reg_its = [100,50,10]
 4592    img_LRdwp = ants.image_clone( reg_LR[ 'motion_corrected' ] )
 4593    if img_RL is not None:
 4594        img_RLdwp = ants.image_clone( reg_RL[ 'motion_corrected' ] )
 4595        if srmodel is not None:
 4596            if verbose:
 4597                print("convert img_RL_dwp to img_RL_dwp_SR")
 4598            img_RLdwp = super_res_mcimage( img_RLdwp, srmodel, isotropic=True,
 4599                        verbose=verbose )
 4600    if srmodel is not None:
 4601        reg_its = [100] + reg_its
 4602        if verbose:
 4603            print("convert img_LR_dwp to img_LR_dwp_SR")
 4604        img_LRdwp = super_res_mcimage( img_LRdwp, srmodel, isotropic=True,
 4605                verbose=verbose )
 4606    if verbose:
 4607        print("recon after distortion correction", flush=True)
 4608
 4609    if impute:
 4610        print("impute begin", flush=True)
 4611        img_LRdwp=impute_dwi( img_LRdwp, verbose=True )
 4612        print("impute done", flush=True)
 4613    elif censor:
 4614        print("censor begin", flush=True)
 4615        img_LRdwp, reg_LR['bvals'], reg_LR['bvecs'] = censor_dwi( img_LRdwp, reg_LR['bvals'], reg_LR['bvecs'], verbose=True )
 4616        print("censor done", flush=True)
 4617    if impute and img_RL is not None:
 4618        img_RLdwp=impute_dwi( img_RLdwp, verbose=True )
 4619    elif censor and img_RL is not None:
 4620        img_RLdwp, reg_RL['bvals'], reg_RL['bvecs'] = censor_dwi( img_RLdwp, reg_RL['bvals'], reg_RL['bvecs'], verbose=True )
 4621
 4622    if img_RL is not None:
 4623        img_LRdwp, bval_LR, bvec_LR = merge_dwi_data(
 4624            img_LRdwp, reg_LR['bvals'], reg_LR['bvecs'],
 4625            img_RLdwp, reg_RL['bvals'], reg_RL['bvecs']
 4626        )
 4627    else:
 4628        bval_LR=reg_LR['bvals']
 4629        bvec_LR=reg_LR['bvecs']
 4630
 4631    if verbose:
 4632        print("final recon", flush=True)
 4633        print(img_LRdwp)
 4634
 4635    recon_LR_dewarp = dipy_dti_recon(
 4636            img_LRdwp, bval_LR, bvec_LR,
 4637            mask = brain_mask,
 4638            fit_method=fit_method,
 4639            mask_dilation=0, diffusion_model=diffusion_model, verbose=True )
 4640    if verbose:
 4641        print("recon done", flush=True)
 4642
 4643    if img_RL is not None:
 4644        fdjoin = [ reg_LR['FD'],
 4645                   reg_RL['FD'] ]
 4646        framewise_displacement=np.concatenate( fdjoin )
 4647    else:
 4648        framewise_displacement=reg_LR['FD']
 4649
 4650    motion_count = ( framewise_displacement > 1.5  ).sum()
 4651    reconFA = recon_LR_dewarp['FA']
 4652    reconMD = recon_LR_dewarp['MD']
 4653
 4654    if verbose:
 4655        print("JHU reg",flush=True)
 4656
 4657    OR_FA2JHUreg = ants.registration( reconFA, jhu_atlas,
 4658        type_of_transform = 'antsRegistrationSyNQuickRepro[s]', 
 4659        reg_iterations=reg_its, verbose=False )
 4660    OR_FA_jhulabels = ants.apply_transforms( reconFA, jhu_labels,
 4661        OR_FA2JHUreg['fwdtransforms'], interpolator='genericLabel')
 4662
 4663    df_FA_JHU_ORRL = antspyt1w.map_intensity_to_dataframe(
 4664        'FA_JHU_labels_edited',
 4665        reconFA,
 4666        OR_FA_jhulabels)
 4667    df_FA_JHU_ORRL_bfwide = antspyt1w.merge_hierarchical_csvs_to_wide_format(
 4668            {'df_FA_JHU_ORRL' : df_FA_JHU_ORRL},
 4669            col_names = ['Mean'] )
 4670
 4671    df_MD_JHU_ORRL = antspyt1w.map_intensity_to_dataframe(
 4672        'MD_JHU_labels_edited',
 4673        reconMD,
 4674        OR_FA_jhulabels)
 4675    df_MD_JHU_ORRL_bfwide = antspyt1w.merge_hierarchical_csvs_to_wide_format(
 4676            {'df_MD_JHU_ORRL' : df_MD_JHU_ORRL},
 4677            col_names = ['Mean'] )
 4678
 4679    temp = segment_timeseries_by_meanvalue( img_LRdwp )
 4680    b0_idx = temp['highermeans']
 4681    non_b0_idx = temp['lowermeans']
 4682
 4683    nonbrainmask = ants.iMath( recon_LR_dewarp['dwi_mask'], "MD",2) - recon_LR_dewarp['dwi_mask']
 4684    fgmask = ants.threshold_image( reconFA, 0.5 , 1.0).iMath("GetLargestComponent")
 4685    bgmask = ants.threshold_image( reconFA, 1e-4 , 0.1)
 4686    fa_SNR = 0.0
 4687    fa_SNR = mask_snr( reconFA, bgmask, fgmask, bias_correct=False )
 4688    fa_evr = antspyt1w.patch_eigenvalue_ratio( reconFA, 512, [16,16,16], evdepth = 0.9, mask=recon_LR_dewarp['dwi_mask'] )
 4689
 4690    dti_itself = get_dti( reconFA, recon_LR_dewarp['tensormodel'], return_image=True )
 4691    return convert_np_in_dict( {
 4692        'dti': dti_itself,
 4693        'recon_fa':reconFA,
 4694        'recon_fa_summary':df_FA_JHU_ORRL_bfwide,
 4695        'recon_md':reconMD,
 4696        'recon_md_summary':df_MD_JHU_ORRL_bfwide,
 4697        'jhu_labels':OR_FA_jhulabels,
 4698        'jhu_registration':OR_FA2JHUreg,
 4699        'reg_LR':reg_LR,
 4700        'reg_RL':reg_RL,
 4701        'dtrecon_LR_dewarp':recon_LR_dewarp,
 4702        'dwi_LR_dewarped':img_LRdwp,
 4703        'bval_unique_count': len(np.unique(bval_LR)),
 4704        'bval_LR':bval_LR,
 4705        'bvec_LR':bvec_LR,
 4706        'bval_RL':bval_RL,
 4707        'bvec_RL':bvec_RL,
 4708        'b0avg': reference_B0,
 4709        'dwiavg': reference_DWI,
 4710        'framewise_displacement':framewise_displacement,
 4711        'high_motion_count': motion_count,
 4712        'tsnr_b0': tsnr( img_LRdwp, recon_LR_dewarp['dwi_mask'], b0_idx),
 4713        'tsnr_dwi': tsnr( img_LRdwp, recon_LR_dewarp['dwi_mask'], non_b0_idx),
 4714        'dvars_b0': dvars( img_LRdwp, recon_LR_dewarp['dwi_mask'], b0_idx),
 4715        'dvars_dwi': dvars( img_LRdwp, recon_LR_dewarp['dwi_mask'], non_b0_idx),
 4716        'ssnr_b0': slice_snr( img_LRdwp, bgmask , fgmask, b0_idx),
 4717        'ssnr_dwi': slice_snr( img_LRdwp, bgmask, fgmask, non_b0_idx),
 4718        'fa_evr': fa_evr,
 4719        'fa_SNR': fa_SNR
 4720    } )
 4721
 4722
 4723def middle_slice_snr( x, background_dilation=5 ):
 4724    """
 4725
 4726    Estimate signal to noise ratio (SNR) in 2D mid image from a 3D image.
 4727    Estimates noise from a background mask which is a
 4728    dilation of the foreground mask minus the foreground mask.
 4729    Actually estimates the reciprocal of the coefficient of variation.
 4730
 4731    Arguments
 4732    ---------
 4733
 4734    x : an antsImage
 4735
 4736    background_dilation : integer - amount to dilate foreground mask
 4737
 4738    """
 4739    xshp = x.shape
 4740    xmidslice = ants.slice_image( x, 2, int( xshp[2]/2 )  )
 4741    xmidslice = ants.iMath( xmidslice - xmidslice.min(), "Normalize" )
 4742    xmidslice = ants.n3_bias_field_correction( xmidslice )
 4743    xmidslice = ants.n3_bias_field_correction( xmidslice )
 4744    xmidslicemask = ants.threshold_image( xmidslice, "Otsu", 1 ).morphology("close",2).iMath("FillHoles")
 4745    xbkgmask = ants.iMath( xmidslicemask, "MD", background_dilation ) - xmidslicemask
 4746    signal = (xmidslice[ xmidslicemask == 1] ).mean()
 4747    noise = (xmidslice[ xbkgmask == 1] ).std()
 4748    return signal / noise
 4749
 4750def foreground_background_snr( x, background_dilation=10,
 4751        erode_foreground=False):
 4752    """
 4753
 4754    Estimate signal to noise ratio (SNR) in an image.
 4755    Estimates noise from a background mask which is a
 4756    dilation of the foreground mask minus the foreground mask.
 4757    Actually estimates the reciprocal of the coefficient of variation.
 4758
 4759    Arguments
 4760    ---------
 4761
 4762    x : an antsImage
 4763
 4764    background_dilation : integer - amount to dilate foreground mask
 4765
 4766    erode_foreground : boolean - 2nd option which erodes the initial
 4767    foregound mask  to create a new foreground mask.  the background
 4768    mask is the initial mask minus the eroded mask.
 4769
 4770    """
 4771    xshp = x.shape
 4772    xbc = ants.iMath( x - x.min(), "Normalize" )
 4773    xbc = ants.n3_bias_field_correction( xbc )
 4774    xmask = ants.threshold_image( xbc, "Otsu", 1 ).morphology("close",2).iMath("FillHoles")
 4775    xbkgmask = ants.iMath( xmask, "MD", background_dilation ) - xmask
 4776    fgmask = xmask
 4777    if erode_foreground:
 4778        fgmask = ants.iMath( xmask, "ME", background_dilation )
 4779        xbkgmask = xmask - fgmask
 4780    signal = (xbc[ fgmask == 1] ).mean()
 4781    noise = (xbc[ xbkgmask == 1] ).std()
 4782    return signal / noise
 4783
 4784def quantile_snr( x,
 4785    lowest_quantile=0.01,
 4786    low_quantile=0.1,
 4787    high_quantile=0.5,
 4788    highest_quantile=0.95 ):
 4789    """
 4790
 4791    Estimate signal to noise ratio (SNR) in an image.
 4792    Estimates noise from a background mask which is a
 4793    dilation of the foreground mask minus the foreground mask.
 4794    Actually estimates the reciprocal of the coefficient of variation.
 4795
 4796    Arguments
 4797    ---------
 4798
 4799    x : an antsImage
 4800
 4801    lowest_quantile : float value < 1 and > 0
 4802
 4803    low_quantile : float value < 1 and > 0
 4804
 4805    high_quantile : float value < 1 and > 0
 4806
 4807    highest_quantile : float value < 1 and > 0
 4808
 4809    """
 4810    import numpy as np
 4811    xshp = x.shape
 4812    xbc = ants.iMath( x - x.min(), "Normalize" )
 4813    xbc = ants.n3_bias_field_correction( xbc )
 4814    xbc = ants.iMath( xbc - xbc.min(), "Normalize" )
 4815    y = xbc.numpy()
 4816    ylowest = np.quantile( y[y>0], lowest_quantile )
 4817    ylo = np.quantile( y[y>0], low_quantile )
 4818    yhi = np.quantile( y[y>0], high_quantile )
 4819    yhiest = np.quantile( y[y>0], highest_quantile )
 4820    xbkgmask = ants.threshold_image( xbc, ylowest, ylo )
 4821    fgmask = ants.threshold_image( xbc, yhi, yhiest )
 4822    signal = (xbc[ fgmask == 1] ).mean()
 4823    noise = (xbc[ xbkgmask == 1] ).std()
 4824    return signal / noise
 4825
 4826def mask_snr( x, background_mask, foreground_mask, bias_correct=True ):
 4827    """
 4828
 4829    Estimate signal to noise ratio (SNR) in an image using
 4830    a user-defined foreground and background mask.
 4831    Actually estimates the reciprocal of the coefficient of variation.
 4832
 4833    Arguments
 4834    ---------
 4835
 4836    x : an antsImage
 4837
 4838    background_mask : binary antsImage
 4839
 4840    foreground_mask : binary antsImage
 4841
 4842    bias_correct : boolean
 4843
 4844    """
 4845    import numpy as np
 4846    if foreground_mask.sum() <= 1 or background_mask.sum() <= 1:
 4847        return 0
 4848    xbc = ants.iMath( x - x.min(), "Normalize" )
 4849    if bias_correct:
 4850        xbc = ants.n3_bias_field_correction( xbc )
 4851    xbc = ants.iMath( xbc - xbc.min(), "Normalize" )
 4852    signal = (xbc[ foreground_mask == 1] ).mean()
 4853    noise = (xbc[ background_mask == 1] ).std()
 4854    return signal / noise
 4855
 4856
 4857def dwi_deterministic_tracking(
 4858    dwi,
 4859    fa,
 4860    bvals,
 4861    bvecs,
 4862    num_processes=1,
 4863    mask=None,
 4864    label_image = None,
 4865    seed_labels = None,
 4866    fa_thresh = 0.05,
 4867    seed_density = 1,
 4868    step_size = 0.15,
 4869    peak_indices = None,
 4870    fit_method='WLS',
 4871    verbose = False ):
 4872    """
 4873
 4874    Performs deterministic tractography from the DWI and returns a tractogram
 4875    and path length data frame.
 4876
 4877    Arguments
 4878    ---------
 4879
 4880    dwi : an antsImage holding DWI acquisition
 4881
 4882    fa : an antsImage holding FA values
 4883
 4884    bvals : bvalues
 4885
 4886    bvecs : bvectors
 4887
 4888    num_processes : number of subprocesses
 4889
 4890    mask : mask within which to do tracking - if None, we will make a mask using the fa_thresh
 4891        and the code ants.threshold_image( fa, fa_thresh, 2.0 ).iMath("GetLargestComponent")
 4892
 4893    label_image : atlas labels
 4894
 4895    seed_labels : list of label numbers from the atlas labels
 4896
 4897    fa_thresh : 0.25 defaults
 4898
 4899    seed_density : 1 default number of seeds per voxel
 4900
 4901    step_size : for tracking
 4902
 4903    peak_indices : pass these in, if they are previously estimated.  otherwise, will
 4904        compute on the fly (slow)
 4905
 4906    fit_method : string one of WLS LS NLLS or restore - see import dipy.reconst.dti as dti and help(dti.TensorModel)
 4907
 4908    verbose : boolean
 4909
 4910    Returns
 4911    -------
 4912    dictionary holding tracts and stateful object.
 4913
 4914    Example
 4915    -------
 4916    >>> import antspymm
 4917    """
 4918    import os
 4919    import re
 4920    import nibabel as nib
 4921    import numpy as np
 4922    import ants
 4923    from dipy.io.gradients import read_bvals_bvecs
 4924    from dipy.core.gradients import gradient_table
 4925    from dipy.tracking import utils
 4926    import dipy.reconst.dti as dti
 4927    from dipy.segment.clustering import QuickBundles
 4928    from dipy.tracking.utils import path_length
 4929    if verbose:
 4930        print("begin tracking",flush=True)
 4931
 4932    affine = ants_to_nibabel_affine(dwi)
 4933
 4934    if isinstance( bvals, str ) or isinstance( bvecs, str ):
 4935        bvals, bvecs = read_bvals_bvecs(bvals, bvecs)
 4936    bvecs = repair_bvecs( bvecs )
 4937    gtab = gradient_table(bvals, bvecs=bvecs, atol=2.0 )
 4938    if mask is None:
 4939        mask = ants.threshold_image( fa, fa_thresh, 2.0 ).iMath("GetLargestComponent")
 4940    dwi_data = dwi.numpy()
 4941    dwi_mask = mask.numpy() == 1
 4942    dti_model = dti.TensorModel(gtab,fit_method=fit_method)
 4943    if verbose:
 4944        print("begin tracking fit",flush=True)
 4945    dti_fit = dti_model.fit(dwi_data, mask=dwi_mask)  # This step may take a while
 4946    evecs_img = dti_fit.evecs
 4947    from dipy.tracking.stopping_criterion import ThresholdStoppingCriterion
 4948    stopping_criterion = ThresholdStoppingCriterion(fa.numpy(), fa_thresh)
 4949    from dipy.data import get_sphere
 4950    sphere = get_sphere(name='symmetric362')
 4951    from dipy.direction import peaks_from_model
 4952    if peak_indices is None:
 4953        # problems with multi-threading ...
 4954        # see https://github.com/dipy/dipy/issues/2519
 4955        if verbose:
 4956            print("begin peaks",flush=True)
 4957        mynump=1
 4958        # if os.getenv("ITK_GLOBAL_DEFAULT_NUMBER_OF_THREADS"):
 4959        #    mynump = os.environ['ITK_GLOBAL_DEFAULT_NUMBER_OF_THREADS']
 4960        # current_openblas = os.environ.get('OPENBLAS_NUM_THREADS', '')
 4961        # current_mkl = os.environ.get('MKL_NUM_THREADS', '')
 4962        # os.environ['DIPY_OPENBLAS_NUM_THREADS'] = current_openblas
 4963        # os.environ['DIPY_MKL_NUM_THREADS'] = current_mkl
 4964        # os.environ['OPENBLAS_NUM_THREADS'] = '1'
 4965        # os.environ['MKL_NUM_THREADS'] = '1'
 4966        peak_indices = peaks_from_model(
 4967            model=dti_model,
 4968            data=dwi_data,
 4969            sphere=sphere,
 4970            relative_peak_threshold=.5,
 4971            min_separation_angle=25,
 4972            mask=dwi_mask,
 4973            npeaks=3, return_odf=False,
 4974            return_sh=False,
 4975            parallel=int(mynump) > 1,
 4976            num_processes=int(mynump)
 4977            )
 4978        if False:
 4979            if 'DIPY_OPENBLAS_NUM_THREADS' in os.environ:
 4980                os.environ['OPENBLAS_NUM_THREADS'] = \
 4981                    os.environ.pop('DIPY_OPENBLAS_NUM_THREADS', '')
 4982                if os.environ['OPENBLAS_NUM_THREADS'] in ['', None]:
 4983                    os.environ.pop('OPENBLAS_NUM_THREADS', '')
 4984            if 'DIPY_MKL_NUM_THREADS' in os.environ:
 4985                os.environ['MKL_NUM_THREADS'] = \
 4986                    os.environ.pop('DIPY_MKL_NUM_THREADS', '')
 4987                if os.environ['MKL_NUM_THREADS'] in ['', None]:
 4988                    os.environ.pop('MKL_NUM_THREADS', '')
 4989
 4990    if label_image is None or seed_labels is None:
 4991        seed_mask = fa.numpy().copy()
 4992        seed_mask[seed_mask >= fa_thresh] = 1
 4993        seed_mask[seed_mask < fa_thresh] = 0
 4994    else:
 4995        labels = label_image.numpy()
 4996        seed_mask = labels * 0
 4997        for u in seed_labels:
 4998            seed_mask[ labels == u ] = 1
 4999    seeds = utils.seeds_from_mask(seed_mask, affine=affine, density=seed_density)
 5000    from dipy.tracking.local_tracking import LocalTracking
 5001    from dipy.tracking.streamline import Streamlines
 5002    if verbose:
 5003        print("streamlines begin ...", flush=True)
 5004    streamlines_generator = LocalTracking(
 5005        peak_indices, stopping_criterion, seeds, affine=affine, step_size=step_size)
 5006    streamlines = Streamlines(streamlines_generator)
 5007    from dipy.io.stateful_tractogram import Space, StatefulTractogram
 5008    from dipy.io.streamline import save_tractogram
 5009    sft = None # StatefulTractogram(streamlines, dwi_img, Space.RASMM)
 5010    if verbose:
 5011        print("streamlines done", flush=True)
 5012    return {
 5013          'tractogram': sft,
 5014          'streamlines': streamlines,
 5015          'peak_indices': peak_indices
 5016          }
 5017
 5018def repair_bvecs( bvecs ):
 5019    bvecnorm = np.linalg.norm(bvecs,axis=1).reshape( bvecs.shape[0],1 )
 5020    # bvecnormnan = np.isnan( bvecnorm )
 5021    # nan_indices = list( np.unique( np.where(np.isnan(bvecs))[0]))
 5022    # bvecs = remove_elements_from_numpy_array( bvecs, nan_indices )
 5023    if abs(np.linalg.norm(bvecs)-1) > 0.009:
 5024        warnings.warn( "Warning: bvecs are not unit norm - we normalize them here but this may indicate a problem with the data.  Norm is : " + str( np.linalg.norm(bvecs) ) + " shape is " + str( bvecs.shape[0] ) + " " + str( bvecs.shape[1] ))
 5025        bvecs=np.where(bvecnorm > 1e-16, bvecs / bvecnorm, 0)
 5026    return bvecs
 5027
 5028
 5029def dwi_closest_peak_tracking(
 5030    dwi,
 5031    fa,
 5032    bvals,
 5033    bvecs,
 5034    num_processes=1,
 5035    mask=None,
 5036    label_image = None,
 5037    seed_labels = None,
 5038    fa_thresh = 0.05,
 5039    seed_density = 1,
 5040    step_size = 0.15,
 5041    peak_indices = None,
 5042    verbose = False ):
 5043    """
 5044
 5045    Performs deterministic tractography from the DWI and returns a tractogram
 5046    and path length data frame.
 5047
 5048    Arguments
 5049    ---------
 5050
 5051    dwi : an antsImage holding DWI acquisition
 5052
 5053    fa : an antsImage holding FA values
 5054
 5055    bvals : bvalues
 5056
 5057    bvecs : bvectors
 5058
 5059    num_processes : number of subprocesses
 5060
 5061    mask : mask within which to do tracking - if None, we will make a mask using the fa_thresh
 5062        and the code ants.threshold_image( fa, fa_thresh, 2.0 ).iMath("GetLargestComponent")
 5063
 5064    label_image : atlas labels
 5065
 5066    seed_labels : list of label numbers from the atlas labels
 5067
 5068    fa_thresh : 0.25 defaults
 5069
 5070    seed_density : 1 default number of seeds per voxel
 5071
 5072    step_size : for tracking
 5073
 5074    peak_indices : pass these in, if they are previously estimated.  otherwise, will
 5075        compute on the fly (slow)
 5076
 5077    verbose : boolean
 5078
 5079    Returns
 5080    -------
 5081    dictionary holding tracts and stateful object.
 5082
 5083    Example
 5084    -------
 5085    >>> import antspymm
 5086    """
 5087    import os
 5088    import re
 5089    import nibabel as nib
 5090    import numpy as np
 5091    import ants
 5092    from dipy.io.gradients import read_bvals_bvecs
 5093    from dipy.core.gradients import gradient_table
 5094    from dipy.tracking import utils
 5095    import dipy.reconst.dti as dti
 5096    from dipy.segment.clustering import QuickBundles
 5097    from dipy.tracking.utils import path_length
 5098    from dipy.core.gradients import gradient_table
 5099    from dipy.data import small_sphere
 5100    from dipy.direction import BootDirectionGetter, ClosestPeakDirectionGetter
 5101    from dipy.reconst.csdeconv import (ConstrainedSphericalDeconvModel,
 5102                                    auto_response_ssst)
 5103    from dipy.reconst.shm import CsaOdfModel
 5104    from dipy.tracking.local_tracking import LocalTracking
 5105    from dipy.tracking.streamline import Streamlines
 5106    from dipy.tracking.stopping_criterion import ThresholdStoppingCriterion
 5107
 5108    if verbose:
 5109        print("begin tracking",flush=True)
 5110
 5111    affine = ants_to_nibabel_affine(dwi)
 5112    if isinstance( bvals, str ) or isinstance( bvecs, str ):
 5113        bvals, bvecs = read_bvals_bvecs(bvals, bvecs)
 5114    bvecs = repair_bvecs( bvecs )
 5115    gtab = gradient_table(bvals, bvecs=bvecs, atol=2.0 )
 5116    if mask is None:
 5117        mask = ants.threshold_image( fa, fa_thresh, 2.0 ).iMath("GetLargestComponent")
 5118    dwi_data = dwi.numpy()
 5119    dwi_mask = mask.numpy() == 1
 5120
 5121
 5122    response, ratio = auto_response_ssst(gtab, dwi_data, roi_radii=10, fa_thr=0.7)
 5123    csd_model = ConstrainedSphericalDeconvModel(gtab, response, sh_order=6)
 5124    csd_fit = csd_model.fit(dwi_data, mask=dwi_mask)
 5125    csa_model = CsaOdfModel(gtab, sh_order=6)
 5126    gfa = csa_model.fit(dwi_data, mask=dwi_mask).gfa
 5127    stopping_criterion = ThresholdStoppingCriterion(gfa, .25)
 5128
 5129
 5130    if label_image is None or seed_labels is None:
 5131        seed_mask = fa.numpy().copy()
 5132        seed_mask[seed_mask >= fa_thresh] = 1
 5133        seed_mask[seed_mask < fa_thresh] = 0
 5134    else:
 5135        labels = label_image.numpy()
 5136        seed_mask = labels * 0
 5137        for u in seed_labels:
 5138            seed_mask[ labels == u ] = 1
 5139    seeds = utils.seeds_from_mask(seed_mask, affine=affine, density=seed_density)
 5140    if verbose:
 5141        print("streamlines begin ...", flush=True)
 5142
 5143    pmf = csd_fit.odf(small_sphere).clip(min=0)
 5144    if verbose:
 5145        print("ClosestPeakDirectionGetter begin ...", flush=True)
 5146    peak_dg = ClosestPeakDirectionGetter.from_pmf(pmf, max_angle=30.,
 5147                                                sphere=small_sphere)
 5148    if verbose:
 5149        print("local tracking begin ...", flush=True)
 5150    streamlines_generator = LocalTracking(peak_dg, stopping_criterion, seeds,
 5151                                            affine, step_size=.5)
 5152    streamlines = Streamlines(streamlines_generator)
 5153    from dipy.io.stateful_tractogram import Space, StatefulTractogram
 5154    from dipy.io.streamline import save_tractogram
 5155    sft = None # StatefulTractogram(streamlines, dwi_img, Space.RASMM)
 5156    if verbose:
 5157        print("streamlines done", flush=True)
 5158    return {
 5159          'tractogram': sft,
 5160          'streamlines': streamlines
 5161          }
 5162
 5163def dwi_streamline_pairwise_connectivity( streamlines, label_image, labels_to_connect=[1,None], verbose=False ):
 5164    """
 5165
 5166    Return streamlines connecting all of the regions in the label set. Ideal
 5167    for just 2 regions.
 5168
 5169    Arguments
 5170    ---------
 5171
 5172    streamlines : streamline object from dipy
 5173
 5174    label_image : atlas labels
 5175
 5176    labels_to_connect : list of 2 labels or [label,None]
 5177
 5178    verbose : boolean
 5179
 5180    Returns
 5181    -------
 5182    the subset of streamlines and a streamline count
 5183
 5184    Example
 5185    -------
 5186    >>> import antspymm
 5187    """
 5188    from dipy.tracking.streamline import Streamlines
 5189    keep_streamlines = Streamlines()
 5190
 5191    affine = ants_to_nibabel_affine(label_image) # to_nibabel(label_image).affine
 5192
 5193    lin_T, offset = utils._mapping_to_voxel(affine)
 5194    label_image_np = label_image.numpy()
 5195    def check_it( sl, target_label, label_image, index, full=False ):
 5196        if full:
 5197            maxind=sl.shape[0]
 5198            for index in range(maxind):
 5199                pt = utils._to_voxel_coordinates(sl[index,:], lin_T, offset)
 5200                mylab = (label_image[ pt[0], pt[1], pt[2] ]).astype(int)
 5201                if mylab == target_label[0] or mylab == target_label[1]:
 5202                    return { 'ok': True, 'label':mylab }
 5203        else:
 5204            pt = utils._to_voxel_coordinates(sl[index,:], lin_T, offset)
 5205            mylab = (label_image[ pt[0], pt[1], pt[2] ]).astype(int)
 5206            if mylab == target_label[0] or mylab == target_label[1]:
 5207                return { 'ok': True, 'label':mylab }
 5208        return { 'ok': False, 'label':None }
 5209    ct=0
 5210    for k in range( len( streamlines ) ):
 5211        sl = streamlines[k]
 5212        mycheck = check_it( sl, labels_to_connect, label_image_np, index=0, full=True )
 5213        if mycheck['ok']:
 5214            otherind=1
 5215            if mycheck['label'] == labels_to_connect[1]:
 5216                otherind=0
 5217            lsl = len( sl )-1
 5218            pt = utils._to_voxel_coordinates(sl[lsl,:], lin_T, offset)
 5219            mylab_end = (label_image_np[ pt[0], pt[1], pt[2] ]).astype(int)
 5220            accept_point = mylab_end == labels_to_connect[otherind]
 5221            if verbose and accept_point:
 5222                print( mylab_end )
 5223            if labels_to_connect[1] is None:
 5224                accept_point = mylab_end != 0
 5225            if accept_point:
 5226                keep_streamlines.append(sl)
 5227                ct=ct+1
 5228    return { 'streamlines': keep_streamlines, 'count': ct }
 5229
 5230def dwi_streamline_pairwise_connectivity_old(
 5231    streamlines,
 5232    label_image,
 5233    exclusion_label = None,
 5234    verbose = False ):
 5235    import os
 5236    import re
 5237    import nibabel as nib
 5238    import numpy as np
 5239    import ants
 5240    from dipy.io.gradients import read_bvals_bvecs
 5241    from dipy.core.gradients import gradient_table
 5242    from dipy.tracking import utils
 5243    import dipy.reconst.dti as dti
 5244    from dipy.segment.clustering import QuickBundles
 5245    from dipy.tracking.utils import path_length
 5246    from dipy.tracking.local_tracking import LocalTracking
 5247    from dipy.tracking.streamline import Streamlines
 5248    volUnit = np.prod( ants.get_spacing( label_image ) )
 5249    labels = label_image.numpy()
 5250
 5251    affine = ants_to_nibabel_affine(label_image) # to_nibabel(label_image).affine
 5252
 5253    import numpy as np
 5254    from dipy.io.image import load_nifti_data, load_nifti, save_nifti
 5255    import pandas as pd
 5256    ulabs = np.unique( labels[ labels > 0 ] )
 5257    if exclusion_label is not None:
 5258        ulabs = ulabs[ ulabs != exclusion_label ]
 5259        exc_slice = labels == exclusion_label
 5260    if verbose:
 5261        print("Begin connectivity")
 5262    tracts = []
 5263    for k in range(len(ulabs)):
 5264        cc_slice = labels == ulabs[k]
 5265        cc_streamlines = utils.target(streamlines, affine, cc_slice)
 5266        cc_streamlines = Streamlines(cc_streamlines)
 5267        if exclusion_label is not None:
 5268            cc_streamlines = utils.target(cc_streamlines, affine, exc_slice, include=False)
 5269            cc_streamlines = Streamlines(cc_streamlines)
 5270        for j in range(len(ulabs)):
 5271            cc_slice2 = labels == ulabs[j]
 5272            cc_streamlines2 = utils.target(cc_streamlines, affine, cc_slice2)
 5273            cc_streamlines2 = Streamlines(cc_streamlines2)
 5274            if exclusion_label is not None:
 5275                cc_streamlines2 = utils.target(cc_streamlines2, affine, exc_slice, include=False)
 5276                cc_streamlines2 = Streamlines(cc_streamlines2)
 5277            tracts.append( cc_streamlines2 )
 5278        if verbose:
 5279            print("end connectivity")
 5280    return {
 5281          'pairwise_tracts': tracts
 5282          }
 5283
 5284
 5285def dwi_streamline_connectivity(
 5286    streamlines,
 5287    label_image,
 5288    label_dataframe,
 5289    verbose = False ):
 5290    """
 5291
 5292    Summarize network connetivity of the input streamlines between all of the
 5293    regions in the label set.
 5294
 5295    Arguments
 5296    ---------
 5297
 5298    streamlines : streamline object from dipy
 5299
 5300    label_image : atlas labels
 5301
 5302    label_dataframe : pandas dataframe containing descriptions for the labels in antspy style (Label,Description columns)
 5303
 5304    verbose : boolean
 5305
 5306    Returns
 5307    -------
 5308    dictionary holding summary connection statistics in wide format and matrix format.
 5309
 5310    Example
 5311    -------
 5312    >>> import antspymm
 5313    """
 5314    import os
 5315    import re
 5316    import nibabel as nib
 5317    import numpy as np
 5318    import ants
 5319    from dipy.io.gradients import read_bvals_bvecs
 5320    from dipy.core.gradients import gradient_table
 5321    from dipy.tracking import utils
 5322    import dipy.reconst.dti as dti
 5323    from dipy.segment.clustering import QuickBundles
 5324    from dipy.tracking.utils import path_length
 5325    from dipy.tracking.local_tracking import LocalTracking
 5326    from dipy.tracking.streamline import Streamlines
 5327    import os
 5328    import re
 5329    import nibabel as nib
 5330    import numpy as np
 5331    import ants
 5332    from dipy.io.gradients import read_bvals_bvecs
 5333    from dipy.core.gradients import gradient_table
 5334    from dipy.tracking import utils
 5335    import dipy.reconst.dti as dti
 5336    from dipy.segment.clustering import QuickBundles
 5337    from dipy.tracking.utils import path_length
 5338    from dipy.tracking.local_tracking import LocalTracking
 5339    from dipy.tracking.streamline import Streamlines
 5340    volUnit = np.prod( ants.get_spacing( label_image ) )
 5341    labels = label_image.numpy()
 5342
 5343    affine = ants_to_nibabel_affine(label_image) # to_nibabel(label_image).affine
 5344
 5345    import numpy as np
 5346    from dipy.io.image import load_nifti_data, load_nifti, save_nifti
 5347    import pandas as pd
 5348    ulabs = label_dataframe['Label']
 5349    labels_to_connect = ulabs[ulabs > 0]
 5350    Ctdf = None
 5351    lin_T, offset = utils._mapping_to_voxel(affine)
 5352    label_image_np = label_image.numpy()
 5353    def check_it( sl, target_label, label_image, index, not_label = None ):
 5354        pt = utils._to_voxel_coordinates(sl[index,:], lin_T, offset)
 5355        mylab = (label_image[ pt[0], pt[1], pt[2] ]).astype(int)
 5356        if not_label is None:
 5357            if ( mylab == target_label ).sum() > 0 :
 5358                return { 'ok': True, 'label':mylab }
 5359        else:
 5360            if ( mylab == target_label ).sum() > 0 and ( mylab == not_label ).sum() == 0:
 5361                return { 'ok': True, 'label':mylab }
 5362        return { 'ok': False, 'label':None }
 5363    ct=0
 5364    which = lambda lst:list(np.where(lst)[0])
 5365    myCount = np.zeros( [len(ulabs),len(ulabs)])
 5366    for k in range( len( streamlines ) ):
 5367            sl = streamlines[k]
 5368            mycheck = check_it( sl, labels_to_connect, label_image_np, index=0 )
 5369            if mycheck['ok']:
 5370                exclabel=mycheck['label']
 5371                lsl = len( sl )-1
 5372                mycheck2 = check_it( sl, labels_to_connect, label_image_np, index=lsl, not_label=exclabel )
 5373                if mycheck2['ok']:
 5374                    myCount[ulabs == mycheck['label'],ulabs == mycheck2['label']]+=1
 5375                    ct=ct+1
 5376    Ctdf = label_dataframe.copy()
 5377    for k in range(len(ulabs)):
 5378            nn3 = "CnxCount"+str(k).zfill(3)
 5379            Ctdf.insert(Ctdf.shape[1], nn3, myCount[k,:] )
 5380    Ctdfw = antspyt1w.merge_hierarchical_csvs_to_wide_format( { 'networkc': Ctdf },  Ctdf.keys()[2:Ctdf.shape[1]] )
 5381    return { 'connectivity_matrix' :  myCount, 'connectivity_wide' : Ctdfw }
 5382
 5383def dwi_streamline_connectivity_old(
 5384    streamlines,
 5385    label_image,
 5386    label_dataframe,
 5387    verbose = False ):
 5388    """
 5389
 5390    Summarize network connetivity of the input streamlines between all of the
 5391    regions in the label set.
 5392
 5393    Arguments
 5394    ---------
 5395
 5396    streamlines : streamline object from dipy
 5397
 5398    label_image : atlas labels
 5399
 5400    label_dataframe : pandas dataframe containing descriptions for the labels in antspy style (Label,Description columns)
 5401
 5402    verbose : boolean
 5403
 5404    Returns
 5405    -------
 5406    dictionary holding summary connection statistics in wide format and matrix format.
 5407
 5408    Example
 5409    -------
 5410    >>> import antspymm
 5411    """
 5412
 5413    if verbose:
 5414        print("streamline connections ...")
 5415
 5416    import os
 5417    import re
 5418    import nibabel as nib
 5419    import numpy as np
 5420    import ants
 5421    from dipy.io.gradients import read_bvals_bvecs
 5422    from dipy.core.gradients import gradient_table
 5423    from dipy.tracking import utils
 5424    import dipy.reconst.dti as dti
 5425    from dipy.segment.clustering import QuickBundles
 5426    from dipy.tracking.utils import path_length
 5427    from dipy.tracking.local_tracking import LocalTracking
 5428    from dipy.tracking.streamline import Streamlines
 5429
 5430    volUnit = np.prod( ants.get_spacing( label_image ) )
 5431    labels = label_image.numpy()
 5432
 5433    affine = ants_to_nibabel_affine(label_image) # to_nibabel(label_image).affine
 5434
 5435    if verbose:
 5436        print("path length begin ... volUnit = " + str( volUnit ) )
 5437    import numpy as np
 5438    from dipy.io.image import load_nifti_data, load_nifti, save_nifti
 5439    import pandas as pd
 5440    ulabs = label_dataframe['Label']
 5441    pathLmean = np.zeros( [len(ulabs)])
 5442    pathLtot = np.zeros( [len(ulabs)])
 5443    pathCt = np.zeros( [len(ulabs)])
 5444    for k in range(len(ulabs)):
 5445        cc_slice = labels == ulabs[k]
 5446        cc_streamlines = utils.target(streamlines, affine, cc_slice)
 5447        cc_streamlines = Streamlines(cc_streamlines)
 5448        if len(cc_streamlines) > 0:
 5449            wmpl = path_length(cc_streamlines, affine, cc_slice)
 5450            mean_path_length = wmpl[wmpl>0].mean()
 5451            total_path_length = wmpl[wmpl>0].sum()
 5452            pathLmean[int(k)] = mean_path_length
 5453            pathLtot[int(k)] = total_path_length
 5454            pathCt[int(k)] = len(cc_streamlines) * volUnit
 5455
 5456    # convert paths to data frames
 5457    pathdf = label_dataframe.copy()
 5458    pathdf.insert(pathdf.shape[1], "mean_path_length", pathLmean )
 5459    pathdf.insert(pathdf.shape[1], "total_path_length", pathLtot )
 5460    pathdf.insert(pathdf.shape[1], "streamline_count", pathCt )
 5461    pathdfw =antspyt1w.merge_hierarchical_csvs_to_wide_format(
 5462        {path_length:pathdf }, ['mean_path_length', 'total_path_length', 'streamline_count'] )
 5463    allconnexwide = pathdfw
 5464
 5465    if verbose:
 5466        print("path length done ...")
 5467
 5468    Mdfw = None
 5469    Tdfw = None
 5470    Mdf = None
 5471    Tdf = None
 5472    Ctdf = None
 5473    Ctdfw = None
 5474    if True:
 5475        if verbose:
 5476            print("Begin connectivity")
 5477        M = np.zeros( [len(ulabs),len(ulabs)])
 5478        T = np.zeros( [len(ulabs),len(ulabs)])
 5479        myCount = np.zeros( [len(ulabs),len(ulabs)])
 5480        for k in range(len(ulabs)):
 5481            cc_slice = labels == ulabs[k]
 5482            cc_streamlines = utils.target(streamlines, affine, cc_slice)
 5483            cc_streamlines = Streamlines(cc_streamlines)
 5484            for j in range(len(ulabs)):
 5485                cc_slice2 = labels == ulabs[j]
 5486                cc_streamlines2 = utils.target(cc_streamlines, affine, cc_slice2)
 5487                cc_streamlines2 = Streamlines(cc_streamlines2)
 5488                if len(cc_streamlines2) > 0 :
 5489                    wmpl = path_length(cc_streamlines2, affine, cc_slice2)
 5490                    mean_path_length = wmpl[wmpl>0].mean()
 5491                    total_path_length = wmpl[wmpl>0].sum()
 5492                    M[int(j),int(k)] = mean_path_length
 5493                    T[int(j),int(k)] = total_path_length
 5494                    myCount[int(j),int(k)] = len( cc_streamlines2 ) * volUnit
 5495        if verbose:
 5496            print("end connectivity")
 5497        Mdf = label_dataframe.copy()
 5498        Tdf = label_dataframe.copy()
 5499        Ctdf = label_dataframe.copy()
 5500        for k in range(len(ulabs)):
 5501            nn1 = "CnxMeanPL"+str(k).zfill(3)
 5502            nn2 = "CnxTotPL"+str(k).zfill(3)
 5503            nn3 = "CnxCount"+str(k).zfill(3)
 5504            Mdf.insert(Mdf.shape[1], nn1, M[k,:] )
 5505            Tdf.insert(Tdf.shape[1], nn2, T[k,:] )
 5506            Ctdf.insert(Ctdf.shape[1], nn3, myCount[k,:] )
 5507        Mdfw = antspyt1w.merge_hierarchical_csvs_to_wide_format( { 'networkm' : Mdf },  Mdf.keys()[2:Mdf.shape[1]] )
 5508        Tdfw = antspyt1w.merge_hierarchical_csvs_to_wide_format( { 'networkt' : Tdf },  Tdf.keys()[2:Tdf.shape[1]] )
 5509        Ctdfw = antspyt1w.merge_hierarchical_csvs_to_wide_format( { 'networkc': Ctdf },  Ctdf.keys()[2:Ctdf.shape[1]] )
 5510        allconnexwide = pd.concat( [
 5511            pathdfw,
 5512            Mdfw,
 5513            Tdfw,
 5514            Ctdfw ], axis=1, ignore_index=False )
 5515
 5516    return {
 5517          'connectivity': allconnexwide,
 5518          'connectivity_matrix_mean': Mdf,
 5519          'connectivity_matrix_total': Tdf,
 5520          'connectivity_matrix_count': Ctdf
 5521          }
 5522
 5523
 5524def hierarchical_modality_summary(
 5525    target_image,
 5526    hier,
 5527    transformlist,
 5528    modality_name,
 5529    return_keys = ["Mean","Volume"],
 5530    verbose = False ):
 5531    """
 5532
 5533    Use output of antspyt1w.hierarchical to summarize a modality
 5534
 5535    Arguments
 5536    ---------
 5537
 5538    target_image : the image to summarize - should be brain extracted
 5539
 5540    hier : dictionary holding antspyt1w.hierarchical output
 5541
 5542    transformlist : spatial transformations mapping from T1 to this modality (e.g. from ants.registration)
 5543
 5544    modality_name : adds the modality name to the data frame columns
 5545
 5546    return_keys = ["Mean","Volume"] keys to return
 5547
 5548    verbose : boolean
 5549
 5550    Returns
 5551    -------
 5552    data frame holding summary statistics in wide format
 5553
 5554    Example
 5555    -------
 5556    >>> import antspymm
 5557    """
 5558    dfout = pd.DataFrame()
 5559    def myhelper( target_image, seg, mytx, mapname, modname, mydf, extra='', verbose=False ):
 5560        if verbose:
 5561            print( mapname )
 5562        target_image_mask = ants.image_clone( target_image ) * 0.0
 5563        target_image_mask[ target_image != 0 ] = 1
 5564        cortmapped = ants.apply_transforms(
 5565            target_image,
 5566            seg,
 5567            mytx, interpolator='nearestNeighbor' ) * target_image_mask
 5568        mapped = antspyt1w.map_intensity_to_dataframe(
 5569            mapname,
 5570            target_image,
 5571            cortmapped )
 5572        mapped.iloc[:,1] = modname + '_' + extra + mapped.iloc[:,1]
 5573        mappedw = antspyt1w.merge_hierarchical_csvs_to_wide_format(
 5574            { 'x' : mapped},
 5575            col_names = return_keys )
 5576        if verbose:
 5577            print( mappedw.keys() )
 5578        if mydf.shape[0] > 0:
 5579            mydf = pd.concat( [ mydf, mappedw], axis=1, ignore_index=False )
 5580        else:
 5581            mydf = mappedw
 5582        return mydf
 5583    if hier['dkt_parc']['dkt_cortex'] is not None:
 5584        dfout = myhelper( target_image, hier['dkt_parc']['dkt_cortex'], transformlist,
 5585            "dkt", modality_name, dfout, extra='', verbose=verbose )
 5586    if hier['deep_cit168lab'] is not None:
 5587        dfout = myhelper( target_image, hier['deep_cit168lab'], transformlist,
 5588            "CIT168_Reinf_Learn_v1_label_descriptions_pad", modality_name, dfout, extra='deep_', verbose=verbose )
 5589    if hier['cit168lab'] is not None:
 5590        dfout = myhelper( target_image, hier['cit168lab'], transformlist,
 5591            "CIT168_Reinf_Learn_v1_label_descriptions_pad", modality_name, dfout, extra='', verbose=verbose  )
 5592    if hier['bf'] is not None:
 5593        dfout = myhelper( target_image, hier['bf'], transformlist,
 5594            "nbm3CH13", modality_name, dfout, extra='', verbose=verbose  )
 5595    # if hier['mtl'] is not None:
 5596    #    dfout = myhelper( target_image, hier['mtl'], reg,
 5597    #        "mtl_description", modality_name, dfout, extra='', verbose=verbose  )
 5598    return dfout
 5599
 5600def get_rsf_outputs( coords ):
 5601    if coords == 'powers':
 5602        return list([ 'meanBold', 'fmri_template', 'alff', 'falff', 'PerAF', 
 5603                   'CinguloopercularTaskControl', 'DefaultMode', 
 5604                   'MemoryRetrieval', 'VentralAttention', 'Visual',
 5605                   'FrontoparietalTaskControl', 'Salience', 'Subcortical', 'DorsalAttention'])
 5606    else:
 5607        yeo = pd.read_csv( get_data('ppmi_template_500Parcels_Yeo2011_17Networks_2023_homotopic', target_extension=".csv")) # yeo 2023 coordinates
 5608        return list( yeo['SystemName'].unique() )
 5609
 5610def tra_initializer( fixed, moving, n_simulations=32, max_rotation=30,
 5611    transform=['rigid'], compreg=None, random_seed=42, verbose=False ):
 5612    """
 5613    multi-start multi-transform registration solution - based on ants.registration
 5614
 5615    fixed: fixed image
 5616
 5617    moving: moving image
 5618
 5619    n_simulations : number of simulations
 5620
 5621    max_rotation : maximum rotation angle
 5622
 5623    transform : list of transforms to loop through
 5624
 5625    compreg : registration results against which to compare
 5626
 5627    random_seed : random seed for reproducibility
 5628
 5629    verbose : boolean
 5630
 5631    """
 5632    import random
 5633    if random_seed is not None:
 5634        random.seed(random_seed)
 5635    if True:
 5636        output_directory = tempfile.mkdtemp()
 5637        output_directory_w = output_directory + "/tra_reg/"
 5638        os.makedirs(output_directory_w,exist_ok=True)
 5639        bestmi = math.inf
 5640        bestvar = 0.0
 5641        myorig = list(ants.get_origin( fixed ))
 5642        mymax = 0;
 5643        for k in range(len( myorig ) ):
 5644            if abs(myorig[k]) > mymax:
 5645                mymax = abs(myorig[k])
 5646        maxtrans = mymax * 0.05
 5647        if compreg is None:
 5648            bestreg=ants.registration( fixed,moving,'Translation',
 5649                outprefix=output_directory_w+"trans")
 5650            initx = ants.read_transform( bestreg['fwdtransforms'][0] )
 5651        else :
 5652            bestreg=compreg
 5653            initx = ants.read_transform( bestreg['fwdtransforms'][0] )
 5654        for mytx in transform:
 5655            regtx = 'antsRegistrationSyNRepro[r]'
 5656            with tempfile.NamedTemporaryFile(suffix='.h5') as tp:
 5657                if mytx == 'translation':
 5658                    regtx = 'Translation'
 5659                    rRotGenerator = ants.contrib.RandomTranslate3D( ( maxtrans*(-1.0), maxtrans ), reference=fixed )
 5660                elif mytx == 'affine':
 5661                    regtx = 'Affine'
 5662                    rRotGenerator = ants.contrib.RandomRotate3D( ( maxtrans*(-1.0), maxtrans ), reference=fixed )
 5663                else:
 5664                    rRotGenerator = ants.contrib.RandomRotate3D( ( max_rotation*(-1.0), max_rotation ), reference=fixed )
 5665                for k in range(n_simulations):
 5666                    simtx = ants.compose_ants_transforms( [rRotGenerator.transform(), initx] )
 5667                    ants.write_transform( simtx, tp.name )
 5668                    if k > 0:
 5669                        reg = ants.registration( fixed, moving, regtx,
 5670                            initial_transform=tp.name,
 5671                            outprefix=output_directory_w+"reg"+str(k),
 5672                            verbose=False )
 5673                    else:
 5674                        reg = ants.registration( fixed, moving,
 5675                            regtx,
 5676                            outprefix=output_directory_w+"reg"+str(k),
 5677                            verbose=False )
 5678                    mymi = math.inf
 5679                    temp = reg['warpedmovout']
 5680                    myvar = temp.numpy().var()
 5681                    if verbose:
 5682                        print( str(k) + " : " + regtx  + " : " + mytx + " _var_ " + str( myvar ) )
 5683                    if myvar > 0 :
 5684                        mymi = ants.image_mutual_information( fixed, temp )
 5685                        if mymi < bestmi:
 5686                            if verbose:
 5687                                print( "mi @ " + str(k) + " : " + str(mymi), flush=True)
 5688                            bestmi = mymi
 5689                            bestreg = reg
 5690                            bestvar = myvar
 5691        if bestvar == 0.0 and compreg is not None:
 5692            return compreg        
 5693        return bestreg
 5694
 5695def neuromelanin( list_nm_images, t1, t1_head, t1lab, brain_stem_dilation=8,
 5696    bias_correct=True,
 5697    denoise=None,
 5698    srmodel=None,
 5699    target_range=[0,1],
 5700    poly_order='hist',
 5701    normalize_nm = False,
 5702    verbose=False ) :
 5703
 5704  """
 5705  Outputs the averaged and registered neuromelanin image, and neuromelanin labels
 5706
 5707  Arguments
 5708  ---------
 5709  list_nm_image : list of ANTsImages
 5710    list of neuromenlanin repeat images
 5711
 5712  t1 : ANTsImage
 5713    input 3-D T1 brain image
 5714
 5715  t1_head : ANTsImage
 5716    input 3-D T1 head image
 5717
 5718  t1lab : ANTsImage
 5719    t1 labels that will be propagated to the NM
 5720
 5721  brain_stem_dilation : integer default 8
 5722    dilates the brain stem mask to better match coverage of NM
 5723
 5724  bias_correct : boolean
 5725
 5726  denoise : None or integer
 5727
 5728  srmodel : None -- this is a work in progress feature, probably not optimal
 5729
 5730  target_range : 2-element tuple
 5731        a tuple or array defining the (min, max) of the input image
 5732        (e.g., [-127.5, 127.5] or [0,1]).  Output images will be scaled back to original
 5733        intensity. This range should match the mapping used in the training
 5734        of the network.
 5735
 5736  poly_order : if not None, will fit a global regression model to map
 5737      intensity back to original histogram space; if 'hist' will match
 5738      by histogram matching - ants.histogram_match_image
 5739
 5740  normalize_nm : boolean - WIP not validated
 5741
 5742  verbose : boolean
 5743
 5744  Returns
 5745  ---------
 5746  Averaged and registered neuromelanin image and neuromelanin labels and wide csv
 5747
 5748  """
 5749
 5750  fnt=os.path.expanduser("~/.antspyt1w/CIT168_T1w_700um_pad_adni.nii.gz" )
 5751  fntNM=os.path.expanduser("~/.antspymm/CIT168_T1w_700um_pad_adni_NM_norm_avg.nii.gz" )
 5752  fntbst=os.path.expanduser("~/.antspyt1w/CIT168_T1w_700um_pad_adni_brainstem.nii.gz")
 5753  fnslab=os.path.expanduser("~/.antspyt1w/CIT168_MT_Slab_adni.nii.gz")
 5754  fntseg=os.path.expanduser("~/.antspyt1w/det_atlas_25_pad_LR_adni.nii.gz")
 5755
 5756  template = mm_read( fnt )
 5757  templateNM = ants.iMath( mm_read( fntNM ), "Normalize" )
 5758  templatebstem = mm_read( fntbst ).threshold_image( 1, 1000 )
 5759  # reg = ants.registration( t1, template, 'antsRegistrationSyNQuickRepro[s]' )
 5760  reg = ants.registration( t1, template, 'antsRegistrationSyNQuickRepro[s]' )
 5761  # map NM avg to t1 for neuromelanin processing
 5762  nmavg2t1 = ants.apply_transforms( t1, templateNM,
 5763    reg['fwdtransforms'], interpolator='linear' )
 5764  slab2t1 = ants.threshold_image( nmavg2t1, "Otsu", 2 ).threshold_image(1,2).iMath("MD",1).iMath("FillHoles")
 5765  # map brain stem and slab to t1 for neuromelanin processing
 5766  bstem2t1 = ants.apply_transforms( t1, templatebstem,
 5767    reg['fwdtransforms'],
 5768    interpolator='nearestNeighbor' ).iMath("MD",1)
 5769  slab2t1B = ants.apply_transforms( t1, mm_read( fnslab ),
 5770    reg['fwdtransforms'], interpolator = 'nearestNeighbor')
 5771  bstem2t1 = ants.crop_image( bstem2t1, slab2t1 )
 5772  cropper = ants.decrop_image( bstem2t1, slab2t1 ).iMath("MD",brain_stem_dilation)
 5773
 5774  # Average images in image_list
 5775  nm_avg = list_nm_images[0]*0.0
 5776  for k in range(len( list_nm_images )):
 5777    if denoise is not None:
 5778        list_nm_images[k] = ants.denoise_image( list_nm_images[k],
 5779            shrink_factor=1,
 5780            p=denoise,
 5781            r=denoise+1,
 5782            noise_model='Gaussian' )
 5783    if bias_correct :
 5784        n4mask = ants.threshold_image( ants.iMath(list_nm_images[k], "Normalize" ), 0.05, 1 )
 5785        list_nm_images[k] = ants.n4_bias_field_correction( list_nm_images[k], mask=n4mask )
 5786    nm_avg = nm_avg + ants.resample_image_to_target( list_nm_images[k], nm_avg ) / len( list_nm_images )
 5787
 5788  if verbose:
 5789      print("Register each nm image in list_nm_images to the averaged nm image (avg)")
 5790  nm_avg_new = nm_avg * 0.0
 5791  txlist = []
 5792  for k in range(len( list_nm_images )):
 5793    if verbose:
 5794        print(str(k) + " of " + str(len( list_nm_images ) ) )
 5795    current_image = ants.registration( list_nm_images[k], nm_avg,
 5796        type_of_transform = 'antsRegistrationSyNRepro[r]' )
 5797    txlist.append( current_image['fwdtransforms'][0] )
 5798    current_image = current_image['warpedfixout']
 5799    nm_avg_new = nm_avg_new + current_image / len( list_nm_images )
 5800  nm_avg = nm_avg_new
 5801
 5802  if verbose:
 5803      print("do slab registration to map anatomy to NM space")
 5804  t1c = ants.crop_image( t1_head, slab2t1 ).iMath("Normalize") # old way
 5805  nmavg2t1c = ants.crop_image( nmavg2t1, slab2t1 ).iMath("Normalize")
 5806  # slabreg = ants.registration( nm_avg, nmavg2t1c, 'antsRegistrationSyNRepro[r]' )
 5807  slabreg = tra_initializer( nm_avg, t1c, verbose=verbose )
 5808  if False:
 5809      slabregT1 = tra_initializer( nm_avg, t1c, verbose=verbose  )
 5810      miNM = ants.image_mutual_information( ants.iMath(nm_avg,"Normalize"),
 5811            ants.iMath(slabreg0['warpedmovout'],"Normalize") )
 5812      miT1 = ants.image_mutual_information( ants.iMath(nm_avg,"Normalize"),
 5813            ants.iMath(slabreg1['warpedmovout'],"Normalize") )
 5814      if miT1 < miNM:
 5815        slabreg = slabregT1
 5816  labels2nm = ants.apply_transforms( nm_avg, t1lab, slabreg['fwdtransforms'],
 5817    interpolator = 'genericLabel' )
 5818  cropper2nm = ants.apply_transforms( nm_avg, cropper, slabreg['fwdtransforms'], interpolator='nearestNeighbor' )
 5819  nm_avg_cropped = ants.crop_image( nm_avg, cropper2nm )
 5820
 5821  if verbose:
 5822      print("now map these labels to each individual nm")
 5823  crop_mask_list = []
 5824  crop_nm_list = []
 5825  for k in range(len( list_nm_images )):
 5826      concattx = []
 5827      concattx.append( txlist[k] )
 5828      concattx.append( slabreg['fwdtransforms'][0] )
 5829      cropmask = ants.apply_transforms( list_nm_images[k], cropper,
 5830        concattx, interpolator = 'nearestNeighbor' )
 5831      crop_mask_list.append( cropmask )
 5832      temp = ants.crop_image( list_nm_images[k], cropmask )
 5833      crop_nm_list.append( temp )
 5834
 5835  if srmodel is not None:
 5836      if verbose:
 5837          print( " start sr " + str(len( crop_nm_list )) )
 5838      for k in range(len( crop_nm_list )):
 5839          if verbose:
 5840              print( " do sr " + str(k) )
 5841              print( crop_nm_list[k] )
 5842          temp = antspynet.apply_super_resolution_model_to_image(
 5843                crop_nm_list[k], srmodel, target_range=target_range,
 5844                regression_order=None )
 5845          if poly_order is not None:
 5846              bilin = ants.resample_image_to_target( crop_nm_list[k], temp )
 5847              if poly_order == 'hist':
 5848                  temp = ants.histogram_match_image( temp, bilin )
 5849              else:
 5850                  temp = antspynet.regression_match_image( temp, bilin, poly_order = poly_order )
 5851          crop_nm_list[k] = temp
 5852
 5853  nm_avg_cropped = crop_nm_list[0]*0.0
 5854  if verbose:
 5855      print( "cropped average" )
 5856      print( nm_avg_cropped )
 5857  for k in range(len( crop_nm_list )):
 5858      nm_avg_cropped = nm_avg_cropped + ants.apply_transforms( nm_avg_cropped,
 5859        crop_nm_list[k], txlist[k] ) / len( crop_nm_list )
 5860  for loop in range( 3 ):
 5861      nm_avg_cropped_new = nm_avg_cropped * 0.0
 5862      for k in range(len( crop_nm_list )):
 5863            myreg = ants.registration(
 5864                ants.iMath(nm_avg_cropped,"Normalize"),
 5865                ants.iMath(crop_nm_list[k],"Normalize"),
 5866                'antsRegistrationSyNRepro[r]' )
 5867            warpednext = ants.apply_transforms(
 5868                nm_avg_cropped_new,
 5869                crop_nm_list[k],
 5870                myreg['fwdtransforms'] )
 5871            nm_avg_cropped_new = nm_avg_cropped_new + warpednext
 5872      nm_avg_cropped = nm_avg_cropped_new / len( crop_nm_list )
 5873
 5874  slabregUpdated = tra_initializer( nm_avg_cropped, t1c, compreg=slabreg,verbose=verbose  )
 5875  tempOrig = ants.apply_transforms( nm_avg_cropped_new, t1c, slabreg['fwdtransforms'] )
 5876  tempUpdate = ants.apply_transforms( nm_avg_cropped_new, t1c, slabregUpdated['fwdtransforms'] )
 5877  miUpdate = ants.image_mutual_information(
 5878    ants.iMath(nm_avg_cropped,"Normalize"), ants.iMath(tempUpdate,"Normalize") )
 5879  miOrig = ants.image_mutual_information(
 5880    ants.iMath(nm_avg_cropped,"Normalize"), ants.iMath(tempOrig,"Normalize") )
 5881  if miUpdate < miOrig :
 5882      slabreg = slabregUpdated
 5883
 5884  if normalize_nm:
 5885      nm_avg_cropped = ants.iMath( nm_avg_cropped, "Normalize" )
 5886      nm_avg_cropped = ants.iMath( nm_avg_cropped, "TruncateIntensity",0.05,0.95)
 5887      nm_avg_cropped = ants.iMath( nm_avg_cropped, "Normalize" )
 5888
 5889  labels2nm = ants.apply_transforms( nm_avg_cropped, t1lab,
 5890        slabreg['fwdtransforms'], interpolator='nearestNeighbor' )
 5891
 5892  # fix the reference region - keep top two parts
 5893  def get_biggest_part( x, labeln ):
 5894      temp33 = ants.threshold_image( x, labeln, labeln ).iMath("GetLargestComponent")
 5895      x[ x == labeln] = 0
 5896      x[ temp33 == 1 ] = labeln
 5897
 5898  get_biggest_part( labels2nm, 33 )
 5899  get_biggest_part( labels2nm, 34 )
 5900
 5901  if verbose:
 5902      print( "map summary measurements to wide format" )
 5903  nmdf = antspyt1w.map_intensity_to_dataframe(
 5904          'CIT168_Reinf_Learn_v1_label_descriptions_pad',
 5905          nm_avg_cropped,
 5906          labels2nm)
 5907  if verbose:
 5908      print( "merge to wide format" )
 5909  nmdf_wide = antspyt1w.merge_hierarchical_csvs_to_wide_format(
 5910              {'NM' : nmdf},
 5911              col_names = ['Mean'] )
 5912
 5913  rr_mask = ants.mask_image( labels2nm, labels2nm, [33,34] , binarize=True )
 5914  sn_mask = ants.mask_image( labels2nm, labels2nm, [7,9,23,25] , binarize=True )
 5915  nmavgsnr = mask_snr( nm_avg_cropped, rr_mask, sn_mask, bias_correct = False )
 5916
 5917  snavg = nm_avg_cropped[ sn_mask == 1].mean()
 5918  rravg = nm_avg_cropped[ rr_mask == 1].mean()
 5919  snstd = nm_avg_cropped[ sn_mask == 1].std()
 5920  rrstd = nm_avg_cropped[ rr_mask == 1].std()
 5921  vol_element = np.prod( ants.get_spacing(sn_mask) )
 5922  snvol = vol_element * sn_mask.sum()
 5923
 5924  # get the mean voxel position of the SN
 5925  if snvol > 0:
 5926      sn_z = ants.transform_physical_point_to_index( sn_mask, ants.get_center_of_mass(sn_mask ))[2]
 5927      sn_z = sn_z/sn_mask.shape[2] # around 0.5 would be nice
 5928  else:
 5929      sn_z = math.nan
 5930
 5931  nm_evr = 0.0
 5932  if cropper2nm.sum() > 0:
 5933    nm_evr = antspyt1w.patch_eigenvalue_ratio( nm_avg, 512, [6,6,6], 
 5934        evdepth = 0.9, mask=cropper2nm )
 5935
 5936  simg = ants.smooth_image( nm_avg_cropped, np.min(ants.get_spacing(nm_avg_cropped)) )
 5937  k = 2.0
 5938  rrthresh = (rravg + k * rrstd)
 5939  nmabovekthresh_mask = sn_mask * ants.threshold_image( simg, rrthresh, math.inf)
 5940  snvolabovethresh = vol_element * nmabovekthresh_mask.sum()
 5941  snintmeanabovethresh = float( ( simg * nmabovekthresh_mask ).mean() )
 5942  snintsumabovethresh = float( ( simg * nmabovekthresh_mask ).sum() )
 5943
 5944  k = 3.0
 5945  rrthresh = (rravg + k * rrstd)
 5946  nmabovekthresh_mask3 = sn_mask * ants.threshold_image( simg, rrthresh, math.inf)
 5947  snvolabovethresh3 = vol_element * nmabovekthresh_mask3.sum()
 5948
 5949  k = 1.0
 5950  rrthresh = (rravg + k * rrstd)
 5951  nmabovekthresh_mask1 = sn_mask * ants.threshold_image( simg, rrthresh, math.inf)
 5952  snvolabovethresh1 = vol_element * nmabovekthresh_mask1.sum()
 5953  
 5954  if verbose:
 5955    print( "nm vol @2std above rrmean: " + str( snvolabovethresh ) )
 5956    print( "nm intmean @2std above rrmean: " + str( snintmeanabovethresh ) )
 5957    print( "nm intsum @2std above rrmean: " + str( snintsumabovethresh ) )
 5958    print( "nm done" )
 5959
 5960  return convert_np_in_dict( {
 5961      'NM_avg' : nm_avg,
 5962      'NM_avg_cropped' : nm_avg_cropped,
 5963      'NM_labels': labels2nm,
 5964      'NM_cropped': crop_nm_list,
 5965      'NM_midbrainROI': cropper2nm,
 5966      'NM_dataframe': nmdf,
 5967      'NM_dataframe_wide': nmdf_wide,
 5968      't1_to_NM': slabreg['warpedmovout'],
 5969      't1_to_NM_transform' : slabreg['fwdtransforms'],
 5970      'NM_avg_signaltonoise' : nmavgsnr,
 5971      'NM_avg_substantianigra' : snavg,
 5972      'NM_std_substantianigra' : snstd,
 5973      'NM_volume_substantianigra' : snvol,
 5974      'NM_volume_substantianigra_1std' : snvolabovethresh1,
 5975      'NM_volume_substantianigra_2std' : snvolabovethresh,
 5976      'NM_intmean_substantianigra_2std' : snintmeanabovethresh,
 5977      'NM_intsum_substantianigra_2std' : snintsumabovethresh,
 5978      'NM_volume_substantianigra_3std' : snvolabovethresh3,
 5979      'NM_avg_refregion' : rravg,
 5980      'NM_std_refregion' : rrstd,
 5981      'NM_min' : nm_avg_cropped.min(),
 5982      'NM_max' : nm_avg_cropped.max(),
 5983      'NM_mean' : nm_avg_cropped.numpy().mean(),
 5984      'NM_sd' : np.std( nm_avg_cropped.numpy() ),
 5985      'NM_q0pt05' : np.quantile( nm_avg_cropped.numpy(), 0.05 ),
 5986      'NM_q0pt10' : np.quantile( nm_avg_cropped.numpy(), 0.10 ),
 5987      'NM_q0pt90' : np.quantile( nm_avg_cropped.numpy(), 0.90 ),
 5988      'NM_q0pt95' : np.quantile( nm_avg_cropped.numpy(), 0.95 ),
 5989      'NM_substantianigra_z_coordinate' : sn_z,
 5990      'NM_evr' : nm_evr,
 5991      'NM_count': len( list_nm_images )
 5992       } )
 5993
 5994
 5995
 5996def estimate_optimal_pca_components(data, variance_threshold=0.80, plot=False):
 5997    """
 5998    Estimate the optimal number of PCA components to represent the given data.
 5999
 6000    :param data: The data matrix (samples x features).
 6001    :param variance_threshold: Threshold for cumulative explained variance (default 0.95).
 6002    :param plot: If True, plot the cumulative explained variance graph (default False).
 6003    :return: The optimal number of principal components.
 6004    """
 6005    import numpy as np
 6006    from sklearn.decomposition import PCA
 6007    import matplotlib.pyplot as plt
 6008
 6009    # Perform PCA
 6010    pca = PCA()
 6011    pca.fit(data)
 6012
 6013    # Calculate cumulative explained variance
 6014    cumulative_variance = np.cumsum(pca.explained_variance_ratio_)
 6015
 6016    # Determine the number of components for desired explained variance
 6017    n_components = np.where(cumulative_variance >= variance_threshold)[0][0] + 1
 6018
 6019    # Optionally plot the explained variance
 6020    if plot:
 6021        plt.figure(figsize=(8, 4))
 6022        plt.plot(cumulative_variance, linewidth=2)
 6023        plt.axhline(y=variance_threshold, color='r', linestyle='--')
 6024        plt.axvline(x=n_components - 1, color='r', linestyle='--')
 6025        plt.xlabel('Number of Components')
 6026        plt.ylabel('Cumulative Explained Variance')
 6027        plt.title('Explained Variance by Number of Principal Components')
 6028        plt.show()
 6029
 6030    return n_components
 6031
 6032import numpy as np
 6033
 6034def compute_PerAF_voxel(time_series):
 6035    """
 6036    Compute the Percentage Amplitude Fluctuation (PerAF) for a given time series.
 6037
 6038    10.1371/journal.pone.0227021
 6039
 6040    PerAF = 100/n * sum(|(x_i - m)/m|) 
 6041    where m = 1/n * sum(x_i), x_i is the signal intensity at each time point, 
 6042    and n is the total number of time points.
 6043
 6044    :param time_series: Numpy array of time series data
 6045    :return: Computed PerAF value
 6046    """
 6047    n = len(time_series)
 6048    m = np.mean(time_series)
 6049    perAF = 100 / n * np.sum(np.abs((time_series - m) / m))
 6050    return perAF
 6051
 6052def calculate_trimmed_mean(data, proportion_to_trim):
 6053    """
 6054    Calculate the trimmed mean for a given data array.
 6055
 6056    :param data: A numpy array of data.
 6057    :param proportion_to_trim: Proportion (0 to 0.5) of data to trim from each end.
 6058    :return: The trimmed mean of the data.
 6059    """
 6060    from scipy import stats
 6061    return stats.trim_mean(data, proportion_to_trim)
 6062
 6063def PerAF( x, mask, globalmean=True ):
 6064    """
 6065    Compute the Percentage Amplitude Fluctuation (PerAF) for a given time series.
 6066
 6067    10.1371/journal.pone.0227021
 6068
 6069    PerAF = 100/n * sum(|(x_i - m)/m|) 
 6070    where m = 1/n * sum(x_i), x_i is the signal intensity at each time point, 
 6071    and n is the total number of time points.
 6072
 6073    :param x: time series antsImage
 6074    :param mask: brain mask
 6075    :param globalmean: boolean if True divide by the globalmean in the brain mask
 6076    :return: Computed PerAF image
 6077    """
 6078    time_series = ants.timeseries_to_matrix( x, mask )
 6079    n = time_series.shape[1]
 6080    vec = np.zeros( n )
 6081    for i in range(n):
 6082        vec[i] = compute_PerAF_voxel( time_series[:,i] )
 6083    outimg = ants.make_image( mask, vec )
 6084    if globalmean:
 6085        outimg = outimg / calculate_trimmed_mean( vec, 0.01 )
 6086    return outimg
 6087
 6088
 6089
 6090def resting_state_fmri_networks( fmri, fmri_template, t1, t1segmentation,
 6091    f=[0.03, 0.08],
 6092    FD_threshold=5.0,
 6093    spa = None,
 6094    spt = None,
 6095    nc = 5,
 6096    outlier_threshold=0.250,
 6097    ica_components = 0,
 6098    impute = True,
 6099    censor = True,
 6100    despike = 2.5,
 6101    motion_as_nuisance = True,
 6102    powers = False,
 6103    upsample = 3.0,
 6104    clean_tmp = None,
 6105    paramset='unset',
 6106    verbose=False ):
 6107  """
 6108  Compute resting state network correlation maps based on the J Power labels.
 6109  This will output a map for each of the major network systems.  This function 
 6110  will by optionally upsample data to 2mm during the registration process if data 
 6111  is below that resolution.
 6112
 6113  registration - despike - anatomy - smooth - nuisance - bandpass - regress.nuisance - censor - falff - correlations
 6114
 6115  Arguments
 6116  ---------
 6117  fmri : BOLD fmri antsImage
 6118
 6119  fmri_template : reference space for BOLD
 6120
 6121  t1 : ANTsImage
 6122    input 3-D T1 brain image (brain extracted)
 6123
 6124  t1segmentation : ANTsImage
 6125    t1 segmentation - a six tissue segmentation image in T1 space
 6126
 6127  f : band pass limits for frequency filtering; we use high-pass here as per Shirer 2015
 6128
 6129  spa : gaussian smoothing for spatial component (physical coordinates)
 6130
 6131  spt : gaussian smoothing for temporal component
 6132
 6133  nc  : number of components for compcor filtering; if less than 1 we estimate on the fly based on explained variance; 10 wrt Shirer 2015 5 from csf and 5 from wm
 6134
 6135  ica_components : integer if greater than 0 then include ica components
 6136
 6137  impute : boolean if True, then use imputation in f/ALFF, PerAF calculation
 6138
 6139  censor : boolean if True, then use censoring (censoring)
 6140
 6141  despike : if this is greater than zero will run voxel-wise despiking in the 3dDespike (afni) sense; after motion-correction
 6142
 6143  motion_as_nuisance: boolean will add motion and first derivative of motion as nuisance
 6144
 6145  powers : boolean if True use Powers nodes otherwise 2023 Yeo 500 homotopic nodes (10.1016/j.neuroimage.2023.120010)
 6146
 6147  upsample : float optionally isotropically upsample data to upsample (the parameter value) in mm during the registration process if data is below that resolution; if the input spacing is less than that provided by the user, the data will simply be resampled to isotropic resolution
 6148
 6149  clean_tmp : will automatically try to clean the tmp directory - not recommended but can be used in distributed computing systems to help prevent failures due to accumulation of tmp files when doing large-scale processing.  if this is set, the float value clean_tmp will be interpreted as the age in hours of files to be cleaned.
 6150
 6151  verbose : boolean
 6152
 6153  Returns
 6154  ---------
 6155  a dictionary containing the derived network maps
 6156
 6157  References
 6158  ---------
 6159
 6160  10.1162/netn_a_00071 "Methods that included global signal regression were the most consistently effective de-noising strategies."
 6161
 6162  10.1016/j.neuroimage.2019.116157 "frontal and default model networks are most reliable whereas subcortical neteworks are least reliable"  "the most comprehensive studies of pipeline effects on edge-level reliability have been done by shirer (2015) and Parkes (2018)" "slice timing correction has minimal impact" "use of low-pass or narrow filter (discarding  high frequency information) reduced both reliability and signal-noise separation"
 6163
 6164  10.1016/j.neuroimage.2017.12.073: Our results indicate that (1) simple linear regression of regional fMRI time series against head motion parameters and WM/CSF signals (with or without expansion terms) is not sufficient to remove head motion artefacts; (2) aCompCor pipelines may only be viable in low-motion data; (3) volume censoring performs well at minimising motion-related artefact but a major benefit of this approach derives from the exclusion of high-motion individuals; (4) while not as effective as volume censoring, ICA-AROMA performed well across our benchmarks for relatively low cost in terms of data loss; (5) the addition of global signal regression improved the performance of nearly all pipelines on most benchmarks, but exacerbated the distance-dependence of correlations between motion and functional connec- tivity; and (6) group comparisons in functional connectivity between healthy controls and schizophrenia patients are highly dependent on preprocessing strategy. We offer some recommendations for best practice and outline simple analyses to facilitate transparent reporting of the degree to which a given set of findings may be affected by motion-related artefact.
 6165
 6166  10.1016/j.dcn.2022.101087 : We found that: 1) the most efficacious pipeline for both noise removal and information recovery included censoring, GSR, bandpass filtering, and head motion parameter (HMP) regression, 2) ICA-AROMA performed similarly to HMP regression and did not obviate the need for censoring, 3) GSR had a minimal impact on connectome fingerprinting but improved ISC, and 4) the strictest censoring approaches reduced motion correlated edges but negatively impacted identifiability.
 6167
 6168  """
 6169
 6170  import warnings
 6171
 6172  if clean_tmp is not None:
 6173    clean_tmp_directory( age_hours = clean_tmp )
 6174
 6175  if nc > 1:
 6176    nc = int(nc)
 6177  else:
 6178    nc=float(nc)
 6179
 6180  type_of_transform="antsRegistrationSyNQuickRepro[r]" # , # should probably not change this
 6181  remove_it=True
 6182  output_directory = tempfile.mkdtemp()
 6183  output_directory_w = output_directory + "/ts_t1_reg/"
 6184  os.makedirs(output_directory_w,exist_ok=True)
 6185  ofnt1tx = tempfile.NamedTemporaryFile(delete=False,suffix='t1_deformation',dir=output_directory_w).name
 6186
 6187  import numpy as np
 6188# Assuming core and utils are modules or packages with necessary functions
 6189
 6190  if upsample > 0.0:
 6191      spc = ants.get_spacing( fmri )
 6192      minspc = upsample
 6193      if min(spc[0:3]) < minspc:
 6194          minspc = min(spc[0:3])
 6195      newspc = [minspc,minspc,minspc]
 6196      fmri_template = ants.resample_image( fmri_template, newspc, interp_type=0 )
 6197
 6198  def temporal_derivative_same_shape(array):
 6199    """
 6200    Compute the temporal derivative of a 2D numpy array along the 0th axis (time)
 6201    and ensure the output has the same shape as the input.
 6202
 6203    :param array: 2D numpy array with time as the 0th axis.
 6204    :return: 2D numpy array of the temporal derivative with the same shape as input.
 6205    """
 6206    derivative = np.diff(array, axis=0)
 6207    
 6208    # Append a row to maintain the same shape
 6209    # You can choose to append a row of zeros or the last row of the derivative
 6210    # Here, a row of zeros is appended
 6211    zeros_row = np.zeros((1, array.shape[1]))
 6212    return np.vstack((zeros_row, derivative ))
 6213
 6214  def compute_tSTD(M, quantile, x=0, axis=0):
 6215    stdM = np.std(M, axis=axis)
 6216    # set bad values to x
 6217    stdM[stdM == 0] = x
 6218    stdM[np.isnan(stdM)] = x
 6219    tt = round(quantile * 100)
 6220    threshold_std = np.percentile(stdM, tt)
 6221    return {'tSTD': stdM, 'threshold_std': threshold_std}
 6222
 6223  def get_compcor_matrix(boldImage, mask, quantile):
 6224    """
 6225    Compute the compcor matrix.
 6226
 6227    :param boldImage: The bold image.
 6228    :param mask: The mask to apply, if None, it will be computed.
 6229    :param quantile: Quantile for computing threshold in tSTD.
 6230    :return: The compor matrix.
 6231    """
 6232    if mask is None:
 6233        temp = ants.slice_image(boldImage, axis=boldImage.dimension - 1, idx=0)
 6234        mask = ants.get_mask(temp)
 6235
 6236    imagematrix = ants.timeseries_to_matrix(boldImage, mask)
 6237    temp = compute_tSTD(imagematrix, quantile, 0)
 6238    tsnrmask = ants.make_image(mask, temp['tSTD'])
 6239    tsnrmask = ants.threshold_image(tsnrmask, temp['threshold_std'], temp['tSTD'].max())
 6240    M = ants.timeseries_to_matrix(boldImage, tsnrmask)
 6241    return M
 6242
 6243
 6244  from sklearn.decomposition import FastICA
 6245  def find_indices(lst, value):
 6246    return [index for index, element in enumerate(lst) if element > value]
 6247
 6248  def mean_of_list(lst):
 6249    if not lst:  # Check if the list is not empty
 6250        return 0  # Return 0 or appropriate value for an empty list
 6251    return sum(lst) / len(lst)
 6252  fmrispc = list( ants.get_spacing( fmri ) )
 6253  if spa is None:
 6254    spa = mean_of_list( fmrispc[0:3] ) * 1.0
 6255  if spt is None:
 6256    spt = fmrispc[3] * 0.5
 6257      
 6258  import numpy as np
 6259  import pandas as pd
 6260  import re
 6261  import math
 6262  # point data resources
 6263  A = np.zeros((1,1))
 6264  dfnname='DefaultMode'
 6265  if powers:
 6266      powers_areal_mni_itk = pd.read_csv( get_data('powers_mni_itk', target_extension=".csv")) # power coordinates
 6267      coords='powers'
 6268  else:
 6269      powers_areal_mni_itk = pd.read_csv( get_data('ppmi_template_500Parcels_Yeo2011_17Networks_2023_homotopic', target_extension=".csv")) # yeo 2023 coordinates
 6270      coords='yeo_17_500_2023'
 6271  fmri = ants.iMath( fmri, 'Normalize' )
 6272  bmask = antspynet.brain_extraction( fmri_template, 'bold' ).threshold_image(0.5,1).iMath("FillHoles")
 6273  if verbose:
 6274      print("Begin rsfmri motion correction")
 6275  debug=False
 6276  if debug:
 6277      ants.image_write( fmri_template, '/tmp/fmri_template.nii.gz' )
 6278      ants.image_write( fmri, '/tmp/fmri.nii.gz' )
 6279      print("debug wrote fmri and fmri_template")
 6280  # mot-co
 6281  corrmo = timeseries_reg(
 6282    fmri, fmri_template,
 6283    type_of_transform=type_of_transform,
 6284    total_sigma=0.5,
 6285    fdOffset=2.0,
 6286    trim = 8,
 6287    output_directory=None,
 6288    verbose=verbose,
 6289    syn_metric='CC',
 6290    syn_sampling=2,
 6291    reg_iterations=[40,20,5],
 6292    return_numpy_motion_parameters=True )
 6293  
 6294  if verbose:
 6295      print("End rsfmri motion correction")
 6296      print("--maximum motion : " + str(corrmo['FD'].max()) )
 6297      print("=== next anatomically based mapping ===")
 6298
 6299  despiking_count = np.zeros( corrmo['motion_corrected'].shape[3] )
 6300  if despike > 0.0:
 6301      corrmo['motion_corrected'], despiking_count = despike_time_series_afni( corrmo['motion_corrected'], c1=despike )
 6302
 6303  despiking_count_summary = despiking_count.sum() / np.prod( corrmo['motion_corrected'].shape )
 6304  high_motion_count=(corrmo['FD'] > FD_threshold ).sum()
 6305  high_motion_pct=high_motion_count / fmri.shape[3]
 6306
 6307  # filter mask based on TSNR
 6308  mytsnr = tsnr( corrmo['motion_corrected'], bmask )
 6309  mytsnrThresh = np.quantile( mytsnr.numpy(), 0.995 )
 6310  tsnrmask = ants.threshold_image( mytsnr, 0, mytsnrThresh ).morphology("close",2)
 6311  bmask = bmask * tsnrmask
 6312
 6313  # anatomical mapping
 6314  und = fmri_template * bmask
 6315  t1reg = ants.registration( und, t1,
 6316     "antsRegistrationSyNQuickRepro[s]", outprefix=ofnt1tx )
 6317  if verbose:
 6318    print("t1 2 bold done")
 6319  gmseg = ants.threshold_image( t1segmentation, 2, 2 )
 6320  gmseg = gmseg + ants.threshold_image( t1segmentation, 4, 4 )
 6321  gmseg = ants.threshold_image( gmseg, 1, 4 )
 6322  gmseg = ants.iMath( gmseg, 'MD', 1 ) # FIXMERSF
 6323  gmseg = ants.apply_transforms( und, gmseg,
 6324    t1reg['fwdtransforms'], interpolator = 'nearestNeighbor' ) * bmask
 6325  csfAndWM = ( ants.threshold_image( t1segmentation, 1, 1 ) +
 6326               ants.threshold_image( t1segmentation, 3, 3 ) ).morphology("erode",1)
 6327  csfAndWM = ants.apply_transforms( und, csfAndWM,
 6328    t1reg['fwdtransforms'], interpolator = 'nearestNeighbor' )  * bmask
 6329  csf = ants.threshold_image( t1segmentation, 1, 1 )
 6330  csf = ants.apply_transforms( und, csf, t1reg['fwdtransforms'], interpolator = 'nearestNeighbor' )  * bmask
 6331  wm = ants.threshold_image( t1segmentation, 3, 3 ).morphology("erode",1)
 6332  wm = ants.apply_transforms( und, wm, t1reg['fwdtransforms'], interpolator = 'nearestNeighbor' )  * bmask
 6333  if powers:
 6334    ch2 = mm_read( ants.get_ants_data( "ch2" ) )
 6335  else:
 6336    ch2 = mm_read( get_data( "PPMI_template0_brain", target_extension='.nii.gz' ) )
 6337  treg = ants.registration( 
 6338    # this is to make the impact of resolution consistent
 6339    ants.resample_image(t1, [1.0,1.0,1.0], interp_type=0), 
 6340    ch2, "antsRegistrationSyNQuickRepro[s]" )
 6341  if powers:
 6342    concatx2 = treg['invtransforms'] + t1reg['invtransforms']
 6343    pts2bold = ants.apply_transforms_to_points( 3, powers_areal_mni_itk, concatx2,
 6344        whichtoinvert = ( True, False, True, False ) )
 6345    locations = pts2bold.iloc[:,:3].values
 6346    ptImg = ants.make_points_image( locations, bmask, radius = 2 )
 6347  else:
 6348    concatx2 = t1reg['fwdtransforms'] + treg['fwdtransforms']    
 6349    rsfsegfn = get_data('ppmi_template_500Parcels_Yeo2011_17Networks_2023_homotopic', target_extension=".nii.gz")
 6350    rsfsegimg = ants.image_read( rsfsegfn )
 6351    ptImg = ants.apply_transforms( und, rsfsegimg, concatx2, interpolator='nearestNeighbor' ) * bmask
 6352    pts2bold = powers_areal_mni_itk
 6353    # ants.plot( und, ptImg, crop=True, axis=2 )
 6354
 6355  # optional smoothing
 6356  tr = ants.get_spacing( corrmo['motion_corrected'] )[3]
 6357  smth = ( spa, spa, spa, spt ) # this is for sigmaInPhysicalCoordinates = TRUE
 6358  simg = ants.smooth_image( corrmo['motion_corrected'], smth, sigma_in_physical_coordinates = True )
 6359
 6360  # collect censoring indices
 6361  hlinds = find_indices( corrmo['FD'], FD_threshold )
 6362  if verbose:
 6363    print("high motion indices")
 6364    print( hlinds )
 6365  if outlier_threshold < 1.0 and outlier_threshold > 0.0:
 6366    fmrimotcorr, hlinds2 = loop_timeseries_censoring( corrmo['motion_corrected'], 
 6367      threshold=outlier_threshold, verbose=verbose )
 6368    hlinds.extend( hlinds2 )
 6369    del fmrimotcorr
 6370  hlinds = list(set(hlinds)) # make unique
 6371
 6372  # nuisance
 6373  globalmat = ants.timeseries_to_matrix( corrmo['motion_corrected'], bmask )
 6374  globalsignal = np.nanmean( globalmat, axis = 1 )
 6375  del globalmat
 6376  compcorquantile=0.50
 6377  nc_wm=nc_csf=nc
 6378  if nc < 1:
 6379    globalmat = get_compcor_matrix( corrmo['motion_corrected'], wm, compcorquantile )
 6380    nc_wm = int(estimate_optimal_pca_components( data=globalmat, variance_threshold=nc))
 6381    globalmat = get_compcor_matrix( corrmo['motion_corrected'], csf, compcorquantile )
 6382    nc_csf = int(estimate_optimal_pca_components( data=globalmat, variance_threshold=nc))
 6383    del globalmat
 6384  if verbose:
 6385    print("include compcor components as nuisance: csf " + str(nc_csf) + " wm " + str(nc_wm))
 6386  mycompcor_csf = ants.compcor( corrmo['motion_corrected'],
 6387    ncompcor=nc_csf, quantile=compcorquantile, mask = csf,
 6388    filter_type='polynomial', degree=2 )
 6389  mycompcor_wm = ants.compcor( corrmo['motion_corrected'],
 6390    ncompcor=nc_wm, quantile=compcorquantile, mask = wm,
 6391    filter_type='polynomial', degree=2 )
 6392  nuisance = np.c_[ mycompcor_csf[ 'components' ], mycompcor_wm[ 'components' ] ]
 6393
 6394  if motion_as_nuisance:
 6395      if verbose:
 6396          print("include motion as nuisance")
 6397          print( corrmo['motion_parameters'].shape )
 6398      deriv = temporal_derivative_same_shape( corrmo['motion_parameters']  )
 6399      nuisance = np.c_[ nuisance, corrmo['motion_parameters'], deriv ]
 6400
 6401  if ica_components > 0:
 6402    if verbose:
 6403        print("include ica components as nuisance: " + str(ica_components))
 6404    ica = FastICA(n_components=ica_components, max_iter=10000, tol=0.001, random_state=42 )
 6405    globalmat = ants.timeseries_to_matrix( corrmo['motion_corrected'], csfAndWM )
 6406    nuisance_ica = ica.fit_transform(globalmat)  # Reconstruct signals
 6407    nuisance = np.c_[ nuisance, nuisance_ica ]
 6408    del globalmat
 6409
 6410  # concat all nuisance data
 6411  # nuisance = np.c_[ nuisance, mycompcor['basis'] ]
 6412  # nuisance = np.c_[ nuisance, corrmo['FD'] ]
 6413  nuisance = np.c_[ nuisance, globalsignal ]
 6414
 6415  if impute:
 6416    simgimp = impute_timeseries( simg, hlinds, method='linear')
 6417  else:
 6418    simgimp = simg
 6419
 6420  # falff/alff stuff  def alff_image( x, mask, flo=0.01, fhi=0.1, nuisance=None ):
 6421  myfalff=alff_image( simgimp, bmask, flo=f[0], fhi=f[1], nuisance=nuisance  )
 6422
 6423  # bandpass any data collected before here -- if bandpass requested
 6424  if f[0] > 0 and f[1] < 1.0:
 6425    if verbose:
 6426        print( "bandpass: " + str(f[0]) + " <=> " + str( f[1] ) )
 6427    nuisance = ants.bandpass_filter_matrix( nuisance, tr = tr, lowf=f[0], highf=f[1] ) # some would argue against this
 6428    globalmat = ants.timeseries_to_matrix( simg, bmask )
 6429    globalmat = ants.bandpass_filter_matrix( globalmat, tr = tr, lowf=f[0], highf=f[1] ) # some would argue against this
 6430    simg = ants.matrix_to_timeseries( simg, globalmat, bmask )
 6431
 6432  if verbose:
 6433    print("now regress nuisance")
 6434
 6435
 6436  if len( hlinds ) > 0 :
 6437    if censor:
 6438        nuisance = remove_elements_from_numpy_array( nuisance, hlinds  )
 6439        simg = remove_volumes_from_timeseries( simg, hlinds )
 6440
 6441  gmmat = ants.timeseries_to_matrix( simg, bmask )
 6442  gmmat = ants.regress_components( gmmat, nuisance )
 6443  simg = ants.matrix_to_timeseries(simg, gmmat, bmask)
 6444
 6445
 6446  # structure the output data
 6447  outdict = {}
 6448  outdict['paramset'] = paramset
 6449  outdict['upsampling'] = upsample
 6450  outdict['coords'] = coords
 6451  outdict['dfnname']=dfnname
 6452  outdict['meanBold'] = und
 6453
 6454  # add correlation matrix that captures each node pair
 6455  # some of the spheres overlap so extract separately from each ROI
 6456  if powers:
 6457    nPoints = int(pts2bold['ROI'].max())
 6458    pointrange = list(range(int(nPoints)))
 6459  else:
 6460    nPoints = int(ptImg.max())
 6461    pointrange = list(range(int(nPoints)))
 6462  nVolumes = simg.shape[3]
 6463  meanROI = np.zeros([nVolumes, nPoints])
 6464  roiNames = []
 6465  if debug:
 6466      ptImgAll = und * 0.
 6467  for i in pointrange:
 6468    # specify name for matrix entries that's links back to ROI number and network; e.g., ROI1_Uncertain
 6469    netLabel = re.sub( " ", "", pts2bold.loc[i,'SystemName'])
 6470    netLabel = re.sub( "-", "", netLabel )
 6471    netLabel = re.sub( "/", "", netLabel )
 6472    roiLabel = "ROI" + str(pts2bold.loc[i,'ROI']) + '_' + netLabel
 6473    roiNames.append( roiLabel )
 6474    if powers:
 6475        ptImage = ants.make_points_image(pts2bold.iloc[[i],:3].values, bmask, radius=1).threshold_image( 1, 1e9 )
 6476    else:
 6477        #print("Doing " + pts2bold.loc[i,'SystemName'] + " at " + str(i) )
 6478        #ptImage = ants.mask_image( ptImg, ptImg, level=pts2bold['ROI'][pts2bold['SystemName']==pts2bold.loc[i,'SystemName']],binarize=True)
 6479        ptImage=ants.threshold_image( ptImg, pts2bold.loc[i,'ROI'], pts2bold.loc[i,'ROI'] )
 6480    if debug:
 6481      ptImgAll = ptImgAll + ptImage
 6482    if ptImage.sum() > 0 :
 6483        meanROI[:,i] = ants.timeseries_to_matrix( simg, ptImage).mean(axis=1)
 6484
 6485  if debug:
 6486      ants.image_write( simg, '/tmp/simg.nii.gz' )
 6487      ants.image_write( ptImgAll, '/tmp/ptImgAll.nii.gz' )
 6488      ants.image_write( und, '/tmp/und.nii.gz' )
 6489      ants.image_write( und, '/tmp/und.nii.gz' )
 6490
 6491  # get full correlation matrix
 6492  corMat = np.corrcoef(meanROI, rowvar=False)
 6493  outputMat = pd.DataFrame(corMat)
 6494  outputMat.columns = roiNames
 6495  outputMat['ROIs'] = roiNames
 6496  # add to dictionary
 6497  outdict['fullCorrMat'] = outputMat
 6498
 6499  networks = powers_areal_mni_itk['SystemName'].unique()
 6500  # this is just for human readability - reminds us of which we choose by default
 6501  if powers:
 6502    netnames = ['Cingulo-opercular Task Control', 'Default Mode',
 6503                    'Memory Retrieval', 'Ventral Attention', 'Visual',
 6504                    'Fronto-parietal Task Control', 'Salience', 'Subcortical',
 6505                    'Dorsal Attention']
 6506    numofnets = [3,5,6,7,8,9,10,11,13]
 6507  else:
 6508    netnames = networks
 6509    numofnets = list(range(len(netnames)))
 6510 
 6511  ct = 0
 6512  for mynet in numofnets:
 6513    netname = re.sub( " ", "", networks[mynet] )
 6514    netname = re.sub( "-", "", netname )
 6515    ww = np.where( powers_areal_mni_itk['SystemName'] == networks[mynet] )[0]
 6516    if powers:
 6517        dfnImg = ants.make_points_image(pts2bold.iloc[ww,:3].values, bmask, radius=1).threshold_image( 1, 1e9 )
 6518    else:
 6519        dfnImg = ants.mask_image( ptImg, ptImg, level=pts2bold['ROI'][pts2bold['SystemName']==networks[mynet]],binarize=True)
 6520    if dfnImg.max() >= 1:
 6521        if verbose:
 6522            print("DO: " + coords + " " + netname )
 6523        dfnmat = ants.timeseries_to_matrix( simg, ants.threshold_image( dfnImg, 1, dfnImg.max() ) )
 6524        dfnsignal = np.nanmean( dfnmat, axis = 1 )
 6525        nan_count_dfn = np.count_nonzero( np.isnan( dfnsignal) )
 6526        if nan_count_dfn > 0 :
 6527            warnings.warn( " mynet " + netnames[ mynet ] + " vs " +  " mean-signal has nans " + str( nan_count_dfn ) ) 
 6528        gmmatDFNCorr = np.zeros( gmmat.shape[1] )
 6529        if nan_count_dfn == 0:
 6530            for k in range( gmmat.shape[1] ):
 6531                nan_count_gm = np.count_nonzero( np.isnan( gmmat[:,k]) )
 6532                if debug and False:
 6533                    print( str( k ) +  " nans gm " + str(nan_count_gm)  )
 6534                if nan_count_gm == 0:
 6535                    gmmatDFNCorr[ k ] = pearsonr( dfnsignal, gmmat[:,k] )[0]
 6536        corrImg = ants.make_image( bmask, gmmatDFNCorr  )
 6537        outdict[ netname ] = corrImg * gmseg
 6538    else:
 6539        outdict[ netname ] = None
 6540    ct = ct + 1
 6541
 6542  A = np.zeros( ( len( numofnets ) , len( numofnets ) ) )
 6543  A_wide = np.zeros( ( 1, len( numofnets ) * len( numofnets ) ) )
 6544  newnames=[]
 6545  newnames_wide=[]
 6546  ct = 0
 6547  for i in range( len( numofnets ) ):
 6548      netnamei = re.sub( " ", "", networks[numofnets[i]] )
 6549      netnamei = re.sub( "-", "", netnamei )
 6550      newnames.append( netnamei  )
 6551      ww = np.where( powers_areal_mni_itk['SystemName'] == networks[numofnets[i]] )[0]
 6552      if powers:
 6553          dfnImg = ants.make_points_image(pts2bold.iloc[ww,:3].values, bmask, radius=1).threshold_image( 1, 1e9 )
 6554      else:
 6555          dfnImg = ants.mask_image( ptImg, ptImg, level=pts2bold['ROI'][pts2bold['SystemName']==networks[numofnets[i]]],binarize=True)
 6556      for j in range( len( numofnets ) ):
 6557          netnamej = re.sub( " ", "", networks[numofnets[j]] )
 6558          netnamej = re.sub( "-", "", netnamej )
 6559          newnames_wide.append( netnamei + "_2_" + netnamej )
 6560          A[i,j] = 0
 6561          if dfnImg is not None and netnamej is not None:
 6562            subbit = dfnImg == 1
 6563            if subbit is not None:
 6564                if subbit.sum() > 0 and netnamej in outdict:
 6565                    A[i,j] = outdict[ netnamej ][ subbit ].mean()
 6566          A_wide[0,ct] = A[i,j]
 6567          ct=ct+1
 6568
 6569  A = pd.DataFrame( A )
 6570  A.columns = newnames
 6571  A['networks']=newnames
 6572  A_wide = pd.DataFrame( A_wide )
 6573  A_wide.columns = newnames_wide
 6574  outdict['corr'] = A
 6575  outdict['corr_wide'] = A_wide
 6576  outdict['fmri_template'] = fmri_template
 6577  outdict['brainmask'] = bmask
 6578  outdict['gmmask'] = gmseg
 6579  outdict['alff'] = myfalff['alff']
 6580  outdict['falff'] = myfalff['falff']
 6581  # add global mean and standard deviation for post-hoc z-scoring
 6582  outdict['alff_mean'] = (myfalff['alff'][myfalff['alff']!=0]).mean()
 6583  outdict['alff_sd'] = (myfalff['alff'][myfalff['alff']!=0]).std()
 6584  outdict['falff_mean'] = (myfalff['falff'][myfalff['falff']!=0]).mean()
 6585  outdict['falff_sd'] = (myfalff['falff'][myfalff['falff']!=0]).std()
 6586
 6587  perafimg = PerAF( simgimp, bmask )
 6588  for k in pointrange:
 6589    anatname=( pts2bold['AAL'][k] )
 6590    if isinstance(anatname, str):
 6591        anatname = re.sub("_","",anatname)
 6592    else:
 6593        anatname='Unk'
 6594    if powers:
 6595        kk = f"{k:0>3}"+"_"
 6596    else:
 6597        kk = f"{k % int(nPoints/2):0>3}"+"_"
 6598    fname='falffPoint'+kk+anatname
 6599    aname='alffPoint'+kk+anatname
 6600    pname='perafPoint'+kk+anatname
 6601    localsel = ptImg == k
 6602    if localsel.sum() > 0 : # check if non-empty
 6603        outdict[fname]=(outdict['falff'][localsel]).mean()
 6604        outdict[aname]=(outdict['alff'][localsel]).mean()
 6605        outdict[pname]=(perafimg[localsel]).mean()
 6606    else:
 6607        outdict[fname]=math.nan
 6608        outdict[aname]=math.nan
 6609        outdict[pname]=math.nan
 6610
 6611  rsfNuisance = pd.DataFrame( nuisance )
 6612  if remove_it:
 6613    import shutil
 6614    shutil.rmtree(output_directory, ignore_errors=True )
 6615
 6616  if not powers:
 6617    dfnsum=outdict['DefaultA']+outdict['DefaultB']+outdict['DefaultC']
 6618    outdict['DefaultMode']=dfnsum
 6619    dfnsum=outdict['VisCent']+outdict['VisPeri']
 6620    outdict['Visual']=dfnsum
 6621
 6622  nonbrainmask = ants.iMath( bmask, "MD",2) - bmask
 6623  trimmask = ants.iMath( bmask, "ME",2)
 6624  edgemask = ants.iMath( bmask, "ME",1) - trimmask
 6625  outdict['motion_corrected'] = corrmo['motion_corrected']
 6626  outdict['nuisance'] = rsfNuisance
 6627  outdict['PerAF'] = perafimg
 6628  outdict['tsnr'] = mytsnr
 6629  outdict['ssnr'] = slice_snr( corrmo['motion_corrected'], csfAndWM, gmseg )
 6630  outdict['dvars'] = dvars( corrmo['motion_corrected'], gmseg )
 6631  outdict['bandpass_freq_0']=f[0]
 6632  outdict['bandpass_freq_1']=f[1]
 6633  outdict['censor']=int(censor)
 6634  outdict['spatial_smoothing']=spa
 6635  outdict['outlier_threshold']=outlier_threshold
 6636  outdict['FD_threshold']=outlier_threshold
 6637  outdict['high_motion_count'] = high_motion_count
 6638  outdict['high_motion_pct'] = high_motion_pct
 6639  outdict['despiking_count_summary'] = despiking_count_summary
 6640  outdict['FD_max'] = corrmo['FD'].max()
 6641  outdict['FD_mean'] = corrmo['FD'].mean()
 6642  outdict['FD_sd'] = corrmo['FD'].std()
 6643  outdict['bold_evr'] =  antspyt1w.patch_eigenvalue_ratio( und, 512, [16,16,16], evdepth = 0.9, mask = bmask )
 6644  outdict['n_outliers'] = len(hlinds)
 6645  outdict['nc_wm'] = int(nc_wm)
 6646  outdict['nc_csf'] = int(nc_csf)
 6647  outdict['minutes_original_data'] = ( tr * fmri.shape[3] ) / 60.0 # minutes of useful data
 6648  outdict['minutes_censored_data'] = ( tr * simg.shape[3] ) / 60.0 # minutes of useful data
 6649  return convert_np_in_dict( outdict )
 6650
 6651
 6652def calculate_CBF(Delta_M, M_0, mask,
 6653                  Lambda=0.9, T_1=0.67, Alpha=0.68, w=1.0, Tau=1.5):
 6654    """
 6655    Calculate the Cerebral Blood Flow (CBF) where Delta_M and M_0 are antsImages 
 6656    and the other variables are scalars.  Guesses at default values are used here. 
 6657    We use the pCASL equation.  NOT YET TESTED.
 6658
 6659    Parameters:
 6660    Delta_M (antsImage): Change in magnetization (matrix)
 6661    M_0 (antsImage): Initial magnetization (matrix)
 6662    mask ( antsImage ): where to do the calculation
 6663    Lambda (float): Scalar
 6664    T_1 (float): Scalar representing relaxation time
 6665    Alpha (float): Scalar representing flip angle
 6666    w (float): Scalar
 6667    Tau (float): Scalar
 6668
 6669    Returns:
 6670    np.ndarray: CBF values (matrix)
 6671    """
 6672    cbf = M_0 * 0.0
 6673    m0thresh = np.quantile( M_0[mask==1], 0.1 )
 6674    sel = mask == 1 and M_0 > m0thresh
 6675    cbf[ sel ] = Delta_M[ sel ] * 60. * 100. * (Lambda * T_1)/( M_0[sel] * 2.0 * Alpha * 
 6676        (np.exp( -w * T_1) - np.exp(-(Tau + w) * T_1)))
 6677    cbf[ cbf < 0.0]=0.0
 6678    return cbf
 6679
 6680def despike_time_series_afni(image, c1=2.5, c2=4):
 6681    """
 6682    Despike a time series image using L1 polynomial fitting and nonlinear filtering.
 6683    Based on afni 3dDespike
 6684
 6685    :param image: ANTsPy image object containing time series data.
 6686    :param c1: Spike threshold value. Default is 2.5.
 6687    :param c2: Upper range of allowed deviation. Default is 4.
 6688    :return: Despiked ANTsPy image object.
 6689    """
 6690    data = image.numpy()  # Convert to numpy array
 6691    despiked_data = np.copy(data)  # Create a copy for despiked data
 6692    curve = despiked_data * 0.0
 6693
 6694    def l1_fit_polynomial(time_series, degree=2):
 6695        """
 6696        Fit a polynomial of given degree to the time series using least squares.
 6697        
 6698        :param time_series: 1D numpy array of voxel time series data.
 6699        :param degree: Degree of the polynomial to fit.
 6700        :return: Fitted polynomial values for the time series.
 6701        """
 6702        t = np.arange(len(time_series))
 6703        coefs = np.polyfit(t, time_series, degree)
 6704        polynomial = np.polyval(coefs, t)
 6705        return polynomial
 6706
 6707    # L1 fit a smooth-ish curve to each voxel time series
 6708    # Curve fitting for each voxel
 6709    for x in range(data.shape[0]):
 6710        for y in range(data.shape[1]):
 6711            for z in range(data.shape[2]):
 6712                voxel_time_series = data[x, y, z, :]
 6713                curve[x, y, z, :] = l1_fit_polynomial(voxel_time_series, degree=2)
 6714
 6715    # Compute the MAD of the residuals
 6716    residuals = data - curve
 6717    mad = np.median(np.abs(residuals - np.median(residuals, axis=-1, keepdims=True)), axis=-1, keepdims=True)
 6718    sigma = np.sqrt(np.pi / 2) * mad
 6719    # Ensure sigma is not zero to avoid division by zero
 6720    sigma_safe = np.where(sigma == 0, 1e-10, sigma)
 6721
 6722    # Optionally, handle NaN or inf values in data, curve, or sigma
 6723    data = np.nan_to_num(data, nan=0.0, posinf=np.finfo(np.float64).max, neginf=np.finfo(np.float64).min)
 6724    curve = np.nan_to_num(curve, nan=0.0, posinf=np.finfo(np.float64).max, neginf=np.finfo(np.float64).min)
 6725    sigma_safe = np.nan_to_num(sigma_safe, nan=1e-10, posinf=np.finfo(np.float64).max, neginf=np.finfo(np.float64).min)
 6726
 6727    # Despike algorithm
 6728    spike_counts = np.zeros( image.shape[3] )
 6729    for i in range(data.shape[-1]):
 6730        s = (data[..., i] - curve[..., i]) / sigma_safe[..., 0]
 6731        ww = s > c1
 6732        s_prime = np.where( ww, c1 + (c2 - c1) * np.tanh((s - c1) / (c2 - c1)), s)
 6733        spike_counts[i] = ww.sum()
 6734        despiked_data[..., i] = curve[..., i] + s_prime * sigma[..., 0]
 6735
 6736    # Convert back to ANTsPy image
 6737    despiked_image = ants.from_numpy(despiked_data)
 6738    return ants.copy_image_info( image, despiked_image ), spike_counts
 6739
 6740def despike_time_series(image, threshold=3.0, replacement='threshold' ):
 6741    """
 6742    Despike a time series image.
 6743    
 6744    :param image: ANTsPy image object containing time series data.
 6745    :param threshold: z-score value to identify spikes. Default is 3.
 6746    :param replacement: median or threshold - the latter is similar 3DDespike but simpler
 6747    :return: Despiked ANTsPy image object.
 6748    """
 6749    # Convert image to numpy array
 6750    data = image.numpy()
 6751    
 6752    # Calculate the mean and standard deviation along the time axis
 6753    mean = np.mean(data, axis=-1)
 6754    std = np.std(data, axis=-1)
 6755
 6756    # Identify spikes: points where the deviation from the mean exceeds the threshold
 6757    spikes = np.abs(data - mean[..., np.newaxis]) > threshold * std[..., np.newaxis]
 6758
 6759    # Replace spike values
 6760    spike_counts = np.zeros( image.shape[3] )
 6761    for i in range(data.shape[-1]):
 6762        slice = data[..., i]
 6763        spike_locations = spikes[..., i]
 6764        spike_counts[i] = spike_locations.sum()
 6765        if replacement == 'median':
 6766            slice[spike_locations] = np.median(slice)  # Replace with median or another method
 6767        else:
 6768	    # Calculate threshold values (mean ± threshold * std)
 6769            threshold_values = mean + np.sign(slice - mean) * threshold * std
 6770            slice[spike_locations] = threshold_values[spike_locations]
 6771        data[..., i] = slice
 6772    # Convert back to ANTsPy image
 6773    despike_image = ants.from_numpy(data)
 6774    despike_image = ants.copy_image_info( image, despike_image )
 6775    return despike_image, spike_counts
 6776
 6777
 6778
 6779def bold_perfusion_minimal( 
 6780        fmri, 
 6781        m0_image = None,
 6782        spa = (0., 0., 0., 0.),
 6783        nc  = 0,
 6784        tc='alternating',
 6785        n_to_trim=0,
 6786        outlier_threshold=0.250,
 6787        plot_brain_mask=False,
 6788        verbose=False ):
 6789  """
 6790  Estimate perfusion from a BOLD time series image.  Will attempt to figure out the T-C labels from the data.  The function uses defaults to quantify CBF but these will usually not be correct for your own data.  See the function calculate_CBF for an example of how one might do quantification based on the outputs of this function specifically the perfusion, m0 and mask images that are part of the output dictionary.
 6791
 6792  This function is intended for use in debugging/testing or when one lacks a T1w image.
 6793
 6794  Arguments
 6795  ---------
 6796
 6797  fmri : BOLD fmri antsImage
 6798
 6799  m0_image: a pre-defined m0 antsImage
 6800
 6801  spa : gaussian smoothing for spatial and temporal component e.g. (1,1,1,0) in physical space coordinates
 6802
 6803  nc  : number of components for compcor filtering
 6804
 6805  tc: string either alternating or split (default is alternating ie CTCTCT; split is CCCCTTTT)
 6806
 6807  n_to_trim: number of volumes to trim off the front of the time series to account for initial magnetic saturation effects or to allow the signal to reach a steady state. in some cases, trailing volumes or other outlier volumes may need to be rejected.  this code does not currently handle that issue.
 6808
 6809  outlier_threshold (numeric): between zero (remove all) and one (remove none); automatically calculates outlierness and uses it to censor the time series.
 6810
 6811  plot_brain_mask : boolean can help with checking data quality visually
 6812
 6813  verbose : boolean
 6814
 6815  Returns
 6816  ---------
 6817  a dictionary containing the derived network maps
 6818
 6819  """
 6820  import numpy as np
 6821  import pandas as pd
 6822  import re
 6823  import math
 6824  from sklearn.linear_model import RANSACRegressor, TheilSenRegressor, HuberRegressor, QuantileRegressor, LinearRegression, SGDRegressor
 6825  from sklearn.multioutput import MultiOutputRegressor
 6826  from sklearn.preprocessing import StandardScaler
 6827
 6828  def replicate_list(user_list, target_size):
 6829    # Calculate the number of times the list should be replicated
 6830    replication_factor = target_size // len(user_list)
 6831    # Replicate the list and handle any remaining elements
 6832    replicated_list = user_list * replication_factor
 6833    remaining_elements = target_size % len(user_list)
 6834    replicated_list += user_list[:remaining_elements]
 6835    return replicated_list
 6836
 6837  def one_hot_encode(char_list):
 6838    unique_chars = list(set(char_list))
 6839    encoding_dict = {char: [1 if char == c else 0 for c in unique_chars] for char in unique_chars}
 6840    encoded_matrix = np.array([encoding_dict[char] for char in char_list])
 6841    return encoded_matrix
 6842  
 6843  A = np.zeros((1,1))
 6844  fmri_template = ants.get_average_of_timeseries( fmri )
 6845  if n_to_trim is None:
 6846    n_to_trim=0
 6847  mytrim=n_to_trim
 6848  perf_total_sigma = 1.5
 6849  corrmo = timeseries_reg(
 6850    fmri, fmri_template,
 6851    type_of_transform='antsRegistrationSyNRepro[r]',
 6852    total_sigma=perf_total_sigma,
 6853    fdOffset=2.0,
 6854    trim = mytrim,
 6855    output_directory=None,
 6856    verbose=verbose,
 6857    syn_metric='CC',
 6858    syn_sampling=2,
 6859    reg_iterations=[40,20,5] )
 6860  if verbose:
 6861      print("End rsfmri motion correction")
 6862      print("--maximum motion : " + str(corrmo['FD'].max()) )
 6863
 6864  if m0_image is not None:
 6865      m0 = m0_image
 6866
 6867  ntp = corrmo['motion_corrected'].shape[3]
 6868  fmri_template = ants.get_average_of_timeseries( corrmo['motion_corrected'] )
 6869  bmask = antspynet.brain_extraction( fmri_template, 'bold' ).threshold_image(0.5,1).iMath("GetLargestComponent").morphology("close",2).iMath("FillHoles")
 6870  if plot_brain_mask:
 6871    ants.plot( fmri_template, bmask, axis=1, crop=True )
 6872    ants.plot( fmri_template, bmask, axis=2, crop=True )
 6873
 6874  if tc == 'alternating':
 6875      tclist = replicate_list( ['C','T'], ntp )
 6876  else:
 6877      tclist = replicate_list( ['C'], int(ntp/2) ) + replicate_list( ['T'],  int(ntp/2) )
 6878
 6879  tclist = one_hot_encode( tclist[0:ntp ] )
 6880  fmrimotcorr=corrmo['motion_corrected']
 6881  hlinds = None
 6882  if outlier_threshold < 1.0 and outlier_threshold > 0.0:
 6883    fmrimotcorr, hlinds = loop_timeseries_censoring( fmrimotcorr, outlier_threshold, mask=None, verbose=verbose )
 6884    tclist = remove_elements_from_numpy_array( tclist, hlinds)
 6885    corrmo['FD'] = remove_elements_from_numpy_array( corrmo['FD'], hlinds )
 6886
 6887  # redo template and registration at (potentially) upsampled scale
 6888  fmri_template = ants.iMath( ants.get_average_of_timeseries( fmrimotcorr ), "Normalize" )
 6889  corrmo = timeseries_reg(
 6890        fmri, fmri_template,
 6891        type_of_transform='antsRegistrationSyNRepro[r]',
 6892        total_sigma=perf_total_sigma,
 6893        fdOffset=2.0,
 6894        trim = mytrim,
 6895        output_directory=None,
 6896        verbose=verbose,
 6897        syn_metric='CC',
 6898        syn_sampling=2,
 6899        reg_iterations=[40,20,5] )
 6900  if verbose:
 6901        print("End 2nd rsfmri motion correction")
 6902        print("--maximum motion : " + str(corrmo['FD'].max()) )
 6903
 6904  if outlier_threshold < 1.0 and outlier_threshold > 0.0:
 6905    corrmo['motion_corrected'] = remove_volumes_from_timeseries( corrmo['motion_corrected'], hlinds )
 6906    corrmo['FD'] = remove_elements_from_numpy_array( corrmo['FD'], hlinds )
 6907
 6908  bmask = antspynet.brain_extraction( fmri_template, 'bold' ).threshold_image(0.5,1).iMath("GetLargestComponent").morphology("close",2).iMath("FillHoles")
 6909  if plot_brain_mask:
 6910    ants.plot( fmri_template, bmask, axis=1, crop=True )
 6911    ants.plot( fmri_template, bmask, axis=2, crop=True )
 6912
 6913  regression_mask = bmask.clone()
 6914  mytsnr = tsnr( corrmo['motion_corrected'], bmask )
 6915  mytsnrThresh = np.quantile( mytsnr.numpy(), 0.995 )
 6916  tsnrmask = ants.threshold_image( mytsnr, 0, mytsnrThresh ).morphology("close",3)
 6917  bmask = bmask * ants.iMath( tsnrmask, "FillHoles" )
 6918  fmrimotcorr=corrmo['motion_corrected']
 6919  und = fmri_template * bmask
 6920  compcorquantile=0.50
 6921  mycompcor = ants.compcor( fmrimotcorr,
 6922    ncompcor=nc, quantile=compcorquantile, mask = bmask,
 6923    filter_type='polynomial', degree=2 )
 6924  tr = ants.get_spacing( fmrimotcorr )[3]
 6925  simg = ants.smooth_image(fmrimotcorr, spa, sigma_in_physical_coordinates = True )
 6926  nuisance = mycompcor['basis']
 6927  nuisance = np.c_[ nuisance, mycompcor['components'] ]
 6928  if verbose:
 6929    print("make sure nuisance is independent of TC")
 6930  nuisance = ants.regress_components( nuisance, tclist )
 6931  regression_mask = bmask.clone()
 6932  gmmat = ants.timeseries_to_matrix( simg, regression_mask )
 6933  regvars = np.hstack( (nuisance, tclist ))
 6934  coefind = regvars.shape[1]-1
 6935  regvars = regvars[:,range(coefind)]
 6936  predictor_of_interest_idx = regvars.shape[1]-1
 6937  valid_perf_models = ['huber','quantile','theilsen','ransac', 'sgd', 'linear','SM']
 6938  perfusion_regression_model='linear'
 6939  if verbose:
 6940    print( "begin perfusion estimation with " + perfusion_regression_model + " model " )
 6941  regression_model = LinearRegression()
 6942  regression_model.fit( regvars, gmmat )
 6943  coefind = regression_model.coef_.shape[1]-1
 6944  perfimg = ants.make_image( regression_mask, regression_model.coef_[:,coefind] )
 6945  gmseg = ants.image_clone( bmask )
 6946  meangmval = ( perfimg[ gmseg == 1 ] ).mean()
 6947  if meangmval < 0:
 6948      perfimg = perfimg * (-1.0)
 6949  negative_voxels = ( perfimg < 0.0 ).sum() / np.prod( perfimg.shape ) * 100.0
 6950  perfimg[ perfimg < 0.0 ] = 0.0 # non-physiological
 6951
 6952  if m0_image is None:
 6953    m0 = ants.get_average_of_timeseries( fmrimotcorr )
 6954  else:
 6955    # register m0 to current template
 6956    m0reg = ants.registration( fmri_template, m0, 'antsRegistrationSyNRepro[r]', verbose=False )
 6957    m0 = m0reg['warpedmovout']
 6958
 6959  if ntp == 2 :
 6960      img0 = ants.slice_image( corrmo['motion_corrected'], axis=3, idx=0 )
 6961      img1 = ants.slice_image( corrmo['motion_corrected'], axis=3, idx=1 )
 6962      if m0_image is None:
 6963        if img0.mean() < img1.mean():
 6964            perfimg=img0
 6965            m0=img1
 6966        else:
 6967            perfimg=img1
 6968            m0=img0
 6969      else:
 6970        if img0.mean() < img1.mean():
 6971            perfimg=img1-img0
 6972        else:
 6973            perfimg=img0-img1
 6974  
 6975  cbf = calculate_CBF( Delta_M=perfimg, M_0=m0, mask=bmask )
 6976  meangmval = ( perfimg[ gmseg == 1 ] ).mean()        
 6977  meangmvalcbf = ( cbf[ gmseg == 1 ] ).mean()
 6978  if verbose:
 6979    print("perfimg.max() " + str(  perfimg.max() ) )
 6980  outdict = {}
 6981  outdict['meanBold'] = und
 6982  outdict['brainmask'] = bmask
 6983  rsfNuisance = pd.DataFrame( nuisance )
 6984  rsfNuisance['FD']=corrmo['FD']
 6985  outdict['perfusion']=perfimg
 6986  outdict['cbf']=cbf
 6987  outdict['m0']=m0
 6988  outdict['perfusion_gm_mean']=meangmval
 6989  outdict['cbf_gm_mean']=meangmvalcbf
 6990  outdict['motion_corrected'] = corrmo['motion_corrected']
 6991  outdict['brain_mask'] = bmask
 6992  outdict['nuisance'] = rsfNuisance
 6993  outdict['tsnr'] = mytsnr
 6994  outdict['dvars'] = dvars( corrmo['motion_corrected'], gmseg )
 6995  outdict['FD_max'] = rsfNuisance['FD'].max()
 6996  outdict['FD_mean'] = rsfNuisance['FD'].mean()
 6997  outdict['FD_sd'] = rsfNuisance['FD'].std()
 6998  outdict['outlier_volumes']=hlinds
 6999  outdict['negative_voxels']=negative_voxels
 7000  return convert_np_in_dict( outdict )
 7001
 7002
 7003
 7004def warn_if_small_mask( mask: ants.ANTsImage, threshold_fraction: float = 0.05, label: str = ' ' ):
 7005    """
 7006    Warn the user if the number of non-zero voxels in the mask
 7007    is less than a given fraction of the total number of voxels in the mask.
 7008
 7009    Parameters
 7010    ----------
 7011    mask : ants.ANTsImage
 7012        The binary mask to evaluate.
 7013    threshold_fraction : float, optional
 7014        Fraction threshold below which a warning is triggered (default is 0.05).
 7015    
 7016    Returns
 7017    -------
 7018    None
 7019    """
 7020    import warnings
 7021    image_size = np.prod(mask.shape)
 7022    mask_size = np.count_nonzero(mask.numpy())
 7023    if mask_size / image_size < threshold_fraction:
 7024        percentage = 100.0 * mask_size / image_size
 7025        warnings.warn(
 7026            f"[ants] Warning: {label} contains only {mask_size} voxels "
 7027            f"({percentage:.2f}% of image volume). "
 7028            f"This is below the threshold of {threshold_fraction * 100:.2f}% and may lead to unreliable results.",
 7029            UserWarning
 7030        )
 7031
 7032def bold_perfusion( 
 7033    fmri, t1head, t1, t1segmentation, t1dktcit,
 7034                   FD_threshold=0.5,
 7035                   spa = (0., 0., 0., 0.),
 7036                   nc = 3,
 7037                   type_of_transform='antsRegistrationSyNRepro[r]',
 7038                   tc='alternating',
 7039                   n_to_trim=0,
 7040                   m0_image = None,
 7041                   m0_indices=None,
 7042                   outlier_threshold=0.250,
 7043                   add_FD_to_nuisance=False,
 7044                   n3=False,
 7045                   segment_timeseries=False,
 7046                   trim_the_mask=4.25,
 7047                   upsample=True,
 7048                   perfusion_regression_model='linear',
 7049                   verbose=False ):
 7050  """
 7051  Estimate perfusion from a BOLD time series image.  Will attempt to figure out the T-C labels from the data.  The function uses defaults to quantify CBF but these will usually not be correct for your own data.  See the function calculate_CBF for an example of how one might do quantification based on the outputs of this function specifically the perfusion, m0 and mask images that are part of the output dictionary.
 7052
 7053  Arguments
 7054  ---------
 7055  fmri : BOLD fmri antsImage
 7056
 7057  t1head : ANTsImage
 7058    input 3-D T1 brain image (not brain extracted)
 7059
 7060  t1 : ANTsImage
 7061    input 3-D T1 brain image (brain extracted)
 7062
 7063  t1segmentation : ANTsImage
 7064    t1 segmentation - a six tissue segmentation image in T1 space
 7065
 7066  t1dktcit : ANTsImage
 7067    t1 dkt cortex plus cit parcellation
 7068
 7069  spa : gaussian smoothing for spatial and temporal component e.g. (1,1,1,0) in physical space coordinates
 7070
 7071  nc  : number of components for compcor filtering
 7072
 7073  type_of_transform : SyN or Rigid
 7074
 7075  tc: string either alternating or split (default is alternating ie CTCTCT; split is CCCCTTTT)
 7076
 7077  n_to_trim: number of volumes to trim off the front of the time series to account for initial magnetic saturation effects or to allow the signal to reach a steady state. in some cases, trailing volumes or other outlier volumes may need to be rejected.  this code does not currently handle that issue.
 7078
 7079  m0_image: a pre-defined m0 image - we expect this to be 3D.  if it is not, we naively 
 7080    average over the 4th dimension.
 7081
 7082  m0_indices: which indices in the perfusion image are the m0.  if set, n_to_trim will be ignored.
 7083
 7084  outlier_threshold (numeric): between zero (remove all) and one (remove none); automatically calculates outlierness and uses it to censor the time series.
 7085
 7086  add_FD_to_nuisance: boolean
 7087
 7088  n3: boolean
 7089
 7090  segment_timeseries : boolean
 7091
 7092  trim_the_mask : float >= 0 post-hoc method for trimming the mask
 7093
 7094  upsample: boolean
 7095
 7096  perfusion_regression_model: string 'linear', 'ransac', 'theilsen', 'huber', 'quantile', 'sgd'; 'linear' and 'huber' are the only ones that work ok by default and are relatively quick to compute.
 7097
 7098  verbose : boolean
 7099
 7100  Returns
 7101  ---------
 7102  a dictionary containing the derived network maps
 7103
 7104  """
 7105  import numpy as np
 7106  import pandas as pd
 7107  import re
 7108  import math
 7109  from sklearn.linear_model import RANSACRegressor, TheilSenRegressor, HuberRegressor, QuantileRegressor, LinearRegression, SGDRegressor
 7110  from sklearn.multioutput import MultiOutputRegressor
 7111  from sklearn.preprocessing import StandardScaler
 7112
 7113  # remove outlier volumes
 7114  if segment_timeseries:
 7115    lo_vs_high = segment_timeseries_by_meanvalue(fmri)
 7116    fmri = remove_volumes_from_timeseries( fmri, lo_vs_high['lowermeans'] )
 7117
 7118  ex_path = os.path.expanduser( "~/.antspyt1w/" )
 7119  cnxcsvfn = ex_path + "dkt_cortex_cit_deep_brain.csv"
 7120
 7121  if n3:
 7122    fmri = timeseries_n3( fmri )
 7123
 7124  if m0_image is not None:
 7125    if m0_image.dimension == 4:
 7126      m0_image = ants.get_average_of_timeseries( m0_image )
 7127
 7128  def select_regression_model(regression_model, min_samples=10 ):
 7129    if regression_model == 'sgd' :
 7130      sgd_regressor = SGDRegressor(penalty='elasticnet', alpha=1e-5, l1_ratio=0.15, max_iter=20000, tol=1e-3, random_state=42)
 7131      return sgd_regressor
 7132    elif regression_model == 'ransac':
 7133      ransac = RANSACRegressor(
 7134            min_samples=0.8,
 7135            max_trials=10,         # Maximum number of iterations
 7136#            min_samples=min_samples, # Minimum number samples to be chosen as inliers in each iteration
 7137#            stop_probability=0.80,  # Probability to stop the algorithm if a good subset is found
 7138#            stop_n_inliers=40,      # Stop if this number of inliers is found
 7139#            stop_score=0.8,         # Stop if the model score reaches this value
 7140#            n_jobs=int(os.getenv("ITK_GLOBAL_DEFAULT_NUMBER_OF_THREADS")) # Use all available CPU cores for parallel processing
 7141        )
 7142      return ransac
 7143    models = {
 7144        'sgd':SGDRegressor,
 7145        'ransac': RANSACRegressor,
 7146        'theilsen': TheilSenRegressor,
 7147        'huber': HuberRegressor,
 7148        'quantile': QuantileRegressor
 7149    }
 7150    return models.get(regression_model.lower(), LinearRegression)()
 7151
 7152  def replicate_list(user_list, target_size):
 7153    # Calculate the number of times the list should be replicated
 7154    replication_factor = target_size // len(user_list)
 7155    # Replicate the list and handle any remaining elements
 7156    replicated_list = user_list * replication_factor
 7157    remaining_elements = target_size % len(user_list)
 7158    replicated_list += user_list[:remaining_elements]
 7159    return replicated_list
 7160
 7161  def one_hot_encode(char_list):
 7162    unique_chars = list(set(char_list))
 7163    encoding_dict = {char: [1 if char == c else 0 for c in unique_chars] for char in unique_chars}
 7164    encoded_matrix = np.array([encoding_dict[char] for char in char_list])
 7165    return encoded_matrix
 7166  
 7167  A = np.zeros((1,1))
 7168  # fmri = ants.iMath( fmri, 'Normalize' )
 7169  fmri_template, hlinds = loop_timeseries_censoring( fmri, 0.10 )
 7170  fmri_template = ants.get_average_of_timeseries( fmri_template )
 7171  del hlinds
 7172  rig = ants.registration( fmri_template, t1head, 'antsRegistrationSyNRepro[r]' )
 7173  bmask = ants.apply_transforms( fmri_template, ants.threshold_image(t1segmentation,1,6), rig['fwdtransforms'][0], interpolator='genericLabel' )
 7174  if m0_indices is None:
 7175    if n_to_trim is None:
 7176        n_to_trim=0
 7177    mytrim=n_to_trim
 7178  else:
 7179    mytrim = 0
 7180  perf_total_sigma = 1.5
 7181  corrmo = timeseries_reg(
 7182    fmri, fmri_template,
 7183    type_of_transform=type_of_transform,
 7184    total_sigma=perf_total_sigma,
 7185    fdOffset=2.0,
 7186    trim = mytrim,
 7187    output_directory=None,
 7188    verbose=verbose,
 7189    syn_metric='CC',
 7190    syn_sampling=2,
 7191    reg_iterations=[40,20,5] )
 7192  if verbose:
 7193      print("End rsfmri motion correction")
 7194
 7195  if m0_image is not None:
 7196      m0 = m0_image
 7197  elif m0_indices is not None:
 7198    not_m0 = list( range( fmri.shape[3] ) )
 7199    not_m0 = [x for x in not_m0 if x not in m0_indices]
 7200    if verbose:
 7201        print( m0_indices )
 7202        print( not_m0 )
 7203    # then remove it from the time series
 7204    m0 = remove_volumes_from_timeseries( corrmo['motion_corrected'], not_m0 )
 7205    m0 = ants.get_average_of_timeseries( m0 )
 7206    corrmo['motion_corrected'] = remove_volumes_from_timeseries( 
 7207        corrmo['motion_corrected'], m0_indices )
 7208    corrmo['FD'] = remove_elements_from_numpy_array( corrmo['FD'], m0_indices )
 7209    fmri = remove_volumes_from_timeseries( fmri, m0_indices )
 7210
 7211  ntp = corrmo['motion_corrected'].shape[3]
 7212  if tc == 'alternating':
 7213      tclist = replicate_list( ['C','T'], ntp )
 7214  else:
 7215      tclist = replicate_list( ['C'], int(ntp/2) ) + replicate_list( ['T'],  int(ntp/2) )
 7216
 7217  tclist = one_hot_encode( tclist[0:ntp ] )
 7218  fmrimotcorr=corrmo['motion_corrected']
 7219  if outlier_threshold < 1.0 and outlier_threshold > 0.0:
 7220    fmrimotcorr, hlinds = loop_timeseries_censoring( fmrimotcorr, outlier_threshold, mask=None, verbose=verbose )
 7221    tclist = remove_elements_from_numpy_array( tclist, hlinds)
 7222    corrmo['FD'] = remove_elements_from_numpy_array( corrmo['FD'], hlinds )
 7223
 7224  # redo template and registration at (potentially) upsampled scale
 7225  fmri_template = ants.iMath( ants.get_average_of_timeseries( fmrimotcorr ), "Normalize" )
 7226  if upsample:
 7227      spc = ants.get_spacing( fmri )
 7228      minspc = 2.0
 7229      if min(spc[0:3]) < minspc:
 7230          minspc = min(spc[0:3])
 7231      newspc = [minspc,minspc,minspc]
 7232      fmri_template = ants.resample_image( fmri_template, newspc, interp_type=0 )
 7233
 7234  if verbose:
 7235      print( 'fmri_template')
 7236      print( fmri_template )
 7237
 7238  rig = ants.registration( fmri_template, t1head, 'antsRegistrationSyNRepro[r]' )
 7239  bmask = ants.apply_transforms( fmri_template, 
 7240    ants.threshold_image(t1segmentation,1,6), 
 7241    rig['fwdtransforms'][0], 
 7242    interpolator='genericLabel' )
 7243
 7244  warn_if_small_mask( bmask, label='bold_perfusion:bmask')
 7245
 7246  corrmo = timeseries_reg(
 7247        fmri, fmri_template,
 7248        type_of_transform=type_of_transform,
 7249        total_sigma=perf_total_sigma,
 7250        fdOffset=2.0,
 7251        trim = mytrim,
 7252        output_directory=None,
 7253        verbose=verbose,
 7254        syn_metric='CC',
 7255        syn_sampling=2,
 7256        reg_iterations=[40,20,5] )
 7257  if verbose:
 7258        print("End 2nd rsfmri motion correction")
 7259
 7260  if outlier_threshold < 1.0 and outlier_threshold > 0.0:
 7261    corrmo['motion_corrected'] = remove_volumes_from_timeseries( corrmo['motion_corrected'], hlinds )
 7262    corrmo['FD'] = remove_elements_from_numpy_array( corrmo['FD'], hlinds )
 7263
 7264  regression_mask = bmask.clone()
 7265  mytsnr = tsnr( corrmo['motion_corrected'], bmask )
 7266  mytsnrThresh = np.quantile( mytsnr.numpy(), 0.995 )
 7267  tsnrmask = ants.threshold_image( mytsnr, 0, mytsnrThresh ).morphology("close",3)
 7268  bmask = bmask * ants.iMath( tsnrmask, "FillHoles" )
 7269  warn_if_small_mask( bmask, label='bold_perfusion:bmask*tsnrmask')
 7270  fmrimotcorr=corrmo['motion_corrected']
 7271  und = fmri_template * bmask
 7272  t1reg = ants.registration( und, t1, "antsRegistrationSyNRepro[s]" )
 7273  gmseg = ants.threshold_image( t1segmentation, 2, 2 )
 7274  gmseg = gmseg + ants.threshold_image( t1segmentation, 4, 4 )
 7275  gmseg = ants.threshold_image( gmseg, 1, 4 )
 7276  gmseg = ants.iMath( gmseg, 'MD', 1 )
 7277  gmseg = ants.apply_transforms( und, gmseg,
 7278    t1reg['fwdtransforms'], interpolator = 'genericLabel' ) * bmask
 7279  csfseg = ants.threshold_image( t1segmentation, 1, 1 )
 7280  wmseg = ants.threshold_image( t1segmentation, 3, 3 )
 7281  csfAndWM = ( csfseg + wmseg ).morphology("erode",1)
 7282  compcorquantile=0.50
 7283  csfAndWM = ants.apply_transforms( und, csfAndWM,
 7284    t1reg['fwdtransforms'], interpolator = 'nearestNeighbor' )  * bmask
 7285  csfseg = ants.apply_transforms( und, csfseg,
 7286    t1reg['fwdtransforms'], interpolator = 'nearestNeighbor' )  * bmask
 7287  wmseg = ants.apply_transforms( und, wmseg,
 7288    t1reg['fwdtransforms'], interpolator = 'nearestNeighbor' )  * bmask
 7289  warn_if_small_mask( wmseg, label='bold_perfusion:wmseg')
 7290  # warn_if_small_mask( csfseg, threshold_fraction=0.01, label='bold_perfusion:csfseg')
 7291  warn_if_small_mask( csfAndWM, label='bold_perfusion:csfAndWM')
 7292  mycompcor = ants.compcor( fmrimotcorr,
 7293    ncompcor=nc, quantile=compcorquantile, mask = csfAndWM,
 7294    filter_type='polynomial', degree=2 )
 7295  tr = ants.get_spacing( fmrimotcorr )[3]
 7296  simg = ants.smooth_image(fmrimotcorr, spa, sigma_in_physical_coordinates = True )
 7297  nuisance = mycompcor['basis']
 7298  nuisance = np.c_[ nuisance, mycompcor['components'] ]
 7299  if add_FD_to_nuisance:
 7300    nuisance = np.c_[ nuisance, corrmo['FD'] ]
 7301  if verbose:
 7302    print("make sure nuisance is independent of TC")
 7303  nuisance = ants.regress_components( nuisance, tclist )
 7304  regression_mask = bmask.clone()
 7305  gmmat = ants.timeseries_to_matrix( simg, regression_mask )
 7306  regvars = np.hstack( (nuisance, tclist ))
 7307  coefind = regvars.shape[1]-1
 7308  regvars = regvars[:,range(coefind)]
 7309  predictor_of_interest_idx = regvars.shape[1]-1
 7310  valid_perf_models = ['huber','quantile','theilsen','ransac', 'sgd', 'linear','SM']
 7311  if verbose:
 7312    print( "begin perfusion estimation with " + perfusion_regression_model + " model " )
 7313  if perfusion_regression_model == 'linear':
 7314    regression_model = LinearRegression()
 7315    regression_model.fit( regvars, gmmat )
 7316    coefind = regression_model.coef_.shape[1]-1
 7317    perfimg = ants.make_image( regression_mask, regression_model.coef_[:,coefind] )
 7318  elif perfusion_regression_model == 'SM': #
 7319    import statsmodels.api as sm
 7320    coeffs = np.zeros( gmmat.shape[1] )
 7321    # Loop over each outcome column in the outcomes matrix
 7322    for outcome_idx in range(gmmat.shape[1]):
 7323        outcome = gmmat[:, outcome_idx]  # Select one outcome column
 7324        model = sm.RLM(outcome, sm.add_constant(regvars), M=sm.robust.norms.HuberT())  # Huber's T norm for robust regression
 7325        results = model.fit()
 7326        coefficients = results.params  # Coefficients of all predictors
 7327        coeffs[outcome_idx] = coefficients[predictor_of_interest_idx]
 7328    perfimg = ants.make_image( regression_mask, coeffs )
 7329  elif perfusion_regression_model in valid_perf_models :
 7330    scaler = StandardScaler()
 7331    gmmat = scaler.fit_transform(gmmat)
 7332    coeffs = np.zeros( gmmat.shape[1] )
 7333    huber_regressor = select_regression_model( perfusion_regression_model )
 7334    multioutput_model = MultiOutputRegressor(huber_regressor)
 7335    multioutput_model.fit( regvars, gmmat )
 7336    ct=0
 7337    for i, estimator in enumerate(multioutput_model.estimators_):
 7338      coefficients = estimator.coef_
 7339      coeffs[ct]=coefficients[predictor_of_interest_idx]
 7340      ct=ct+1
 7341    perfimg = ants.make_image( regression_mask, coeffs )
 7342  else:
 7343    raise ValueError( perfusion_regression_model + " regression model is not found.")
 7344  meangmval = ( perfimg[ gmseg == 1 ] ).mean()
 7345  if meangmval < 0:
 7346      perfimg = perfimg * (-1.0)
 7347  negative_voxels = ( perfimg < 0.0 ).sum() / np.prod( perfimg.shape ) * 100.0
 7348  perfimg[ perfimg < 0.0 ] = 0.0 # non-physiological
 7349
 7350  # LaTeX code for Cerebral Blood Flow (CBF) calculation using ASL MRI
 7351  """
 7352CBF = \\frac{\\Delta M \\cdot \\lambda}{2 \\cdot T_1 \\cdot \\alpha \\cdot M_0 \\cdot (e^{-\\frac{w}{T_1}} - e^{-\\frac{w + \\tau}{T_1}})}
 7353
 7354Where:
 7355- \\Delta M is the difference in magnetization between labeled and control images.
 7356- \\lambda is the brain-blood partition coefficient, typically around 0.9 mL/g.
 7357- T_1 is the longitudinal relaxation time of blood, which is a tissue-specific constant.
 7358- \\alpha is the labeling efficiency.
 7359- M_0 is the equilibrium magnetization of brain tissue (from the M0 image).
 7360- w is the post-labeling delay, the time between the end of the labeling and the acquisition of the image.
 7361- \\tau is the labeling duration.
 7362  """
 7363  if m0_indices is None and m0_image is None:
 7364    m0 = ants.get_average_of_timeseries( fmrimotcorr )
 7365  else:
 7366    # register m0 to current template
 7367    m0reg = ants.registration( fmri_template, m0, 'antsRegistrationSyNRepro[r]', verbose=False )
 7368    m0 = m0reg['warpedmovout']
 7369
 7370  if ntp == 2 :
 7371      img0 = ants.slice_image( corrmo['motion_corrected'], axis=3, idx=0 )
 7372      img1 = ants.slice_image( corrmo['motion_corrected'], axis=3, idx=1 )
 7373      if m0_image is None:
 7374        if img0.mean() < img1.mean():
 7375            perfimg=img0
 7376            m0=img1
 7377        else:
 7378            perfimg=img1
 7379            m0=img0
 7380      else:
 7381        if img0.mean() < img1.mean():
 7382            perfimg=img1-img0
 7383        else:
 7384            perfimg=img0-img1
 7385
 7386  cbf = calculate_CBF(
 7387      Delta_M=perfimg, M_0=m0, mask=bmask )
 7388  if trim_the_mask > 0.0 :
 7389    bmask = trim_dti_mask( cbf, bmask, trim_the_mask )
 7390    perfimg = perfimg * bmask
 7391    cbf = cbf * bmask
 7392
 7393  meangmval = ( perfimg[ gmseg == 1 ] ).mean()        
 7394  meangmvalcbf = ( cbf[ gmseg == 1 ] ).mean()
 7395  if verbose:
 7396    print("perfimg.max() " + str(  perfimg.max() ) )
 7397  outdict = {}
 7398  outdict['meanBold'] = und
 7399  outdict['brainmask'] = bmask
 7400  rsfNuisance = pd.DataFrame( nuisance )
 7401  rsfNuisance['FD']=corrmo['FD']
 7402
 7403  if verbose:
 7404      print("perfusion dataframe begin")
 7405  dktseg = ants.apply_transforms( und, t1dktcit,
 7406    t1reg['fwdtransforms'], interpolator = 'genericLabel' ) * bmask
 7407  df_perf = antspyt1w.map_intensity_to_dataframe(
 7408        'dkt_cortex_cit_deep_brain',
 7409        perfimg,
 7410        dktseg)
 7411  df_perf = antspyt1w.merge_hierarchical_csvs_to_wide_format(
 7412              {'perf' : df_perf},
 7413              col_names = ['Mean'] )
 7414  df_cbf = antspyt1w.map_intensity_to_dataframe(
 7415        'dkt_cortex_cit_deep_brain',
 7416        cbf,
 7417        dktseg)
 7418  df_cbf = antspyt1w.merge_hierarchical_csvs_to_wide_format(
 7419              {'cbf' : df_cbf},
 7420              col_names = ['Mean'] )
 7421  df_cbf = df_cbf.add_prefix('cbf_')
 7422  df_perf = pd.concat( [df_perf,df_cbf], axis=1, ignore_index=False )
 7423  if verbose:
 7424      print("perfusion dataframe end")
 7425
 7426  outdict['perfusion']=perfimg
 7427  outdict['cbf']=cbf
 7428  outdict['m0']=m0
 7429  outdict['perfusion_gm_mean']=meangmval
 7430  outdict['cbf_gm_mean']=meangmvalcbf
 7431  outdict['perf_dataframe']=df_perf
 7432  outdict['motion_corrected'] = corrmo['motion_corrected']
 7433  outdict['gmseg'] = gmseg
 7434  outdict['brain_mask'] = bmask
 7435  outdict['nuisance'] = rsfNuisance
 7436  outdict['tsnr'] = mytsnr
 7437  outdict['ssnr'] = slice_snr( corrmo['motion_corrected'], csfAndWM, gmseg )
 7438  outdict['dvars'] = dvars( corrmo['motion_corrected'], gmseg )
 7439  outdict['high_motion_count'] = (rsfNuisance['FD'] > FD_threshold ).sum()
 7440  outdict['high_motion_pct'] = (rsfNuisance['FD'] > FD_threshold ).sum() / rsfNuisance.shape[0]
 7441  outdict['FD_max'] = rsfNuisance['FD'].max()
 7442  outdict['FD_mean'] = rsfNuisance['FD'].mean()
 7443  outdict['FD_sd'] = rsfNuisance['FD'].std()
 7444  outdict['bold_evr'] =  antspyt1w.patch_eigenvalue_ratio( und, 512, [16,16,16], evdepth = 0.9, mask = bmask )
 7445  outdict['t1reg'] = t1reg
 7446  outdict['outlier_volumes']=hlinds
 7447  outdict['n_outliers']=len(hlinds)
 7448  outdict['negative_voxels']=negative_voxels
 7449  return convert_np_in_dict( outdict )
 7450
 7451
 7452def pet3d_summary( pet3d, t1head, t1, t1segmentation, t1dktcit,
 7453                   spa = (0., 0., 0.),
 7454                   type_of_transform='antsRegistrationSyNRepro[r]',
 7455                   upsample=True,
 7456                   verbose=False ):
 7457  """
 7458  Estimate perfusion from a BOLD time series image.  Will attempt to figure out the T-C labels from the data.  The function uses defaults to quantify CBF but these will usually not be correct for your own data.  See the function calculate_CBF for an example of how one might do quantification based on the outputs of this function specifically the perfusion, m0 and mask images that are part of the output dictionary.
 7459
 7460  Arguments
 7461  ---------
 7462  pet3d : 3D PET antsImage
 7463
 7464  t1head : ANTsImage
 7465    input 3-D T1 brain image (not brain extracted)
 7466
 7467  t1 : ANTsImage
 7468    input 3-D T1 brain image (brain extracted)
 7469
 7470  t1segmentation : ANTsImage
 7471    t1 segmentation - a six tissue segmentation image in T1 space
 7472
 7473  t1dktcit : ANTsImage
 7474    t1 dkt cortex plus cit parcellation
 7475
 7476  type_of_transform : SyN or Rigid
 7477
 7478  upsample: boolean
 7479
 7480  verbose : boolean
 7481
 7482  Returns
 7483  ---------
 7484  a dictionary containing the derived network maps
 7485
 7486  """
 7487  import numpy as np
 7488  import pandas as pd
 7489  import re
 7490  import math
 7491
 7492  ex_path = os.path.expanduser( "~/.antspyt1w/" )
 7493  cnxcsvfn = ex_path + "dkt_cortex_cit_deep_brain.csv"
 7494  
 7495  pet3dr=pet3d
 7496  if upsample:
 7497      spc = ants.get_spacing( pet3d )
 7498      minspc = 1.0
 7499      if min(spc) < minspc:
 7500        minspc = min(spc)
 7501      newspc = [minspc,minspc,minspc]
 7502      pet3dr = ants.resample_image( pet3d, newspc, interp_type=0 )
 7503
 7504  rig = ants.registration( pet3dr, t1head, 'antsRegistrationSyNRepro[r]' )
 7505  bmask = ants.apply_transforms( pet3dr, 
 7506    ants.threshold_image(t1segmentation,1,6), 
 7507    rig['fwdtransforms'][0], 
 7508    interpolator='genericLabel' )
 7509  if verbose:
 7510    print("End t1=>pet registration")
 7511
 7512  und = pet3dr * bmask
 7513#  t1reg = ants.registration( und, t1, "antsRegistrationSyNQuickRepro[s]" )
 7514  t1reg = rig # ants.registration( und, t1, "Rigid" )
 7515  gmseg = ants.threshold_image( t1segmentation, 2, 2 )
 7516  gmseg = gmseg + ants.threshold_image( t1segmentation, 4, 4 )
 7517  gmseg = ants.threshold_image( gmseg, 1, 4 )
 7518  gmseg = ants.iMath( gmseg, 'MD', 1 )
 7519  gmseg = ants.apply_transforms( und, gmseg,
 7520    t1reg['fwdtransforms'], interpolator = 'genericLabel' ) * bmask
 7521  csfseg = ants.threshold_image( t1segmentation, 1, 1 )
 7522  wmseg = ants.threshold_image( t1segmentation, 3, 3 )
 7523  csfAndWM = ( csfseg + wmseg ).morphology("erode",1)
 7524  csfAndWM = ants.apply_transforms( und, csfAndWM,
 7525    t1reg['fwdtransforms'], interpolator = 'nearestNeighbor' )  * bmask
 7526  csfseg = ants.apply_transforms( und, csfseg,
 7527    t1reg['fwdtransforms'], interpolator = 'nearestNeighbor' )  * bmask
 7528  wmseg = ants.apply_transforms( und, wmseg,
 7529    t1reg['fwdtransforms'], interpolator = 'nearestNeighbor' )  * bmask
 7530  wmsignal = pet3dr[ ants.iMath(wmseg,"ME",1) == 1 ].mean()
 7531  gmsignal = pet3dr[ gmseg == 1 ].mean()
 7532  csfsignal = pet3dr[ csfseg == 1 ].mean()
 7533  if verbose:
 7534    print("pet3dr.max() " + str(  pet3dr.max() ) )
 7535  if verbose:
 7536      print("pet3d dataframe begin")
 7537  dktseg = ants.apply_transforms( und, t1dktcit,
 7538    t1reg['fwdtransforms'], interpolator = 'genericLabel' ) * bmask
 7539  df_pet3d = antspyt1w.map_intensity_to_dataframe(
 7540        'dkt_cortex_cit_deep_brain',
 7541        und,
 7542        dktseg)
 7543  df_pet3d = antspyt1w.merge_hierarchical_csvs_to_wide_format(
 7544              {'pet3d' : df_pet3d},
 7545              col_names = ['Mean'] )
 7546  if verbose:
 7547      print("pet3d dataframe end")
 7548
 7549  outdict = {}
 7550  outdict['pet3d_dataframe']=df_pet3d
 7551  outdict['pet3d'] = pet3dr
 7552  outdict['brainmask'] = bmask
 7553  outdict['gm_mean']=gmsignal
 7554  outdict['wm_mean']=wmsignal
 7555  outdict['csf_mean']=csfsignal
 7556  outdict['pet3d_dataframe']=df_pet3d
 7557  outdict['t1reg'] = t1reg
 7558  return convert_np_in_dict( outdict )
 7559
 7560
 7561def write_bvals_bvecs(bvals, bvecs, prefix ):
 7562    ''' Write FSL FDT bvals and bvecs files
 7563
 7564    adapted from dipy.external code
 7565
 7566    Parameters
 7567    -------------
 7568    bvals : (N,) sequence
 7569       Vector with diffusion gradient strength (one per diffusion
 7570       acquisition, N=no of acquisitions)
 7571    bvecs : (N, 3) array-like
 7572       diffusion gradient directions
 7573    prefix : string
 7574       path to write FDT bvals, bvecs text files
 7575       None results in current working directory.
 7576    '''
 7577    _VAL_FMT = '   %e'
 7578    bvals = tuple(bvals)
 7579    bvecs = np.asarray(bvecs)
 7580    bvecs[np.isnan(bvecs)] = 0
 7581    N = len(bvals)
 7582    fname = prefix + '.bval'
 7583    fmt = _VAL_FMT * N + '\n'
 7584    myfile = open(fname, 'wt')
 7585    myfile.write(fmt % bvals)
 7586    myfile.close()
 7587    fname = prefix + '.bvec'
 7588    bvf = open(fname, 'wt')
 7589    for dim_vals in bvecs.T:
 7590        bvf.write(fmt % tuple(dim_vals))
 7591    bvf.close()
 7592    
 7593
 7594def crop_mcimage( x, mask, padder=None ):
 7595    """
 7596    crop a time series (4D) image by a 3D mask
 7597
 7598    Parameters
 7599    -------------
 7600
 7601    x : raw image
 7602
 7603    mask  : mask for cropping
 7604
 7605    """
 7606    cropmask = ants.crop_image( mask, mask )
 7607    myorig = list( ants.get_origin(cropmask) )
 7608    myorig.append( ants.get_origin( x )[3] )
 7609    croplist = []
 7610    if len(x.shape) > 3:
 7611        for k in range(x.shape[3]):
 7612            temp = ants.slice_image( x, axis=3, idx=k )
 7613            temp = ants.crop_image( temp, mask )
 7614            if padder is not None:
 7615                temp = ants.pad_image( temp, pad_width=padder )
 7616            croplist.append( temp )
 7617        temp = ants.list_to_ndimage( x, croplist )
 7618        temp.set_origin( myorig )
 7619        return temp
 7620    else:
 7621        return( ants.crop_image( x, mask ) )
 7622
 7623
 7624def mm(
 7625    t1_image,
 7626    hier,
 7627    rsf_image=[],
 7628    flair_image=None,
 7629    nm_image_list=None,
 7630    dw_image=[], bvals=[], bvecs=[],
 7631    perfusion_image=None,
 7632    srmodel=None,
 7633    do_tractography = False,
 7634    do_kk = False,
 7635    do_normalization = None,
 7636    group_template = None,
 7637    group_transform = None,
 7638    target_range = [0,1],
 7639    dti_motion_correct = 'antsRegistrationSyNQuickRepro[r]',
 7640    dti_denoise = False,
 7641    perfusion_trim=10,
 7642    perfusion_m0_image=None,
 7643    perfusion_m0=None,
 7644    rsf_upsampling=3.0,
 7645    pet_3d_image=None,
 7646    test_run = False,
 7647    verbose = False ):
 7648    """
 7649    Multiple modality processing and normalization
 7650
 7651    aggregates modality-specific processing under one roof.  see individual
 7652    modality specific functions for details.
 7653
 7654    Parameters
 7655    -------------
 7656
 7657    t1_image : raw t1 image
 7658
 7659    hier  : output of antspyt1w.hierarchical ( see read hierarchical )
 7660
 7661    rsf_image : list of resting state fmri
 7662
 7663    flair_image : flair
 7664
 7665    nm_image_list : list of neuromelanin images
 7666
 7667    dw_image : list of diffusion weighted images
 7668
 7669    bvals : list of bvals file names
 7670
 7671    bvecs : list of bvecs file names
 7672
 7673    perfusion_image : single perfusion image
 7674
 7675    srmodel : optional srmodel
 7676
 7677    do_tractography : boolean
 7678
 7679    do_kk : boolean to control whether we compute kelly kapowski thickness image (slow)
 7680
 7681    do_normalization : template transformation if available
 7682
 7683    group_template : optional reference template corresponding to the group_transform
 7684
 7685    group_transform : optional transforms corresponding to the group_template
 7686
 7687    target_range : 2-element tuple
 7688        a tuple or array defining the (min, max) of the input image
 7689        (e.g., [-127.5, 127.5] or [0,1]).  Output images will be scaled back to original
 7690        intensity. This range should match the mapping used in the training
 7691        of the network.
 7692    
 7693    dti_motion_correct : None Rigid or SyN
 7694
 7695    dti_denoise : boolean
 7696
 7697    perfusion_trim : optional integer number of time volumes to exclude from the front of the perfusion time series
 7698
 7699    perfusion_m0_image : optional antsImage m0 associated with the perfusion time series
 7700
 7701    perfusion_m0 : optional list containing indices of the m0 in the perfusion time series
 7702
 7703    rsf_upsampling : optional upsampling parameter value in mm; if set to zero, no upsampling is done
 7704
 7705    pet_3d_image : optional antsImage for a 3D pet; we make no assumptions about the contents of 
 7706        this image.  we just process it and provide summary information.
 7707
 7708    test_run : boolean 
 7709
 7710    verbose : boolean
 7711
 7712    """
 7713    from os.path import exists
 7714    ex_path = os.path.expanduser( "~/.antspyt1w/" )
 7715    ex_path_mm = os.path.expanduser( "~/.antspymm/" )
 7716    mycsvfn = ex_path + "FA_JHU_labels_edited.csv"
 7717    citcsvfn = ex_path + "CIT168_Reinf_Learn_v1_label_descriptions_pad.csv"
 7718    dktcsvfn = ex_path + "dkt.csv"
 7719    cnxcsvfn = ex_path + "dkt_cortex_cit_deep_brain.csv"
 7720    JHU_atlasfn = ex_path + 'JHU-ICBM-FA-1mm.nii.gz' # Read in JHU atlas
 7721    JHU_labelsfn = ex_path + 'JHU-ICBM-labels-1mm.nii.gz' # Read in JHU labels
 7722    templatefn = ex_path + 'CIT168_T1w_700um_pad_adni.nii.gz'
 7723    if not exists( mycsvfn ) or not exists( citcsvfn ) or not exists( cnxcsvfn ) or not exists( dktcsvfn ) or not exists( JHU_atlasfn ) or not exists( JHU_labelsfn ) or not exists( templatefn ):
 7724        print( "**missing files** => call get_data from latest antspyt1w and antspymm." )
 7725        raise ValueError('**missing files** => call get_data from latest antspyt1w and antspymm.')
 7726    mycsv = pd.read_csv(  mycsvfn )
 7727    citcsv = pd.read_csv(  os.path.expanduser( citcsvfn ) )
 7728    dktcsv = pd.read_csv(  os.path.expanduser( dktcsvfn ) )
 7729    cnxcsv = pd.read_csv(  os.path.expanduser( cnxcsvfn ) )
 7730    JHU_atlas = mm_read( JHU_atlasfn ) # Read in JHU atlas
 7731    JHU_labels = mm_read( JHU_labelsfn ) # Read in JHU labels
 7732    template = mm_read( templatefn ) # Read in template
 7733    if group_template is None:
 7734        group_template = template
 7735        group_transform = do_normalization['fwdtransforms']
 7736    if verbose:
 7737        print("Using group template:")
 7738        print( group_template )
 7739    #####################
 7740    #  T1 hierarchical  #
 7741    #####################
 7742    t1imgbrn = hier['brain_n4_dnz']
 7743    t1atropos = hier['dkt_parc']['tissue_segmentation']
 7744    output_dict = {
 7745        'kk': None,
 7746        'rsf': None,
 7747        'flair' : None,
 7748        'NM' : None,
 7749        'DTI' : None,
 7750        'FA_summ' : None,
 7751        'MD_summ' : None,
 7752        'tractography' : None,
 7753        'tractography_connectivity' : None,
 7754        'perf' : None,
 7755        'pet3d' : None,
 7756    }
 7757    normalization_dict = {
 7758        'kk_norm': None,
 7759        'NM_norm' : None,
 7760        'DTI_norm': None,
 7761        'FA_norm' : None,
 7762        'MD_norm' : None,
 7763        'perf_norm' : None,
 7764        'alff_norm' : None,
 7765        'falff_norm' : None,
 7766        'CinguloopercularTaskControl_norm' : None,
 7767        'DefaultMode_norm' : None,
 7768        'MemoryRetrieval_norm' : None,
 7769        'VentralAttention_norm' : None,
 7770        'Visual_norm' : None,
 7771        'FrontoparietalTaskControl_norm' : None,
 7772        'Salience_norm' : None,
 7773        'Subcortical_norm' : None,
 7774        'DorsalAttention_norm' : None,
 7775        'pet3d_norm' : None
 7776    }
 7777    if test_run:
 7778        return output_dict, normalization_dict
 7779
 7780    if do_kk:
 7781        if verbose:
 7782            print('kk in mm')
 7783        output_dict['kk'] = antspyt1w.kelly_kapowski_thickness( t1_image,
 7784            labels=hier['dkt_parc']['dkt_cortex'], iterations=45 )
 7785
 7786    if perfusion_image is not None:
 7787        if perfusion_image.shape[3] > 1: # FIXME - better heuristic?
 7788            output_dict['perf'] = bold_perfusion(
 7789                perfusion_image,
 7790                t1_image,
 7791                hier['brain_n4_dnz'],
 7792                t1atropos,
 7793                hier['dkt_parc']['dkt_cortex'] + hier['cit168lab'],
 7794                n_to_trim = perfusion_trim,
 7795                m0_image = perfusion_m0_image,
 7796                m0_indices = perfusion_m0,
 7797                verbose=verbose )
 7798
 7799    if pet_3d_image is not None:
 7800        if pet_3d_image.dimension == 3: # FIXME - better heuristic?
 7801            output_dict['pet3d'] = pet3d_summary(
 7802                pet_3d_image,
 7803                t1_image,
 7804                hier['brain_n4_dnz'],
 7805                t1atropos,
 7806                hier['dkt_parc']['dkt_cortex'] + hier['cit168lab'],
 7807                verbose=verbose )
 7808    ################################## do the rsf .....
 7809    if len(rsf_image) > 0:
 7810        my_motion_tx = 'antsRegistrationSyNRepro[r]'
 7811        rsf_image = [i for i in rsf_image if i is not None]
 7812        if verbose:
 7813            print('rsf length ' + str( len( rsf_image ) ) )
 7814        if len( rsf_image ) >= 2: # assume 2 is the largest possible value
 7815            rsf_image1 = rsf_image[0]
 7816            rsf_image2 = rsf_image[1]
 7817            # build a template then join the images
 7818            if verbose:
 7819                print("initial average for rsf")
 7820            rsfavg1, hlinds = loop_timeseries_censoring( rsf_image1, 0.1 )
 7821            rsfavg1=get_average_rsf(rsfavg1)
 7822            rsfavg2, hlinds = loop_timeseries_censoring( rsf_image2, 0.1 )
 7823            rsfavg2=get_average_rsf(rsfavg2)
 7824            if verbose:
 7825                print("template average for rsf")
 7826            init_temp = ants.image_clone( rsfavg1 )
 7827            if rsf_image1.shape[3] < rsf_image2.shape[3]:
 7828                init_temp = ants.image_clone( rsfavg2 )
 7829            boldTemplate = ants.build_template(
 7830                initial_template = init_temp,
 7831                image_list=[rsfavg1,rsfavg2],
 7832                type_of_transform="antsRegistrationSyNQuickRepro[s]",
 7833                iterations=5, verbose=False )
 7834            if verbose:
 7835                print("join the 2 rsf")
 7836            if rsf_image1.shape[3] > 10 and rsf_image2.shape[3] > 10:
 7837                leadvols = list(range(8))
 7838                rsf_image2 = remove_volumes_from_timeseries( rsf_image2, leadvols )
 7839                rsf_image = merge_timeseries_data( rsf_image1, rsf_image2 )
 7840            elif rsf_image1.shape[3] > rsf_image2.shape[3]:
 7841                rsf_image = rsf_image1
 7842            else:
 7843                rsf_image = rsf_image2
 7844        elif len( rsf_image ) == 1:
 7845            rsf_image = rsf_image[0]
 7846            boldTemplate, hlinds = loop_timeseries_censoring( rsf_image, 0.1 )
 7847            boldTemplate = get_average_rsf(boldTemplate)
 7848        if rsf_image.shape[3] > 10: # FIXME - better heuristic?
 7849            rsfprolist = [] # FIXMERSF
 7850            # Create the parameter DataFrame
 7851            df = pd.DataFrame({
 7852                "num": [134, 122, 129],
 7853                "loop": [0.50, 0.25, 0.50],
 7854                "cens": [True, True, True],
 7855                "HM": [1.0, 5.0, 0.5],
 7856                "ff": ["tight", "tight", "tight"],
 7857                "CC": [5, 5, 0.8],
 7858                "imp": [True, True, True],
 7859                "up": [rsf_upsampling, rsf_upsampling, rsf_upsampling],
 7860                "coords": [False,False,False]
 7861            }, index=[0, 1, 2])
 7862            for p in range(df.shape[0]):
 7863                if verbose:
 7864                    print("rsf parameters")
 7865                    print( df.iloc[p] )
 7866                if df['ff'].iloc[p] == 'broad':
 7867                    f=[ 0.008, 0.15 ]
 7868                elif df['ff'].iloc[p] == 'tight':
 7869                    f=[ 0.03, 0.08 ]
 7870                elif df['ff'].iloc[p] == 'mid':
 7871                    f=[ 0.01, 0.1 ]
 7872                elif df['ff'].iloc[p] == 'mid2':
 7873                    f=[ 0.01, 0.08 ]
 7874                else:
 7875                    raise ValueError("we do not recognize this parameter choice for frequency filtering: " + df['ff'].iloc[p] )
 7876                HM = df['HM'].iloc[p]
 7877                CC = df['CC'].iloc[p]
 7878                loop= df['loop'].iloc[p]
 7879                cens =df['cens'].iloc[p]
 7880                imp = df['imp'].iloc[p]
 7881                rsf0 = resting_state_fmri_networks(
 7882                                            rsf_image,
 7883                                            boldTemplate,
 7884                                            hier['brain_n4_dnz'],
 7885                                            t1atropos,
 7886                                            f=f,
 7887                                            FD_threshold=HM, 
 7888                                            spa = None, 
 7889                                            spt = None, 
 7890                                            nc = CC,
 7891                                            outlier_threshold=loop,
 7892                                            ica_components = 0,
 7893                                            impute = imp,
 7894                                            censor = cens,
 7895                                            despike = 2.5,
 7896                                            motion_as_nuisance = True,
 7897                                            upsample=df['up'].iloc[p],
 7898                                            clean_tmp=0.66,
 7899                                            paramset=df['num'].iloc[p],
 7900                                            powers=df['coords'].iloc[p],
 7901                                            verbose=verbose ) # default
 7902                rsfprolist.append( rsf0 )
 7903            output_dict['rsf'] = rsfprolist
 7904
 7905    if nm_image_list is not None:
 7906        if verbose:
 7907            print('nm')
 7908        if srmodel is None:
 7909            output_dict['NM'] = neuromelanin( nm_image_list, t1imgbrn, t1_image, hier['deep_cit168lab'], verbose=verbose )
 7910        else:
 7911            output_dict['NM'] = neuromelanin( nm_image_list, t1imgbrn, t1_image, hier['deep_cit168lab'], srmodel=srmodel, target_range=target_range, verbose=verbose  )
 7912################################## do the dti .....
 7913    if len(dw_image) > 0 :
 7914        if verbose:
 7915            print('dti-x')
 7916        if len( dw_image ) == 1: # use T1 for distortion correction and brain extraction
 7917            if verbose:
 7918                print("We have only one DTI: " + str(len(dw_image)))
 7919            dw_image = dw_image[0]
 7920            btpB0, btpDW = get_average_dwi_b0(dw_image)
 7921            initrig = ants.registration( btpDW, hier['brain_n4_dnz'], 'antsRegistrationSyNRepro[r]' )['fwdtransforms'][0]
 7922            tempreg = ants.registration( btpDW, hier['brain_n4_dnz'], 'SyNOnly',
 7923                syn_metric='CC', syn_sampling=2,
 7924                reg_iterations=[50,50,20],
 7925                multivariate_extras=[ [ "CC", btpB0, hier['brain_n4_dnz'], 1, 2 ]],
 7926                initial_transform=initrig
 7927                )
 7928            mybxt = ants.threshold_image( ants.iMath(hier['brain_n4_dnz'], "Normalize" ), 0.001, 1 )
 7929            btpDW = ants.apply_transforms( btpDW, btpDW,
 7930                tempreg['invtransforms'][1], interpolator='linear')
 7931            btpB0 = ants.apply_transforms( btpB0, btpB0,
 7932                tempreg['invtransforms'][1], interpolator='linear')
 7933            dwimask = ants.apply_transforms( btpDW, mybxt, tempreg['fwdtransforms'][1], interpolator='nearestNeighbor')
 7934            # dwimask = ants.iMath(dwimask,'MD',1)
 7935            t12dwi = ants.apply_transforms( btpDW, hier['brain_n4_dnz'], tempreg['fwdtransforms'][1], interpolator='linear')
 7936            output_dict['DTI'] = joint_dti_recon(
 7937                dw_image,
 7938                bvals[0],
 7939                bvecs[0],
 7940                jhu_atlas=JHU_atlas,
 7941                jhu_labels=JHU_labels,
 7942                brain_mask = dwimask,
 7943                reference_B0 = btpB0,
 7944                reference_DWI = btpDW,
 7945                srmodel=srmodel,
 7946                motion_correct=dti_motion_correct, # set to False if using input from qsiprep
 7947                denoise=dti_denoise,
 7948                verbose = verbose)
 7949        else :  # use phase encoding acquisitions for distortion correction and T1 for brain extraction
 7950            if verbose:
 7951                print("We have both DTI_LR and DTI_RL: " + str(len(dw_image)))
 7952            a1b,a1w=get_average_dwi_b0(dw_image[0])
 7953            a2b,a2w=get_average_dwi_b0(dw_image[1],fixed_b0=a1b,fixed_dwi=a1w)
 7954            btpB0, btpDW = dti_template(
 7955                b_image_list=[a1b,a2b],
 7956                w_image_list=[a1w,a2w],
 7957                iterations=7, verbose=verbose )
 7958            initrig = ants.registration( btpDW, hier['brain_n4_dnz'], 'antsRegistrationSyNRepro[r]' )['fwdtransforms'][0]
 7959            tempreg = ants.registration( btpDW, hier['brain_n4_dnz'], 'SyNOnly',
 7960                syn_metric='CC', syn_sampling=2,
 7961                reg_iterations=[50,50,20],
 7962                multivariate_extras=[ [ "CC", btpB0, hier['brain_n4_dnz'], 1, 2 ]],
 7963                initial_transform=initrig
 7964                )
 7965            mybxt = ants.threshold_image( ants.iMath(hier['brain_n4_dnz'], "Normalize" ), 0.001, 1 )
 7966            dwimask = ants.apply_transforms( btpDW, mybxt, tempreg['fwdtransforms'], interpolator='nearestNeighbor')
 7967            output_dict['DTI'] = joint_dti_recon(
 7968                dw_image[0],
 7969                bvals[0],
 7970                bvecs[0],
 7971                jhu_atlas=JHU_atlas,
 7972                jhu_labels=JHU_labels,
 7973                brain_mask = dwimask,
 7974                reference_B0 = btpB0,
 7975                reference_DWI = btpDW,
 7976                srmodel=srmodel,
 7977                img_RL=dw_image[1],
 7978                bval_RL=bvals[1],
 7979                bvec_RL=bvecs[1],
 7980                motion_correct=dti_motion_correct, # set to False if using input from qsiprep
 7981                denoise=dti_denoise,
 7982                verbose = verbose)
 7983        mydti = output_dict['DTI']
 7984        # summarize dwi with T1 outputs
 7985        # first - register ....
 7986        reg = ants.registration( mydti['recon_fa'], hier['brain_n4_dnz'], 'antsRegistrationSyNRepro[s]', total_sigma=1.0 )
 7987        ##################################################
 7988        output_dict['FA_summ'] = hierarchical_modality_summary(
 7989            mydti['recon_fa'],
 7990            hier=hier,
 7991            modality_name='fa',
 7992            transformlist=reg['fwdtransforms'],
 7993            verbose = False )
 7994        ##################################################
 7995        output_dict['MD_summ'] = hierarchical_modality_summary(
 7996            mydti['recon_md'],
 7997            hier=hier,
 7998            modality_name='md',
 7999            transformlist=reg['fwdtransforms'],
 8000            verbose = False )
 8001        # these inputs should come from nicely processed data
 8002        dktmapped = ants.apply_transforms(
 8003            mydti['recon_fa'],
 8004            hier['dkt_parc']['dkt_cortex'],
 8005            reg['fwdtransforms'], interpolator='nearestNeighbor' )
 8006        citmapped = ants.apply_transforms(
 8007            mydti['recon_fa'],
 8008            hier['cit168lab'],
 8009            reg['fwdtransforms'], interpolator='nearestNeighbor' )
 8010        dktmapped[ citmapped > 0]=0
 8011        mask = ants.threshold_image( mydti['recon_fa'], 0.01, 2.0 ).iMath("GetLargestComponent")
 8012        if do_tractography: # dwi_deterministic_tracking dwi_closest_peak_tracking
 8013            output_dict['tractography'] = dwi_deterministic_tracking(
 8014                mydti['dwi_LR_dewarped'],
 8015                mydti['recon_fa'],
 8016                mydti['bval_LR'],
 8017                mydti['bvec_LR'],
 8018                seed_density = 1,
 8019                mask=mask,
 8020                verbose = verbose )
 8021            mystr = output_dict['tractography']
 8022            output_dict['tractography_connectivity'] = dwi_streamline_connectivity( mystr['streamlines'], dktmapped+citmapped, cnxcsv, verbose=verbose )
 8023    ################################## do the flair .....
 8024    if flair_image is not None:
 8025        if verbose:
 8026            print('flair')
 8027        wmhprior = None
 8028        priorfn = ex_path_mm + 'CIT168_wmhprior_700um_pad_adni.nii.gz'
 8029        if ( exists( priorfn ) ):
 8030            wmhprior = ants.image_read( priorfn )
 8031            wmhprior = ants.apply_transforms( t1_image, wmhprior, do_normalization['invtransforms'] )
 8032        output_dict['flair'] = boot_wmh( flair_image, t1_image, t1atropos,
 8033            prior_probability=wmhprior, verbose=verbose )
 8034    #################################################################
 8035    ### NOTES: deforming to a common space and writing out images ###
 8036    ### images we want come from: DTI, NM, rsf, thickness ###########
 8037    #################################################################
 8038    if do_normalization is not None:
 8039        if verbose:
 8040            print('normalization')
 8041        # might reconsider this template space - cropped and/or higher res?
 8042        # template = ants.resample_image( template, [1,1,1], use_voxels=False )
 8043        # t1reg = ants.registration( template, hier['brain_n4_dnz'], "antsRegistrationSyNQuickRepro[s]")
 8044        t1reg = do_normalization
 8045        if do_kk:
 8046            normalization_dict['kk_norm'] = ants.apply_transforms( group_template, output_dict['kk']['thickness_image'], group_transform )
 8047        if output_dict['DTI'] is not None:
 8048            mydti = output_dict['DTI']
 8049            dtirig = ants.registration( hier['brain_n4_dnz'], mydti['recon_fa'], 'antsRegistrationSyNRepro[r]' )
 8050            normalization_dict['MD_norm'] = ants.apply_transforms( group_template, mydti['recon_md'],group_transform+dtirig['fwdtransforms'] )
 8051            normalization_dict['FA_norm'] = ants.apply_transforms( group_template, mydti['recon_fa'],group_transform+dtirig['fwdtransforms'] )
 8052            output_directory = tempfile.mkdtemp()
 8053            do_dti_norm=False
 8054            if do_dti_norm:
 8055                comptx = ants.apply_transforms( group_template, group_template, group_transform+dtirig['fwdtransforms'], compose = output_directory + '/xxx' )
 8056                tspc=[2.,2.,2.]
 8057                if srmodel is not None:
 8058                    tspc=[1.,1.,1.]
 8059                group_template2mm = ants.resample_image( group_template, tspc  )
 8060                normalization_dict['DTI_norm'] = transform_and_reorient_dti( group_template2mm, mydti['dti'], comptx, verbose=False )
 8061            import shutil
 8062            shutil.rmtree(output_directory, ignore_errors=True )
 8063        if output_dict['rsf'] is not None:
 8064            if False:
 8065                rsfpro = output_dict['rsf'] # FIXME
 8066                rsfrig = ants.registration( hier['brain_n4_dnz'], rsfpro['meanBold'], 'antsRegistrationSyNRepro[r]' )
 8067                for netid in get_antsimage_keys( rsfpro ):
 8068                    rsfkey = netid + "_norm"
 8069                    normalization_dict[rsfkey] = ants.apply_transforms(
 8070                        group_template, rsfpro[netid],
 8071                        group_transform+rsfrig['fwdtransforms'] )
 8072        if output_dict['perf'] is not None: # zizzer
 8073            comptx = group_transform + output_dict['perf']['t1reg']['invtransforms']
 8074            normalization_dict['perf_norm'] = ants.apply_transforms( group_template,
 8075                output_dict['perf']['perfusion'], comptx,
 8076                whichtoinvert=[False,False,True,False] )
 8077            normalization_dict['cbf_norm'] = ants.apply_transforms( group_template,
 8078                output_dict['perf']['cbf'], comptx,
 8079                whichtoinvert=[False,False,True,False] )
 8080        if output_dict['pet3d'] is not None: # zizzer
 8081            secondTx=output_dict['pet3d']['t1reg']['invtransforms']
 8082            comptx = group_transform + secondTx
 8083            if len( secondTx ) == 2:
 8084                wti=[False,False,True,False]
 8085            else:
 8086                wti=[False,False,True]
 8087            normalization_dict['pet3d_norm'] = ants.apply_transforms( group_template,
 8088                output_dict['pet3d']['pet3d'], comptx,
 8089                whichtoinvert=wti )
 8090        if nm_image_list is not None:
 8091            nmpro = output_dict['NM']
 8092            nmrig = nmpro['t1_to_NM_transform'] # this is an inverse tx
 8093            normalization_dict['NM_norm'] = ants.apply_transforms( group_template, nmpro['NM_avg'], group_transform+nmrig,
 8094                whichtoinvert=[False,False,True])
 8095
 8096    if verbose:
 8097        print('mm done')
 8098    return output_dict, normalization_dict
 8099
 8100
 8101def write_mm( output_prefix, mm, mm_norm=None, t1wide=None, separator='_', verbose=False ):
 8102    """
 8103    write the tabular and normalization output of the mm function
 8104
 8105    Parameters
 8106    -------------
 8107
 8108    output_prefix : prefix for file outputs - modality specific postfix will be added
 8109
 8110    mm  : output of mm function for modality-space processing should be a dictionary with 
 8111        dictionary entries for each modality.
 8112
 8113    mm_norm : output of mm function for normalized processing
 8114
 8115    t1wide : wide output data frame from t1 hierarchical
 8116
 8117    separator : string or character separator for filenames
 8118
 8119    verbose : boolean
 8120
 8121    Returns
 8122    ---------
 8123
 8124    both csv and image files written to disk.  the primary outputs will be
 8125    output_prefix + separator + 'mmwide.csv' and *norm.nii.gz images
 8126
 8127    """
 8128    from dipy.io.streamline import save_tractogram
 8129    if mm_norm is not None:
 8130        for mykey in mm_norm:
 8131            tempfn = output_prefix + separator + mykey + '.nii.gz'
 8132            if mm_norm[mykey] is not None:
 8133                image_write_with_thumbnail( mm_norm[mykey], tempfn )
 8134    thkderk = None
 8135    if t1wide is not None:
 8136        thkderk = t1wide.iloc[: , 1:]
 8137    kkderk = None
 8138    if 'kk' in mm:
 8139        if mm['kk'] is not None:
 8140            kkderk = mm['kk']['thickness_dataframe'].iloc[: , 1:]
 8141            mykey='thickness_image'
 8142            tempfn = output_prefix + separator + mykey + '.nii.gz'
 8143            image_write_with_thumbnail( mm['kk'][mykey], tempfn )
 8144    nmderk = None
 8145    if 'NM' in mm:
 8146        if mm['NM'] is not None:
 8147            nmderk = mm['NM']['NM_dataframe_wide'].iloc[: , 1:]
 8148            for mykey in get_antsimage_keys( mm['NM'] ):
 8149                tempfn = output_prefix + separator + mykey + '.nii.gz'
 8150                image_write_with_thumbnail( mm['NM'][mykey], tempfn, thumb=False )
 8151
 8152    faderk = mdderk = fat1derk = mdt1derk = None
 8153
 8154    if 'DTI' in mm:
 8155        if mm['DTI'] is not None:
 8156            mydti = mm['DTI']
 8157            myop = output_prefix + separator
 8158            ants.image_write( mydti['dti'],  myop + 'dti.nii.gz' )
 8159            write_bvals_bvecs( mydti['bval_LR'], mydti['bvec_LR'], myop + 'reoriented' )
 8160            image_write_with_thumbnail( mydti['dwi_LR_dewarped'],  myop + 'dwi.nii.gz' )
 8161            image_write_with_thumbnail( mydti['dtrecon_LR_dewarp']['RGB'] ,  myop + 'DTIRGB.nii.gz' )
 8162            image_write_with_thumbnail( mydti['jhu_labels'],  myop+'dtijhulabels.nii.gz', mydti['recon_fa'] )
 8163            image_write_with_thumbnail( mydti['recon_fa'],  myop+'dtifa.nii.gz' )
 8164            image_write_with_thumbnail( mydti['recon_md'],  myop+'dtimd.nii.gz' )
 8165            image_write_with_thumbnail( mydti['b0avg'],  myop+'b0avg.nii.gz' )
 8166            image_write_with_thumbnail( mydti['dwiavg'],  myop+'dwiavg.nii.gz' )
 8167            faderk = mm['DTI']['recon_fa_summary'].iloc[: , 1:]
 8168            mdderk = mm['DTI']['recon_md_summary'].iloc[: , 1:]
 8169            fat1derk = mm['FA_summ'].iloc[: , 1:]
 8170            mdt1derk = mm['MD_summ'].iloc[: , 1:]
 8171    if 'tractography' in mm:
 8172        if mm['tractography'] is not None:
 8173            ofn = output_prefix + separator + 'tractogram.trk'
 8174            if mm['tractography']['tractogram'] is not None:
 8175                save_tractogram( mm['tractography']['tractogram'], ofn )
 8176    cnxderk = None
 8177    if 'tractography_connectivity' in mm:
 8178        if mm['tractography_connectivity'] is not None:
 8179            cnxderk = mm['tractography_connectivity']['connectivity_wide'].iloc[: , 1:] # NOTE: connectivity_wide is not much tested
 8180            ofn = output_prefix + separator + 'dtistreamlineconn.csv'
 8181            pd.DataFrame(mm['tractography_connectivity']['connectivity_matrix']).to_csv( ofn )
 8182
 8183    dlist = [
 8184        thkderk,
 8185        kkderk,
 8186        nmderk,
 8187        faderk,
 8188        mdderk,
 8189        fat1derk,
 8190        mdt1derk,
 8191        cnxderk
 8192        ]
 8193    is_all_none = all(element is None for element in dlist)
 8194    if is_all_none:
 8195        mm_wide = pd.DataFrame({'u_hier_id': [output_prefix] })
 8196    else:
 8197        mm_wide = pd.concat( dlist, axis=1, ignore_index=False )
 8198
 8199    mm_wide = mm_wide.copy()
 8200    if 'NM' in mm:
 8201        if mm['NM'] is not None:
 8202            nmwide = dict_to_dataframe( mm['NM'] )
 8203            if mm_wide.shape[0] > 0 and nmwide.shape[0] > 0:
 8204                nmwide.set_index( mm_wide.index, inplace=True )
 8205            mm_wide = pd.concat( [mm_wide, nmwide ], axis=1, ignore_index=False )
 8206    if 'flair' in mm:
 8207        if mm['flair'] is not None:
 8208            myop = output_prefix + separator + 'wmh.nii.gz'
 8209            pngfnb = output_prefix + separator + 'wmh_seg.png'
 8210            ants.plot( mm['flair']['flair'], mm['flair']['WMH_posterior_probability_map'], axis=2, nslices=21, ncol=7, filename=pngfnb, crop=True )
 8211            if mm['flair']['WMH_probability_map'] is not None:
 8212                image_write_with_thumbnail( mm['flair']['WMH_probability_map'], myop, thumb=False )
 8213            flwide = dict_to_dataframe( mm['flair'] )
 8214            if mm_wide.shape[0] > 0 and flwide.shape[0] > 0:
 8215                flwide.set_index( mm_wide.index, inplace=True )
 8216            mm_wide = pd.concat( [mm_wide, flwide ], axis=1, ignore_index=False )
 8217    if 'rsf' in mm:
 8218        if mm['rsf'] is not None:
 8219            fcnxpro=99
 8220            rsfdata = mm['rsf']
 8221            if not isinstance( rsfdata, list ):
 8222                rsfdata = [ rsfdata ]
 8223            for rsfpro in rsfdata:
 8224                fcnxpro=str( rsfpro['paramset']  )
 8225                pronum = 'fcnxpro'+str(fcnxpro)+"_"
 8226                if verbose:
 8227                    print("Collect rsf data " + pronum)
 8228                new_rsf_wide = dict_to_dataframe( rsfpro )
 8229                new_rsf_wide = pd.concat( [new_rsf_wide, rsfpro['corr_wide'] ], axis=1, ignore_index=False )
 8230                new_rsf_wide = new_rsf_wide.add_prefix( pronum )
 8231                new_rsf_wide.set_index( mm_wide.index, inplace=True )
 8232                ofn = output_prefix + separator + pronum + '.csv'
 8233                new_rsf_wide.to_csv( ofn )
 8234                mm_wide = pd.concat( [mm_wide, new_rsf_wide ], axis=1, ignore_index=False )
 8235                for mykey in get_antsimage_keys( rsfpro ):
 8236                    myop = output_prefix + separator + pronum + mykey + '.nii.gz'
 8237                    image_write_with_thumbnail( rsfpro[mykey], myop, thumb=True )
 8238                ofn = output_prefix + separator + pronum + 'rsfcorr.csv'
 8239                rsfpro['corr'].to_csv( ofn )
 8240                # apply same principle to new correlation matrix, doesn't need to be incorporated with mm_wide
 8241                ofn2 = output_prefix + separator + pronum + 'nodescorr.csv'
 8242                rsfpro['fullCorrMat'].to_csv( ofn2 )
 8243    if 'DTI' in mm:
 8244        if mm['DTI'] is not None:
 8245            mydti = mm['DTI']
 8246            mm_wide['dti_tsnr_b0_mean'] =  mydti['tsnr_b0'].mean()
 8247            mm_wide['dti_tsnr_dwi_mean'] =  mydti['tsnr_dwi'].mean()
 8248            mm_wide['dti_dvars_b0_mean'] =  mydti['dvars_b0'].mean()
 8249            mm_wide['dti_dvars_dwi_mean'] =  mydti['dvars_dwi'].mean()
 8250            mm_wide['dti_ssnr_b0_mean'] =  mydti['ssnr_b0'].mean()
 8251            mm_wide['dti_ssnr_dwi_mean'] =  mydti['ssnr_dwi'].mean()
 8252            mm_wide['dti_fa_evr'] =  mydti['fa_evr']
 8253            mm_wide['dti_fa_SNR'] =  mydti['fa_SNR']
 8254            if mydti['framewise_displacement'] is not None:
 8255                mm_wide['dti_high_motion_count'] =  mydti['high_motion_count']
 8256                mm_wide['dti_FD_mean'] = mydti['framewise_displacement'].mean()
 8257                mm_wide['dti_FD_max'] = mydti['framewise_displacement'].max()
 8258                mm_wide['dti_FD_sd'] = mydti['framewise_displacement'].std()
 8259                fdfn = output_prefix + separator + '_fd.csv'
 8260            else:
 8261                mm_wide['dti_FD_mean'] = mm_wide['dti_FD_max'] = mm_wide['dti_FD_sd'] = 'NA'
 8262
 8263    if 'perf' in mm:
 8264        if mm['perf'] is not None:
 8265            perfpro = mm['perf']
 8266            prwide = dict_to_dataframe( perfpro )
 8267            if mm_wide.shape[0] > 0 and prwide.shape[0] > 0:
 8268                prwide.set_index( mm_wide.index, inplace=True )
 8269            mm_wide = pd.concat( [mm_wide, prwide ], axis=1, ignore_index=False )
 8270            if 'perf_dataframe' in perfpro.keys():
 8271                pderk = perfpro['perf_dataframe'].iloc[: , 1:]
 8272                pderk.set_index( mm_wide.index, inplace=True )
 8273                mm_wide = pd.concat( [ mm_wide, pderk ], axis=1, ignore_index=False )
 8274            else:
 8275                print("FIXME - perfusion dataframe")
 8276            for mykey in get_antsimage_keys( mm['perf'] ):
 8277                tempfn = output_prefix + separator + mykey + '.nii.gz'
 8278                image_write_with_thumbnail( mm['perf'][mykey], tempfn, thumb=False )
 8279
 8280    if 'pet3d' in mm:
 8281        if mm['pet3d'] is not None:
 8282            pet3dpro = mm['pet3d']
 8283            prwide = dict_to_dataframe( pet3dpro )
 8284            if mm_wide.shape[0] > 0 and prwide.shape[0] > 0:
 8285                prwide.set_index( mm_wide.index, inplace=True )
 8286            mm_wide = pd.concat( [mm_wide, prwide ], axis=1, ignore_index=False )
 8287            if 'pet3d_dataframe' in pet3dpro.keys():
 8288                pderk = pet3dpro['pet3d_dataframe'].iloc[: , 1:]
 8289                pderk.set_index( mm_wide.index, inplace=True )
 8290                mm_wide = pd.concat( [ mm_wide, pderk ], axis=1, ignore_index=False )
 8291            else:
 8292                print("FIXME - pet3dusion dataframe")
 8293            for mykey in get_antsimage_keys( mm['pet3d'] ):
 8294                tempfn = output_prefix + separator + mykey + '.nii.gz'
 8295                image_write_with_thumbnail( mm['pet3d'][mykey], tempfn, thumb=False )
 8296
 8297    mmwidefn = output_prefix + separator + 'mmwide.csv'
 8298    mm_wide.to_csv( mmwidefn )
 8299    if verbose:
 8300        print( output_prefix + " write_mm done." )
 8301    return
 8302
 8303
 8304def mm_nrg(
 8305    studyid,   # pandas data frame
 8306    sourcedir = os.path.expanduser( "~/data/PPMI/MV/example_s3_b/images/PPMI/" ),
 8307    sourcedatafoldername = 'images', # root for source data
 8308    processDir = "processed", # where output will go - parallel to sourcedatafoldername
 8309    mysep = '-', # define a separator for filename components
 8310    srmodel_T1 = False, # optional - will add a great deal of time
 8311    srmodel_NM = False, # optional - will add a great deal of time
 8312    srmodel_DTI = False, # optional - will add a great deal of time
 8313    visualize = True,
 8314    nrg_modality_list = ["T1w", "NM2DMT", "DTI","T2Flair", "rsfMRI" ],
 8315    verbose = True
 8316):
 8317    """
 8318    too dangerous to document ... use with care.
 8319
 8320    processes multiple modality MRI specifically:
 8321
 8322    * T1w
 8323    * T2Flair
 8324    * DTI, DTI_LR, DTI_RL
 8325    * rsfMRI, rsfMRI_LR, rsfMRI_RL
 8326    * NM2DMT (neuromelanin)
 8327
 8328    other modalities may be added later ...
 8329
 8330    "trust me, i know what i'm doing" - sledgehammer
 8331
 8332    convert to pynb via:
 8333        p2j mm.py -o
 8334
 8335    convert the ipynb to html via:
 8336        jupyter nbconvert ANTsPyMM/tests/mm.ipynb --execute --to html
 8337
 8338    this function assumes NRG format for the input data ....
 8339    we also assume that t1w hierarchical (if already done) was written
 8340    via its standardized write function.
 8341    NRG = https://github.com/stnava/biomedicalDataOrganization
 8342
 8343    this function is verbose
 8344
 8345    Parameters
 8346    -------------
 8347
 8348    studyid : must have columns 1. subjectID 2. date (in form 20220228) and 3. imageID
 8349        other relevant columns include nmid1-10, rsfid1, rsfid2, dtid1, dtid2, flairid;
 8350        these provide unique image IDs for these modalities: nm=neuromelanin, dti=diffusion tensor,
 8351        rsf=resting state fmri, flair=T2Flair.  none of these are required. only
 8352        t1 is required.  rsfid1/rsfid2 will be processed jointly. same for dtid1/dtid2 and nmid*.  see antspymm.generate_mm_dataframe
 8353
 8354    sourcedir : a study specific folder containing individual subject folders
 8355
 8356    sourcedatafoldername : root for source data e.g. "images"
 8357
 8358    processDir : where output will go - parallel to sourcedatafoldername e.g.
 8359        "processed"
 8360
 8361    mysep : define a character separator for filename components
 8362
 8363    srmodel_T1 : False (default) - will add a great deal of time - or h5 filename, 2 chan
 8364
 8365    srmodel_NM : False (default) - will add a great deal of time - or h5 filename, 1 chan
 8366
 8367    srmodel_DTI : False (default) - will add a great deal of time - or h5 filename, 1 chan
 8368
 8369    visualize : True - will plot some results to png
 8370
 8371    nrg_modality_list : list of permissible modalities - always include [T1w] as base
 8372
 8373    verbose : boolean
 8374
 8375    Returns
 8376    ---------
 8377
 8378    writes output to disk and potentially produces figures that may be
 8379    captured in a ipynb / html file.
 8380
 8381    """
 8382    studyid = studyid.dropna(axis=1)
 8383    if studyid.shape[0] < 1:
 8384        raise ValueError('studyid has no rows')
 8385    musthavecols = ['subjectID','date','imageID']
 8386    for k in range(len(musthavecols)):
 8387        if not musthavecols[k] in studyid.keys():
 8388            raise ValueError('studyid is missing column ' +musthavecols[k] )
 8389    def makewideout( x, separator = '-' ):
 8390        return x + separator + 'mmwide.csv'
 8391    if nrg_modality_list[0] != 'T1w':
 8392        nrg_modality_list.insert(0, "T1w" )
 8393    testloop = False
 8394    counter=0
 8395    import glob as glob
 8396    from os.path import exists
 8397    ex_path = os.path.expanduser( "~/.antspyt1w/" )
 8398    ex_pathmm = os.path.expanduser( "~/.antspymm/" )
 8399    templatefn = ex_path + 'CIT168_T1w_700um_pad_adni.nii.gz'
 8400    if not exists( templatefn ):
 8401        print( "**missing files** => call get_data from latest antspyt1w and antspymm." )
 8402        antspyt1w.get_data( force_download=True )
 8403        get_data( force_download=True )
 8404    temp = sourcedir.split( "/" )
 8405    splitCount = len( temp )
 8406    template = mm_read( templatefn ) # Read in template
 8407    test_run = False
 8408    if test_run:
 8409        visualize=False
 8410    # get sid and dtid from studyid
 8411    sid = str(studyid['subjectID'].iloc[0])
 8412    dtid = str(studyid['date'].iloc[0])
 8413    iid = str(studyid['imageID'].iloc[0])
 8414    subjectrootpath = os.path.join(sourcedir,sid, dtid)
 8415    if verbose:
 8416        print("subjectrootpath: "+ subjectrootpath )
 8417    myimgsInput = glob.glob( subjectrootpath+"/*" )
 8418    myimgsInput.sort( )
 8419    if verbose:
 8420        print( myimgsInput )
 8421    # hierarchical
 8422    # NOTE: if there are multiple T1s for this time point, should take
 8423    # the one with the highest resnetGrade
 8424    t1_search_path = os.path.join(subjectrootpath, "T1w", iid, "*nii.gz")
 8425    if verbose:
 8426        print(f"t1 search path: {t1_search_path}")
 8427    t1fn = glob.glob(t1_search_path)
 8428    t1fn.sort()
 8429    if len( t1fn ) < 1:
 8430        raise ValueError('mm_nrg cannot find the T1w with uid ' + iid + ' @ ' + subjectrootpath )
 8431    t1fn = t1fn[0]
 8432    t1 = mm_read( t1fn )
 8433    hierfn0 = re.sub( sourcedatafoldername, processDir, t1fn)
 8434    hierfn0 = re.sub( ".nii.gz", "", hierfn0)
 8435    hierfn = re.sub( "T1w", "T1wHierarchical", hierfn0)
 8436    hierfn = hierfn + mysep
 8437    hierfntest = hierfn + 'snseg.csv'
 8438    regout = hierfn0 + mysep + "syn"
 8439    templateTx = {
 8440        'fwdtransforms': [ regout+'1Warp.nii.gz', regout+'0GenericAffine.mat'],
 8441        'invtransforms': [ regout+'0GenericAffine.mat', regout+'1InverseWarp.nii.gz']  }
 8442    if verbose:
 8443        print( "-<REGISTRATION EXISTENCE>-: \n" + 
 8444              "NAMING: " + regout+'0GenericAffine.mat' + " \n " +
 8445            str(exists( templateTx['fwdtransforms'][0])) + " " +
 8446            str(exists( templateTx['fwdtransforms'][1])) + " " +
 8447            str(exists( templateTx['invtransforms'][0])) + " " +
 8448            str(exists( templateTx['invtransforms'][1])) )
 8449    if verbose:
 8450        print( hierfntest )
 8451    hierexists = exists( hierfntest ) # FIXME should test this explicitly but we assume it here
 8452    hier = None
 8453    if not hierexists and not testloop:
 8454        subjectpropath = os.path.dirname( hierfn )
 8455        if verbose:
 8456            print( subjectpropath )
 8457        os.makedirs( subjectpropath, exist_ok=True  )
 8458        hier = antspyt1w.hierarchical( t1, hierfn, labels_to_register=None )
 8459        antspyt1w.write_hierarchical( hier, hierfn )
 8460        t1wide = antspyt1w.merge_hierarchical_csvs_to_wide_format(
 8461                hier['dataframes'], identifier=None )
 8462        t1wide.to_csv( hierfn + 'mmwide.csv' )
 8463    ################# read the hierarchical data ###############################
 8464    hier = antspyt1w.read_hierarchical( hierfn )
 8465    if exists( hierfn + 'mmwide.csv' ) :
 8466        t1wide = pd.read_csv( hierfn + 'mmwide.csv' )
 8467    elif not testloop:
 8468        t1wide = antspyt1w.merge_hierarchical_csvs_to_wide_format(
 8469                hier['dataframes'], identifier=None )
 8470    if srmodel_T1 is not None :
 8471        hierfnSR = re.sub( sourcedatafoldername, processDir, t1fn)
 8472        hierfnSR = re.sub( "T1w", "T1wHierarchicalSR", hierfnSR)
 8473        hierfnSR = re.sub( ".nii.gz", "", hierfnSR)
 8474        hierfnSR = hierfnSR + mysep
 8475        hierfntest = hierfnSR + 'mtl.csv'
 8476        if verbose:
 8477            print( hierfntest )
 8478        hierexists = exists( hierfntest ) # FIXME should test this explicitly but we assume it here
 8479        if not hierexists:
 8480            subjectpropath = os.path.dirname( hierfnSR )
 8481            if verbose:
 8482                print( subjectpropath )
 8483            os.makedirs( subjectpropath, exist_ok=True  )
 8484            # hierarchical_to_sr(t1hier, sr_model, tissue_sr=False, blending=0.5, verbose=False)
 8485            bestup = siq.optimize_upsampling_shape( ants.get_spacing(t1), modality='T1' )
 8486            mdlfn = re.sub( 'bestup', bestup, srmodel_T1 ) 
 8487            if verbose:
 8488                print( mdlfn )
 8489            if exists( mdlfn ):
 8490                srmodel_T1_mdl = tf.keras.models.load_model( mdlfn, compile=False )
 8491            else:
 8492                print( mdlfn + " does not exist - will not run.")
 8493            hierSR = antspyt1w.hierarchical_to_sr( hier, srmodel_T1_mdl, blending=None, tissue_sr=False )
 8494            antspyt1w.write_hierarchical( hierSR, hierfnSR )
 8495            t1wideSR = antspyt1w.merge_hierarchical_csvs_to_wide_format(
 8496                    hierSR['dataframes'], identifier=None )
 8497            t1wideSR.to_csv( hierfnSR + 'mmwide.csv' )
 8498    hier = antspyt1w.read_hierarchical( hierfn )
 8499    if exists( hierfn + 'mmwide.csv' ) :
 8500        t1wide = pd.read_csv( hierfn + 'mmwide.csv' )
 8501    elif not testloop:
 8502        t1wide = antspyt1w.merge_hierarchical_csvs_to_wide_format(
 8503                hier['dataframes'], identifier=None )
 8504    if not testloop:
 8505        t1imgbrn = hier['brain_n4_dnz']
 8506        t1atropos = hier['dkt_parc']['tissue_segmentation']
 8507    # loop over modalities and then unique image IDs
 8508    # we treat NM in a "special" way -- aggregating repeats
 8509    # other modalities (beyond T1) are treated individually
 8510    nimages = len(myimgsInput)
 8511    if verbose:
 8512        print(  " we have : " + str(nimages) + " modalities.")
 8513    for overmodX in nrg_modality_list:
 8514        counter=counter+1
 8515        if counter > (len(nrg_modality_list)+1):
 8516            print("This is weird. " + str(counter))
 8517            return
 8518        if overmodX == 'T1w':
 8519            iidOtherMod = iid
 8520            mod_search_path = os.path.join(subjectrootpath, overmodX, iidOtherMod, "*nii.gz")
 8521            myimgsr = glob.glob(mod_search_path)
 8522        elif overmodX == 'NM2DMT' and ('nmid1' in studyid.keys() ):
 8523            iidOtherMod = str( int(studyid['nmid1'].iloc[0]) )
 8524            mod_search_path = os.path.join(subjectrootpath, overmodX, iidOtherMod, "*nii.gz")
 8525            myimgsr = glob.glob(mod_search_path)
 8526            for nmnum in range(2,11):
 8527                locnmnum = 'nmid'+str(nmnum)
 8528                if locnmnum in studyid.keys() :
 8529                    iidOtherMod = str( int(studyid[locnmnum].iloc[0]) )
 8530                    mod_search_path = os.path.join(subjectrootpath, overmodX, iidOtherMod, "*nii.gz")
 8531                    myimgsr.append( glob.glob(mod_search_path)[0] )
 8532        elif 'rsfMRI' in overmodX and ( ( 'rsfid1' in studyid.keys() ) or ('rsfid2' in studyid.keys() ) ):
 8533            myimgsr = []
 8534            if  'rsfid1' in studyid.keys():
 8535                iidOtherMod = str( int(studyid['rsfid1'].iloc[0]) )
 8536                mod_search_path = os.path.join(subjectrootpath, overmodX+"*", iidOtherMod, "*nii.gz")
 8537                myimgsr.append( glob.glob(mod_search_path)[0] )
 8538            if  'rsfid2' in studyid.keys():
 8539                iidOtherMod = str( int(studyid['rsfid2'].iloc[0]) )
 8540                mod_search_path = os.path.join(subjectrootpath, overmodX+"*", iidOtherMod, "*nii.gz")
 8541                myimgsr.append( glob.glob(mod_search_path)[0] )
 8542        elif 'DTI' in overmodX and (  'dtid1' in studyid.keys() or  'dtid2' in studyid.keys() ):
 8543            myimgsr = []
 8544            if  'dtid1' in studyid.keys():
 8545                iidOtherMod = str( int(studyid['dtid1'].iloc[0]) )
 8546                mod_search_path = os.path.join(subjectrootpath, overmodX+"*", iidOtherMod, "*nii.gz")
 8547                myimgsr.append( glob.glob(mod_search_path)[0] )
 8548            if  'dtid2' in studyid.keys():
 8549                iidOtherMod = str( int(studyid['dtid2'].iloc[0]) )
 8550                mod_search_path = os.path.join(subjectrootpath, overmodX+"*", iidOtherMod, "*nii.gz")
 8551                myimgsr.append( glob.glob(mod_search_path)[0] )
 8552        elif 'T2Flair' in overmodX and ('flairid' in studyid.keys() ):
 8553            iidOtherMod = str( int(studyid['flairid'].iloc[0]) )
 8554            mod_search_path = os.path.join(subjectrootpath, overmodX, iidOtherMod, "*nii.gz")
 8555            myimgsr = glob.glob(mod_search_path)
 8556        if verbose:
 8557            print( "overmod " + overmodX + " " + iidOtherMod )
 8558            print(f"modality search path: {mod_search_path}")
 8559        myimgsr.sort()
 8560        if len(myimgsr) > 0:
 8561            overmodXx = str(overmodX)
 8562            dowrite=False
 8563            if verbose:
 8564                print( 'overmodX is : ' + overmodXx )
 8565                print( 'example image name is : '  )
 8566                print( myimgsr )
 8567            if overmodXx == 'NM2DMT':
 8568                myimgsr2 = myimgsr
 8569                myimgsr2.sort()
 8570                is4d = False
 8571                temp = ants.image_read( myimgsr2[0] )
 8572                if temp.dimension == 4:
 8573                    is4d = True
 8574                if len( myimgsr2 ) == 1 and not is4d: # check dimension
 8575                    myimgsr2 = myimgsr2 + myimgsr2
 8576                subjectpropath = os.path.dirname( myimgsr2[0] )
 8577                subjectpropath = re.sub( sourcedatafoldername, processDir,subjectpropath )
 8578                if verbose:
 8579                    print( "subjectpropath " + subjectpropath )
 8580                mysplit = subjectpropath.split( "/" )
 8581                os.makedirs( subjectpropath, exist_ok=True  )
 8582                mysplitCount = len( mysplit )
 8583                project = mysplit[mysplitCount-5]
 8584                subject = mysplit[mysplitCount-4]
 8585                date = mysplit[mysplitCount-3]
 8586                modality = mysplit[mysplitCount-2]
 8587                uider = mysplit[mysplitCount-1]
 8588                identifier = mysep.join([project, subject, date, modality ])
 8589                identifier = identifier + "_" + iid
 8590                mymm = subjectpropath + "/" + identifier
 8591                mymmout = makewideout( mymm )
 8592                if verbose and not exists( mymmout ):
 8593                    print( "NM " + mymm  + ' execution ')
 8594                elif verbose and exists( mymmout ) :
 8595                    print( "NM " + mymm + ' complete ' )
 8596                if exists( mymmout ):
 8597                    continue
 8598                if is4d:
 8599                    nmlist = ants.ndimage_to_list( mm_read( myimgsr2[0] ) )
 8600                else:
 8601                    nmlist = []
 8602                    for zz in myimgsr2:
 8603                        nmlist.append( mm_read( zz ) )
 8604                srmodel_NM_mdl = None
 8605                if srmodel_NM is not None:
 8606                    bestup = siq.optimize_upsampling_shape( ants.get_spacing(nmlist[0]), modality='NM', roundit=True )
 8607                    if isinstance( srmodel_NM, str ):
 8608                        mdlfn = re.sub( "bestup", bestup, srmodel_NM )
 8609                    if exists( mdlfn ):
 8610                        if verbose:
 8611                            print(mdlfn)
 8612                        srmodel_NM_mdl = tf.keras.models.load_model( mdlfn, compile=False  )
 8613                    else:
 8614                        print( mdlfn + " does not exist - wont use SR")
 8615                if not testloop:
 8616                    tabPro, normPro = mm( t1, hier,
 8617                            nm_image_list = nmlist,
 8618                            srmodel=srmodel_NM_mdl,
 8619                            do_tractography=False,
 8620                            do_kk=False,
 8621                            do_normalization=templateTx,
 8622                            test_run=test_run,
 8623                            verbose=True )
 8624                    if not test_run:
 8625                        write_mm( output_prefix=mymm, mm=tabPro, mm_norm=normPro, t1wide=None, separator=mysep )
 8626                        nmpro = tabPro['NM']
 8627                        mysl = range( nmpro['NM_avg'].shape[2] )
 8628                    if visualize:
 8629                        mysl = range( nmpro['NM_avg'].shape[2] )
 8630                        ants.plot( nmpro['NM_avg'],  nmpro['t1_to_NM'], slices=mysl, axis=2, title='nm + t1', filename=mymm+mysep+"NMavg.png" )
 8631                        mysl = range( nmpro['NM_avg_cropped'].shape[2] )
 8632                        ants.plot( nmpro['NM_avg_cropped'], axis=2, slices=mysl, overlay_alpha=0.3, title='nm crop', filename=mymm+mysep+"NMavgcrop.png" )
 8633                        ants.plot( nmpro['NM_avg_cropped'], nmpro['t1_to_NM'], axis=2, slices=mysl, overlay_alpha=0.3, title='nm crop + t1', filename=mymm+mysep+"NMavgcropt1.png" )
 8634                        ants.plot( nmpro['NM_avg_cropped'], nmpro['NM_labels'], axis=2, slices=mysl, title='nm crop + labels', filename=mymm+mysep+"NMavgcroplabels.png" )
 8635            else :
 8636                if len( myimgsr ) > 0:
 8637                    dowrite=False
 8638                    myimgcount = 0
 8639                    if len( myimgsr ) > 0 :
 8640                        myimg = myimgsr[myimgcount]
 8641                        subjectpropath = os.path.dirname( myimg )
 8642                        subjectpropath = re.sub( sourcedatafoldername, processDir, subjectpropath )
 8643                        mysplit = subjectpropath.split("/")
 8644                        mysplitCount = len( mysplit )
 8645                        project = mysplit[mysplitCount-5]
 8646                        date = mysplit[mysplitCount-4]
 8647                        subject = mysplit[mysplitCount-3]
 8648                        mymod = mysplit[mysplitCount-2] # FIXME system dependent
 8649                        uid = mysplit[mysplitCount-1] # unique image id
 8650                        os.makedirs( subjectpropath, exist_ok=True  )
 8651                        if mymod == 'T1w':
 8652                            identifier = mysep.join([project, date, subject, mymod, uid])
 8653                        else:  # add the T1 unique id since that drives a lot of the analysis
 8654                            identifier = mysep.join([project, date, subject, mymod, uid ])
 8655                            identifier = identifier + "_" + iid
 8656                        mymm = subjectpropath + "/" + identifier
 8657                        mymmout = makewideout( mymm )
 8658                        if verbose and not exists( mymmout ):
 8659                            print("Modality specific processing: " + mymod + " execution " )
 8660                            print( mymm )
 8661                        elif verbose and exists( mymmout ) :
 8662                            print("Modality specific processing: " + mymod + " complete " )
 8663                        if exists( mymmout ) :
 8664                            continue
 8665                        if verbose:
 8666                            print(subjectpropath)
 8667                            print(identifier)
 8668                            print( myimg )
 8669                        if not testloop:
 8670                            img = mm_read( myimg )
 8671                            ishapelen = len( img.shape )
 8672                            if mymod == 'T1w' and ishapelen == 3: # for a real run, set to True
 8673                                if not exists( regout + "logjacobian.nii.gz" ) or not exists( regout+'1Warp.nii.gz' ):
 8674                                    if verbose:
 8675                                        print('start t1 registration')
 8676                                    ex_path = os.path.expanduser( "~/.antspyt1w/" )
 8677                                    templatefn = ex_path + 'CIT168_T1w_700um_pad_adni.nii.gz'
 8678                                    template = mm_read( templatefn )
 8679                                    template = ants.resample_image( template, [1,1,1], use_voxels=False )
 8680                                    t1reg = ants.registration( template, hier['brain_n4_dnz'],
 8681                                        "antsRegistrationSyNQuickRepro[s]", outprefix = regout, verbose=False )
 8682                                    myjac = ants.create_jacobian_determinant_image( template,
 8683                                        t1reg['fwdtransforms'][0], do_log=True, geom=True )
 8684                                    image_write_with_thumbnail( myjac, regout + "logjacobian.nii.gz", thumb=False )
 8685                                    if visualize:
 8686                                        ants.plot( ants.iMath(t1reg['warpedmovout'],"Normalize"),  axis=2, nslices=21, ncol=7, crop=True, title='warped to template', filename=regout+"totemplate.png" )
 8687                                        ants.plot( ants.iMath(myjac,"Normalize"),  axis=2, nslices=21, ncol=7, crop=True, title='jacobian', filename=regout+"jacobian.png" )
 8688                                if not exists( mymm + mysep + "kk_norm.nii.gz" ):
 8689                                    dowrite=True
 8690                                    if verbose:
 8691                                        print('start kk')
 8692                                    tabPro, normPro = mm( t1, hier,
 8693                                        srmodel=None,
 8694                                        do_tractography=False,
 8695                                        do_kk=True,
 8696                                        do_normalization=templateTx,
 8697                                        test_run=test_run,
 8698                                        verbose=True )
 8699                                    if visualize:
 8700                                        maxslice = np.min( [21, hier['brain_n4_dnz'].shape[2] ] )
 8701                                        ants.plot( hier['brain_n4_dnz'],  axis=2, nslices=maxslice, ncol=7, crop=True, title='brain extraction', filename=mymm+mysep+"brainextraction.png" )
 8702                                        ants.plot( tabPro['kk']['thickness_image'], axis=2, nslices=maxslice, ncol=7, crop=True, title='kk',
 8703                                        cmap='plasma', filename=mymm+mysep+"kkthickness.png" )
 8704                            if mymod == 'T2Flair' and ishapelen == 3:
 8705                                dowrite=True
 8706                                tabPro, normPro = mm( t1, hier,
 8707                                    flair_image = img,
 8708                                    srmodel=None,
 8709                                    do_tractography=False,
 8710                                    do_kk=False,
 8711                                    do_normalization=templateTx,
 8712                                    test_run=test_run,
 8713                                    verbose=True )
 8714                                if visualize:
 8715                                    maxslice = np.min( [21, img.shape[2] ] )
 8716                                    ants.plot_ortho( img, crop=True, title='Flair', filename=mymm+mysep+"flair.png", flat=True )
 8717                                    ants.plot_ortho( img, tabPro['flair']['WMH_probability_map'], crop=True, title='Flair + WMH', filename=mymm+mysep+"flairWMH.png", flat=True )
 8718                                    if tabPro['flair']['WMH_posterior_probability_map'] is not None:
 8719                                        ants.plot_ortho( img, tabPro['flair']['WMH_posterior_probability_map'],  crop=True, title='Flair + prior WMH', filename=mymm+mysep+"flairpriorWMH.png", flat=True )
 8720                            if ( mymod == 'rsfMRILR' or mymod == 'rsfMRIRL' or mymod == 'rsfMRI_LR' or mymod == 'rsfMRI_RL' or mymod == 'rsfMRI' )  and ishapelen == 4:
 8721                                img2 = None
 8722                                if len( myimgsr ) > 1:
 8723                                    img2 = mm_read( myimgsr[myimgcount+1] )
 8724                                    ishapelen2 = len( img2.shape )
 8725                                    if ishapelen2 != 4 :
 8726                                        img2 = None
 8727                                dowrite=True
 8728                                tabPro, normPro = mm( t1, hier,
 8729                                    rsf_image=[img,img2],
 8730                                    srmodel=None,
 8731                                    do_tractography=False,
 8732                                    do_kk=False,
 8733                                    do_normalization=templateTx,
 8734                                    test_run=test_run,
 8735                                    verbose=True )
 8736                                if tabPro['rsf'] is not None and visualize:
 8737                                    dfn=tabPro['rsf']['dfnname']
 8738                                    maxslice = np.min( [21, tabPro['rsf']['meanBold'].shape[2] ] )
 8739                                    ants.plot( tabPro['rsf']['meanBold'],
 8740                                        axis=2, nslices=maxslice, ncol=7, crop=True, title='meanBOLD', filename=mymm+mysep+"meanBOLD.png" )
 8741                                    ants.plot( tabPro['rsf']['meanBold'], ants.iMath(tabPro['rsf']['alff'],"Normalize"),
 8742                                        axis=2, nslices=maxslice, ncol=7, crop=True, title='ALFF', filename=mymm+mysep+"boldALFF.png" )
 8743                                    ants.plot( tabPro['rsf']['meanBold'], ants.iMath(tabPro['rsf']['falff'],"Normalize"),
 8744                                        axis=2, nslices=maxslice, ncol=7, crop=True, title='fALFF', filename=mymm+mysep+"boldfALFF.png" )
 8745                                    ants.plot( tabPro['rsf']['meanBold'], tabPro['rsf'][dfn],
 8746                                        axis=2, nslices=maxslice, ncol=7, crop=True, title='DefaultMode', filename=mymm+mysep+"boldDefaultMode.png" )
 8747                            if ( mymod == 'DTILR' or mymod == 'DTIRL' or mymod == 'DTI_LR' or mymod == 'DTI_RL' or mymod == 'DTI' ) and ishapelen == 4:
 8748                                dowrite=True
 8749                                bvalfn = re.sub( '.nii.gz', '.bval' , myimg )
 8750                                bvecfn = re.sub( '.nii.gz', '.bvec' , myimg )
 8751                                imgList = [ img ]
 8752                                bvalfnList = [ bvalfn ]
 8753                                bvecfnList = [ bvecfn ]
 8754                                if len( myimgsr ) > 1:  # find DTI_RL
 8755                                    dtilrfn = myimgsr[myimgcount+1]
 8756                                    if len( dtilrfn ) == 1:
 8757                                        bvalfnRL = re.sub( '.nii.gz', '.bval' , dtilrfn )
 8758                                        bvecfnRL = re.sub( '.nii.gz', '.bvec' , dtilrfn )
 8759                                        imgRL = ants.image_read( dtilrfn )
 8760                                        imgList.append( imgRL )
 8761                                        bvalfnList.append( bvalfnRL )
 8762                                        bvecfnList.append( bvecfnRL )
 8763                                srmodel_DTI_mdl=None
 8764                                if srmodel_DTI is not None:
 8765                                    temp = ants.get_spacing(img)
 8766                                    dtspc=[temp[0],temp[1],temp[2]]
 8767                                    bestup = siq.optimize_upsampling_shape( dtspc, modality='DTI' )
 8768                                    if isinstance( srmodel_DTI, str ):
 8769                                        mdlfn = re.sub( "bestup", bestup, srmodel_DTI )
 8770                                    if exists( mdlfn ):
 8771                                        if verbose:
 8772                                            print(mdlfn)
 8773                                        srmodel_DTI_mdl = tf.keras.models.load_model( mdlfn, compile=False )
 8774                                    else:
 8775                                        print(mdlfn + " does not exist - wont use SR")
 8776                                tabPro, normPro = mm( t1, hier,
 8777                                    dw_image=imgList,
 8778                                    bvals = bvalfnList,
 8779                                    bvecs = bvecfnList,
 8780                                    srmodel=srmodel_DTI_mdl,
 8781                                    do_tractography=not test_run,
 8782                                    do_kk=False,
 8783                                    do_normalization=templateTx,
 8784                                    test_run=test_run,
 8785                                    verbose=True )
 8786                                mydti = tabPro['DTI']
 8787                                if visualize:
 8788                                    maxslice = np.min( [21, mydti['recon_fa'] ] )
 8789                                    ants.plot( mydti['recon_fa'],  axis=2, nslices=maxslice, ncol=7, crop=True, title='FA', filename=mymm+mysep+"FAbetter.png"  )
 8790                                    ants.plot( mydti['recon_fa'], mydti['jhu_labels'], axis=2, nslices=maxslice, ncol=7, crop=True, title='FA + JHU', filename=mymm+mysep+"FAJHU.png"  )
 8791                                    ants.plot( mydti['recon_md'],  axis=2, nslices=maxslice, ncol=7, crop=True, title='MD', filename=mymm+mysep+"MD.png"  )
 8792                            if dowrite:
 8793                                write_mm( output_prefix=mymm, mm=tabPro, mm_norm=normPro, t1wide=t1wide, separator=mysep, verbose=True )
 8794                                for mykey in normPro.keys():
 8795                                    if normPro[mykey] is not None:
 8796                                        if visualize and normPro[mykey].components == 1 and False:
 8797                                            ants.plot( template, normPro[mykey], axis=2, nslices=21, ncol=7, crop=True, title=mykey, filename=mymm+mysep+mykey+".png"   )
 8798        if overmodX == nrg_modality_list[ len( nrg_modality_list ) - 1 ]:
 8799            return
 8800        if verbose:
 8801            print("done with " + overmodX )
 8802    if verbose:
 8803        print("mm_nrg complete.")
 8804    return
 8805
 8806
 8807
 8808def mm_csv(
 8809    studycsv,   # pandas data frame
 8810    mysep = '-', # or "_" for BIDS
 8811    srmodel_T1 = False, # optional - will add a great deal of time
 8812    srmodel_NM = False, # optional - will add a great deal of time
 8813    srmodel_DTI = False, # optional - will add a great deal of time
 8814    dti_motion_correct = 'antsRegistrationSyNQuickRepro[r]',
 8815    dti_denoise = False,
 8816    nrg_modality_list = None,
 8817    normalization_template = None,
 8818    normalization_template_output = None,
 8819    normalization_template_transform_type = "antsRegistrationSyNRepro[s]",
 8820    normalization_template_spacing=None,
 8821    enantiomorphic=False,
 8822    perfusion_trim = 10,
 8823    perfusion_m0_image = None,
 8824    perfusion_m0 = None,
 8825    rsf_upsampling = 3.0,
 8826    pet3d = None,
 8827    min_t1_spacing_for_sr = 0.8,
 8828):
 8829    """
 8830    too dangerous to document ... use with care.
 8831
 8832    processes multiple modality MRI specifically:
 8833
 8834    * T1w
 8835    * T2Flair
 8836    * DTI, DTI_LR, DTI_RL
 8837    * rsfMRI, rsfMRI_LR, rsfMRI_RL
 8838    * NM2DMT (neuromelanin)
 8839
 8840    other modalities may be added later ...
 8841
 8842    "trust me, i know what i'm doing" - sledgehammer
 8843
 8844    convert to pynb via:
 8845        p2j mm.py -o
 8846
 8847    convert the ipynb to html via:
 8848        jupyter nbconvert ANTsPyMM/tests/mm.ipynb --execute --to html
 8849
 8850    this function does not assume NRG format for the input data ....
 8851
 8852    Parameters
 8853    -------------
 8854
 8855    studycsv : must have columns:
 8856        - subjectID
 8857        - date or session
 8858        - imageID
 8859        - modality
 8860        - sourcedir
 8861        - outputdir
 8862        - filename (path to the t1 image)
 8863        other relevant columns include nmid1-10, rsfid1, rsfid2, dtid1, dtid2, flairid;
 8864        these provide filenames for these modalities: nm=neuromelanin, dti=diffusion tensor,
 8865        rsf=resting state fmri, flair=T2Flair.  none of these are required. only
 8866        t1 is required. rsfid1/rsfid2 will be processed jointly. same for dtid1/dtid2 and nmid*.
 8867        see antspymm.generate_mm_dataframe
 8868
 8869    sourcedir : a study specific folder containing individual subject folders
 8870
 8871    outputdir : a study specific folder where individual output subject folders will go
 8872
 8873    filename : the raw image filename (full path)
 8874
 8875    srmodel_T1 : None (default) - .keras or h5 filename for SR model (siq generated). 
 8876
 8877    srmodel_NM : None (default) - .keras or h5 filename for SR model (siq generated)
 8878    the model name should follow a style like prefix_bestup_postfix where bestup will be replaced with an optimal upsampling factor eg 2x2x2 based on the data.  see siq.optimize_upsampling_shape.
 8879
 8880    srmodel_DTI : None (default) - .keras or h5 filename for SR model (siq generated). 
 8881    the model name should follow a style like prefix_bestup_postfix where bestup will be replaced with an optimal upsampling factor eg 2x2x2 based on the data.  see siq.optimize_upsampling_shape.
 8882
 8883    dti_motion_correct : None, Rigid or SyN
 8884
 8885    dti_denoise : boolean
 8886
 8887    nrg_modality_list : optional; defaults to None; use to focus on a given modality
 8888
 8889    normalization_template : optional; defaults to None; if present, all images will
 8890        be deformed into this space and the deformation will be stored with an extension
 8891        related to this variable.  this should be a brain extracted T1w image.
 8892
 8893    normalization_template_output : optional string; defaults to None; naming for the 
 8894        normalization_template outputs which will be in the T1w directory.
 8895
 8896    normalization_template_transform_type : optional string transform type passed to ants.registration
 8897
 8898    normalization_template_spacing : 3-tuple controlling the resolution at which registration is computed 
 8899    
 8900    enantiomorphic: boolean (WIP)
 8901
 8902    perfusion_trim : optional integer number of time volumes to exclude from the front of the perfusion time series
 8903
 8904    perfusion_m0_image : optional m0 antsImage associated with the perfusion time series
 8905
 8906    perfusion_m0 : optional list containing indices of the m0 in the perfusion time series
 8907
 8908    rsf_upsampling : optional upsampling parameter value in mm; if set to zero, no upsampling is done
 8909
 8910    pet3d : optional antsImage for PET (or other 3d scalar) data which we want to summarize
 8911
 8912    min_t1_spacing_for_sr : float 
 8913        if the minimum input image spacing is less than this value, 
 8914        the function will return the original image.  Default 0.8.
 8915
 8916    Returns
 8917    ---------
 8918
 8919    writes output to disk and produces figures
 8920
 8921    """
 8922    import traceback
 8923    visualize = True
 8924    verbose = True
 8925    if verbose:
 8926        print( version() )
 8927    if nrg_modality_list is None:
 8928        nrg_modality_list = get_valid_modalities()
 8929    if studycsv.shape[0] < 1:
 8930        raise ValueError('studycsv has no rows')
 8931    musthavecols = ['projectID', 'subjectID','date','imageID','modality','sourcedir','outputdir','filename']
 8932    for k in range(len(musthavecols)):
 8933        if not musthavecols[k] in studycsv.keys():
 8934            raise ValueError('studycsv is missing column ' +musthavecols[k] )
 8935    def makewideout( x, separator = mysep ):
 8936        return x + separator + 'mmwide.csv'
 8937    testloop = False
 8938    counter=0
 8939    import glob as glob
 8940    from os.path import exists
 8941    ex_path = os.path.expanduser( "~/.antspyt1w/" )
 8942    ex_pathmm = os.path.expanduser( "~/.antspymm/" )
 8943    templatefn = ex_path + 'CIT168_T1w_700um_pad_adni.nii.gz'
 8944    if not exists( templatefn ):
 8945        print( "**missing files** => call get_data from latest antspyt1w and antspymm." )
 8946        antspyt1w.get_data( force_download=True )
 8947        get_data( force_download=True )
 8948    template = mm_read( templatefn ) # Read in template
 8949    test_run = False
 8950    if test_run:
 8951        visualize=False
 8952    # get sid and dtid from studycsv
 8953    # musthavecols = ['projectID','subjectID','date','imageID','modality','sourcedir','outputdir','filename']
 8954    projid = str(studycsv['projectID'].iloc[0])
 8955    sid = str(studycsv['subjectID'].iloc[0])
 8956    dtid = str(studycsv['date'].iloc[0])
 8957    iid = str(studycsv['imageID'].iloc[0])
 8958    t1iidUse=iid
 8959    modality = str(studycsv['modality'].iloc[0])
 8960    sourcedir = str(studycsv['sourcedir'].iloc[0])
 8961    outputdir = str(studycsv['outputdir'].iloc[0])
 8962    filename = str(studycsv['filename'].iloc[0])
 8963    if not exists(filename):
 8964            raise ValueError('mm_nrg cannot find filename ' + filename + ' in mm_csv' )
 8965
 8966    # hierarchical
 8967    # NOTE: if there are multiple T1s for this time point, should take
 8968    # the one with the highest resnetGrade
 8969    t1fn = filename
 8970    if not exists( t1fn ):
 8971        raise ValueError('mm_nrg cannot find the T1w with uid ' + t1fn )
 8972    t1 = mm_read( t1fn, modality='T1w' )
 8973    minspc = np.min(ants.get_spacing(t1))
 8974    minshape = np.min(t1.shape)
 8975    if minspc < 1e-16:
 8976        warnings.warn('minimum spacing in T1w is too small - cannot process. ' + str(minspc) )
 8977        return
 8978    if minshape < 32:
 8979        warnings.warn('minimum shape in T1w is too small - cannot process. ' + str(minshape) )
 8980        return
 8981
 8982    if enantiomorphic:
 8983        t1 = enantiomorphic_filling_without_mask( t1, axis=0 )[0]
 8984    hierfn = outputdir + "/"  + projid + "/" + sid + "/" + dtid + "/" + "T1wHierarchical" + '/' + iid + "/" + projid + mysep + sid + mysep + dtid + mysep + "T1wHierarchical" + mysep + iid + mysep
 8985    hierfnSR = outputdir + "/" + projid + "/"  + sid + "/" + dtid + "/" + "T1wHierarchicalSR" + '/' + iid + "/" + projid + mysep + sid + mysep + dtid + mysep + "T1wHierarchicalSR" + mysep + iid + mysep
 8986    hierfntest = hierfn + 'cerebellum.csv'
 8987    if verbose:
 8988        print( hierfntest )
 8989    regout = re.sub("T1wHierarchical","T1w",hierfn) + "syn"
 8990    templateTx = {
 8991        'fwdtransforms': [ regout+'1Warp.nii.gz', regout+'0GenericAffine.mat'],
 8992        'invtransforms': [ regout+'0GenericAffine.mat', regout+'1InverseWarp.nii.gz']  }
 8993    groupTx = None
 8994    # make the T1w directory
 8995    os.makedirs( os.path.dirname(re.sub("T1wHierarchical","T1w",hierfn)), exist_ok=True  )
 8996    if normalization_template_output is not None:
 8997        normout = re.sub("T1wHierarchical","T1w",hierfn) +  normalization_template_output
 8998        templateNormTx = {
 8999            'fwdtransforms': [ normout+'1Warp.nii.gz', normout+'0GenericAffine.mat'],
 9000            'invtransforms': [ normout+'0GenericAffine.mat', normout+'1InverseWarp.nii.gz']  }
 9001        groupTx = templateNormTx['fwdtransforms']
 9002    if verbose:
 9003        print( "-<REGISTRATION EXISTENCE>-: \n" + 
 9004              "NAMING: " + regout+'0GenericAffine.mat' + " \n " +
 9005            str(exists( templateTx['fwdtransforms'][0])) + " " +
 9006            str(exists( templateTx['fwdtransforms'][1])) + " " +
 9007            str(exists( templateTx['invtransforms'][0])) + " " +
 9008            str(exists( templateTx['invtransforms'][1])) )
 9009    if verbose:
 9010        print( hierfntest )
 9011    hierexists = exists( hierfntest ) and exists( templateTx['fwdtransforms'][0]) and exists( templateTx['fwdtransforms'][1]) and exists( templateTx['invtransforms'][0]) and exists( templateTx['invtransforms'][1])
 9012    hier = None
 9013    if srmodel_T1 is not None:
 9014        srmodel_T1_mdl = tf.keras.models.load_model( srmodel_T1, compile=False )
 9015        if verbose:
 9016            print("Convert T1w to SR via model ", srmodel_T1 )
 9017        t1 = t1w_super_resolution_with_hemispheres( t1, srmodel_T1_mdl,
 9018            min_spacing=min_t1_spacing_for_sr )
 9019    if not hierexists and not testloop:
 9020        subjectpropath = os.path.dirname( hierfn )
 9021        if verbose:
 9022            print( subjectpropath )
 9023        os.makedirs( subjectpropath, exist_ok=True  )
 9024        ants.image_write( t1, hierfn + 'head.nii.gz' )
 9025        hier = antspyt1w.hierarchical( t1, hierfn, labels_to_register=None )
 9026        antspyt1w.write_hierarchical( hier, hierfn )
 9027        t1wide = antspyt1w.merge_hierarchical_csvs_to_wide_format(
 9028                hier['dataframes'], identifier=None )
 9029        t1wide.to_csv( hierfn + 'mmwide.csv' )
 9030    ################# read the hierarchical data ###############################
 9031    # over-write the rbp data with a consistent and recent approach ############
 9032    redograding = True
 9033    if redograding:
 9034        myx = antspyt1w.inspect_raw_t1( 
 9035            ants.image_read(t1fn), hierfn + 'rbp' , option='both' )
 9036        myx['brain'].to_csv( hierfn + 'rbp.csv', index=False )
 9037        myx['brain'].to_csv( hierfn + 'rbpbrain.csv', index=False )
 9038        del myx
 9039
 9040    hier = antspyt1w.read_hierarchical( hierfn )
 9041    t1wide = antspyt1w.merge_hierarchical_csvs_to_wide_format(
 9042        hier['dataframes'], identifier=None )
 9043    rgrade = str( t1wide['resnetGrade'].iloc[0] )
 9044    if t1wide['resnetGrade'].iloc[0] < 0.20:
 9045        warnings.warn('T1w quality check indicates failure: ' + rgrade + " will not process." )
 9046        return
 9047    else:
 9048        print('T1w quality check indicates success: ' + rgrade + " will process." )
 9049
 9050    if srmodel_T1 is not None and False : # deprecated
 9051        hierfntest = hierfnSR + 'mtl.csv'
 9052        if verbose:
 9053            print( hierfntest )
 9054        hierexists = exists( hierfntest ) # FIXME should test this explicitly but we assume it here
 9055        if not hierexists:
 9056            subjectpropath = os.path.dirname( hierfnSR )
 9057            if verbose:
 9058                print( subjectpropath )
 9059            os.makedirs( subjectpropath, exist_ok=True  )
 9060            # hierarchical_to_sr(t1hier, sr_model, tissue_sr=False, blending=0.5, verbose=False)
 9061            bestup = siq.optimize_upsampling_shape( ants.get_spacing(t1), modality='T1' )
 9062            if isinstance( srmodel_T1, str ):
 9063                mdlfn = re.sub( 'bestup', bestup, srmodel_T1 )
 9064            if verbose:
 9065                print( mdlfn )
 9066            if exists( mdlfn ):
 9067                srmodel_T1_mdl = tf.keras.models.load_model( mdlfn, compile=False )
 9068            else:
 9069                print( mdlfn + " does not exist - will not run.")
 9070            hierSR = antspyt1w.hierarchical_to_sr( hier, srmodel_T1_mdl, blending=None, tissue_sr=False )
 9071            antspyt1w.write_hierarchical( hierSR, hierfnSR )
 9072            t1wideSR = antspyt1w.merge_hierarchical_csvs_to_wide_format(
 9073                    hierSR['dataframes'], identifier=None )
 9074            t1wideSR.to_csv( hierfnSR + 'mmwide.csv' )
 9075    hier = antspyt1w.read_hierarchical( hierfn )
 9076    if exists( hierfn + 'mmwide.csv' ) :
 9077        t1wide = pd.read_csv( hierfn + 'mmwide.csv' )
 9078    elif not testloop:
 9079        t1wide = antspyt1w.merge_hierarchical_csvs_to_wide_format(
 9080                hier['dataframes'], identifier=None )
 9081    if not testloop:
 9082        t1imgbrn = hier['brain_n4_dnz']
 9083        t1atropos = hier['dkt_parc']['tissue_segmentation']
 9084
 9085    if not exists( regout + "logjacobian.nii.gz" ) or not exists( regout+'1Warp.nii.gz' ):
 9086        if verbose:
 9087            print('start t1 registration')
 9088        ex_path = os.path.expanduser( "~/.antspyt1w/" )
 9089        templatefn = ex_path + 'CIT168_T1w_700um_pad_adni.nii.gz'
 9090        template = mm_read( templatefn )
 9091        template = ants.resample_image( template, [1,1,1], use_voxels=False )
 9092        t1reg = ants.registration( template, 
 9093            hier['brain_n4_dnz'],
 9094            "antsRegistrationSyNQuickRepro[s]", outprefix = regout, verbose=False )
 9095        myjac = ants.create_jacobian_determinant_image( template,
 9096            t1reg['fwdtransforms'][0], do_log=True, geom=True )
 9097        image_write_with_thumbnail( myjac, regout + "logjacobian.nii.gz", thumb=False )
 9098        if visualize:
 9099            ants.plot( ants.iMath(t1reg['warpedmovout'],"Normalize"),  axis=2, nslices=21, ncol=7, crop=True, title='warped to template', filename=regout+"totemplate.png" )
 9100            ants.plot( ants.iMath(myjac,"Normalize"),  axis=2, nslices=21, ncol=7, crop=True, title='jacobian', filename=regout+"jacobian.png" )
 9101
 9102    if normalization_template_output is not None and normalization_template is not None:
 9103        if verbose:
 9104            print("begin group template registration")
 9105        if not exists( normout+'0GenericAffine.mat' ):
 9106            if normalization_template_spacing is not None:
 9107                normalization_template_rr=ants.resample_image(normalization_template,normalization_template_spacing)
 9108            else:
 9109                normalization_template_rr=normalization_template
 9110            greg = ants.registration( 
 9111                normalization_template_rr, 
 9112                hier['brain_n4_dnz'],
 9113                normalization_template_transform_type,
 9114                outprefix = normout, verbose=False )
 9115            myjac = ants.create_jacobian_determinant_image( template,
 9116                    greg['fwdtransforms'][0], do_log=True, geom=True )
 9117            image_write_with_thumbnail( myjac, normout + "logjacobian.nii.gz", thumb=False )
 9118            if verbose:
 9119                print("end group template registration")
 9120        else:
 9121            if verbose:
 9122                print("group template registration already done")
 9123
 9124    # loop over modalities and then unique image IDs
 9125    # we treat NM in a "special" way -- aggregating repeats
 9126    # other modalities (beyond T1) are treated individually
 9127    for overmodX in nrg_modality_list:
 9128        # define 1. input images 2. output prefix
 9129        mydoc = docsamson( overmodX, studycsv=studycsv, outputdir=outputdir, projid=projid, sid=sid, dtid=dtid, mysep=mysep,t1iid=t1iidUse )
 9130        myimgsr = mydoc['images']
 9131        mymm = mydoc['outprefix']
 9132        mymod = mydoc['modality']
 9133        if verbose:
 9134            print( mydoc )
 9135        if len(myimgsr) > 0:
 9136            dowrite=False
 9137            if verbose:
 9138                print( 'overmodX is : ' + overmodX )
 9139                print( 'example image name is : '  )
 9140                print( myimgsr )
 9141            if overmodX == 'NM2DMT':
 9142                dowrite = True
 9143                visualize = True
 9144                subjectpropath = os.path.dirname( mydoc['outprefix'] )
 9145                if verbose:
 9146                    print("subjectpropath is")
 9147                    print(subjectpropath)
 9148                    os.makedirs( subjectpropath, exist_ok=True  )
 9149                myimgsr2 = myimgsr
 9150                myimgsr2.sort()
 9151                is4d = False
 9152                temp = ants.image_read( myimgsr2[0] )
 9153                if temp.dimension == 4:
 9154                    is4d = True
 9155                if len( myimgsr2 ) == 1 and not is4d: # check dimension
 9156                    myimgsr2 = myimgsr2 + myimgsr2
 9157                mymmout = makewideout( mymm )
 9158                if verbose and not exists( mymmout ):
 9159                    print( "NM " + mymm  + ' execution ')
 9160                elif verbose and exists( mymmout ) :
 9161                    print( "NM " + mymm + ' complete ' )
 9162                if exists( mymmout ):
 9163                    continue
 9164                if is4d:
 9165                    nmlist = ants.ndimage_to_list( mm_read( myimgsr2[0] ) )
 9166                else:
 9167                    nmlist = []
 9168                    for zz in myimgsr2:
 9169                        nmlist.append( mm_read( zz ) )
 9170                srmodel_NM_mdl = None
 9171                if srmodel_NM is not None:
 9172                    bestup = siq.optimize_upsampling_shape( ants.get_spacing(nmlist[0]), modality='NM', roundit=True )
 9173                    mdlfn = ex_pathmm + "siq_default_sisr_" + bestup + "_1chan_featvggL6_best_mdl.keras"
 9174                    if isinstance( srmodel_NM, str ):
 9175                        srmodel_NM = re.sub( "bestup", bestup, srmodel_NM )
 9176                        mdlfn = os.path.join( ex_pathmm, srmodel_NM )
 9177                    if exists( mdlfn ):
 9178                        if verbose:
 9179                            print(mdlfn)
 9180                        srmodel_NM_mdl = tf.keras.models.load_model( mdlfn, compile=False  )
 9181                    else:
 9182                        print( mdlfn + " does not exist - wont use SR")
 9183                if not testloop:
 9184                    try:
 9185                        tabPro, normPro = mm( t1, hier,
 9186                            nm_image_list = nmlist,
 9187                            srmodel=srmodel_NM_mdl,
 9188                            do_tractography=False,
 9189                            do_kk=False,
 9190                            do_normalization=templateTx,
 9191                            group_template = normalization_template,
 9192                            group_transform = groupTx,
 9193                            test_run=test_run,
 9194                            verbose=True )
 9195                    except Exception as e:
 9196                        error_info = traceback.format_exc()
 9197                        print(error_info)
 9198                        visualize=False
 9199                        dowrite=False
 9200                        print(f"antspymmerror occurred while processing {overmodX}: {e}")
 9201                        pass
 9202                    if not test_run:
 9203                        if dowrite:
 9204                            write_mm( output_prefix=mymm, mm=tabPro,
 9205                                mm_norm=normPro, t1wide=None, separator=mysep )
 9206                        if visualize :
 9207                            nmpro = tabPro['NM']
 9208                            mysl = range( nmpro['NM_avg'].shape[2] )
 9209                            ants.plot( nmpro['NM_avg'],  nmpro['t1_to_NM'], slices=mysl, axis=2, title='nm + t1', filename=mymm+mysep+"NMavg.png" )
 9210                            mysl = range( nmpro['NM_avg_cropped'].shape[2] )
 9211                            ants.plot( nmpro['NM_avg_cropped'], axis=2, slices=mysl, overlay_alpha=0.3, title='nm crop', filename=mymm+mysep+"NMavgcrop.png" )
 9212                            ants.plot( nmpro['NM_avg_cropped'], nmpro['t1_to_NM'], axis=2, slices=mysl, overlay_alpha=0.3, title='nm crop + t1', filename=mymm+mysep+"NMavgcropt1.png" )
 9213                            ants.plot( nmpro['NM_avg_cropped'], nmpro['NM_labels'], axis=2, slices=mysl, title='nm crop + labels', filename=mymm+mysep+"NMavgcroplabels.png" )
 9214            else :
 9215                if len( myimgsr ) > 0 :
 9216                    dowrite=False
 9217                    myimgcount=0
 9218                    if len( myimgsr ) > 0 :
 9219                        myimg = myimgsr[ myimgcount ]
 9220                        subjectpropath = os.path.dirname( mydoc['outprefix'] )
 9221                        if verbose:
 9222                            print("subjectpropath is")
 9223                            print(subjectpropath)
 9224                        os.makedirs( subjectpropath, exist_ok=True  )
 9225                        mymmout = makewideout( mymm )
 9226                        if verbose and not exists( mymmout ):
 9227                            print( "Modality specific processing: " + mymod + " execution " )
 9228                            print( mymm )
 9229                        elif verbose and exists( mymmout ) :
 9230                            print("Modality specific processing: " + mymod + " complete " )
 9231                        if exists( mymmout ) :
 9232                            continue
 9233                        if verbose:
 9234                            print( subjectpropath )
 9235                            print( myimg )
 9236                        if not testloop:
 9237                            img = mm_read( myimg )
 9238                            ishapelen = len( img.shape )
 9239                            if mymod == 'T1w' and ishapelen == 3:
 9240                                if not exists( mymm + mysep + "kk_norm.nii.gz" ):
 9241                                    dowrite=True
 9242                                    if verbose:
 9243                                        print('start kk')
 9244                                    try:
 9245                                        tabPro, normPro = mm( t1, hier,
 9246                                            srmodel=None,
 9247                                            do_tractography=False,
 9248                                            do_kk=True,
 9249                                            do_normalization=templateTx,
 9250                                            group_template = normalization_template,
 9251                                            group_transform = groupTx,
 9252                                            test_run=test_run,
 9253                                            verbose=True )
 9254                                    except Exception as e:
 9255                                        error_info = traceback.format_exc()
 9256                                        print(error_info)
 9257                                        visualize=False
 9258                                        dowrite=False
 9259                                        print(f"antspymmerror occurred while processing {overmodX}: {e}")
 9260                                        pass
 9261                                    if visualize:
 9262                                        maxslice = np.min( [21, hier['brain_n4_dnz'].shape[2] ] )
 9263                                        ants.plot( hier['brain_n4_dnz'],  axis=2, nslices=maxslice, ncol=7, crop=True, title='brain extraction', filename=mymm+mysep+"brainextraction.png" )
 9264                                        ants.plot( tabPro['kk']['thickness_image'], axis=2, nslices=maxslice, ncol=7, crop=True, title='kk',
 9265                                        cmap='plasma', filename=mymm+mysep+"kkthickness.png" )
 9266                            if mymod == 'T2Flair' and ishapelen == 3 and np.min(img.shape) > 15:
 9267                                dowrite=True
 9268                                try:
 9269                                    tabPro, normPro = mm( t1, hier,
 9270                                        flair_image = img,
 9271                                        srmodel=None,
 9272                                        do_tractography=False,
 9273                                        do_kk=False,
 9274                                        do_normalization=templateTx,
 9275                                        group_template = normalization_template,
 9276                                        group_transform = groupTx,
 9277                                        test_run=test_run,
 9278                                        verbose=True )
 9279                                except Exception as e:
 9280                                        error_info = traceback.format_exc()
 9281                                        print(error_info)
 9282                                        visualize=False
 9283                                        dowrite=False
 9284                                        print(f"antspymmerror occurred while processing {overmodX}: {e}")
 9285                                        pass
 9286                                if visualize:
 9287                                    maxslice = np.min( [21, img.shape[2] ] )
 9288                                    ants.plot_ortho( img, crop=True, title='Flair', filename=mymm+mysep+"flair.png", flat=True )
 9289                                    ants.plot_ortho( img, tabPro['flair']['WMH_probability_map'], crop=True, title='Flair + WMH', filename=mymm+mysep+"flairWMH.png", flat=True )
 9290                                    if tabPro['flair']['WMH_posterior_probability_map'] is not None:
 9291                                        ants.plot_ortho( img, tabPro['flair']['WMH_posterior_probability_map'],  crop=True, title='Flair + prior WMH', filename=mymm+mysep+"flairpriorWMH.png", flat=True )
 9292                            print( " ASS + " + mymod )
 9293                            if ( mymod in ['rsfMRILR', 'rsfMRIRL', 'rsfMRI_LR', 'rsfMRI_RL', 'rsfMRI'] ) and ishapelen == 4:
 9294                                img2 = None
 9295                                if len( myimgsr ) > 1:
 9296                                    img2 = mm_read( myimgsr[myimgcount+1] )
 9297                                    ishapelen2 = len( img2.shape )
 9298                                    if ishapelen2 != 4 or 1 in img2.shape:
 9299                                        img2 = None
 9300                                if 1 in img.shape:
 9301                                    warnings.warn( 'rsfMRI image shape suggests it is an incorrectly converted mosaic image - will not process.')
 9302                                    dowrite=False
 9303                                    tabPro={'rsf':None}
 9304                                    normPro={'rsf':None}
 9305                                else:
 9306                                    dowrite=True
 9307                                    try:
 9308                                        tabPro, normPro = mm( t1, hier,
 9309                                            rsf_image=[img,img2],
 9310                                            srmodel=None,
 9311                                            do_tractography=False,
 9312                                            do_kk=False,
 9313                                            do_normalization=templateTx,
 9314                                            group_template = normalization_template,
 9315                                            group_transform = groupTx,
 9316                                            rsf_upsampling = rsf_upsampling,
 9317                                            test_run=test_run,
 9318                                            verbose=True )
 9319                                    except Exception as e:
 9320                                        error_info = traceback.format_exc()
 9321                                        print(error_info)
 9322                                        visualize=False
 9323                                        dowrite=False
 9324                                        tabPro={'rsf':None}
 9325                                        normPro={'rsf':None}
 9326                                        print(f"antspymmerror occurred while processing {overmodX}: {e}")
 9327                                        pass
 9328                                if tabPro['rsf'] is not None and visualize:
 9329                                    for tpro in tabPro['rsf']: # FIXMERSF
 9330                                        maxslice = np.min( [21, tpro['meanBold'].shape[2] ] )
 9331                                        tproprefix = mymm+mysep+str(tpro['paramset'])+mysep
 9332                                        ants.plot( tpro['meanBold'],
 9333                                            axis=2, nslices=maxslice, ncol=7, crop=True, title='meanBOLD', filename=tproprefix+"meanBOLD.png" )
 9334                                        ants.plot( tpro['meanBold'], ants.iMath(tpro['alff'],"Normalize"),
 9335                                            axis=2, nslices=maxslice, ncol=7, crop=True, title='ALFF', filename=tproprefix+"boldALFF.png" )
 9336                                        ants.plot( tpro['meanBold'], ants.iMath(tpro['falff'],"Normalize"),
 9337                                            axis=2, nslices=maxslice, ncol=7, crop=True, title='fALFF', filename=tproprefix+"boldfALFF.png" )
 9338                                        dfn=tpro['dfnname']
 9339                                        ants.plot( tpro['meanBold'], tpro[dfn],
 9340                                            axis=2, nslices=maxslice, ncol=7, crop=True, title=dfn, filename=tproprefix+"boldDefaultMode.png" )
 9341                            if ( mymod == 'perf' ) and ishapelen == 4:
 9342                                dowrite=True
 9343                                try:
 9344                                    tabPro, normPro = mm( t1, hier,
 9345                                        perfusion_image=img,
 9346                                        srmodel=None,
 9347                                        do_tractography=False,
 9348                                        do_kk=False,
 9349                                        do_normalization=templateTx,
 9350                                        group_template = normalization_template,
 9351                                        group_transform = groupTx,
 9352                                        test_run=test_run,
 9353                                        perfusion_trim=perfusion_trim,
 9354                                        perfusion_m0_image=perfusion_m0_image,
 9355                                        perfusion_m0=perfusion_m0,
 9356                                        verbose=True )
 9357                                except Exception as e:
 9358                                        error_info = traceback.format_exc()
 9359                                        print(error_info)
 9360                                        visualize=False
 9361                                        dowrite=False
 9362                                        tabPro={'perf':None}
 9363                                        print(f"antspymmerror occurred while processing {overmodX}: {e}")
 9364                                        pass
 9365                                if tabPro['perf'] is not None and visualize:
 9366                                    maxslice = np.min( [21, tabPro['perf']['meanBold'].shape[2] ] )
 9367                                    ants.plot( tabPro['perf']['perfusion'],
 9368                                        axis=2, nslices=maxslice, ncol=7, crop=True, title='perfusion image', filename=mymm+mysep+"perfusion.png" )
 9369                                    ants.plot( tabPro['perf']['cbf'],
 9370                                        axis=2, nslices=maxslice, ncol=7, crop=True, title='CBF image', filename=mymm+mysep+"cbf.png" )
 9371                                    ants.plot( tabPro['perf']['m0'],
 9372                                        axis=2, nslices=maxslice, ncol=7, crop=True, title='M0 image', filename=mymm+mysep+"m0.png" )
 9373
 9374                            if ( mymod == 'pet3d' ) and ishapelen == 3:
 9375                                dowrite=True
 9376                                try:
 9377                                    tabPro, normPro = mm( t1, hier,
 9378                                        srmodel=None,
 9379                                        do_tractography=False,
 9380                                        do_kk=False,
 9381                                        do_normalization=templateTx,
 9382                                        group_template = normalization_template,
 9383                                        group_transform = groupTx,
 9384                                        test_run=test_run,
 9385                                        pet_3d_image=img,
 9386                                        verbose=True )
 9387                                except Exception as e:
 9388                                        error_info = traceback.format_exc()
 9389                                        print(error_info)
 9390                                        visualize=False
 9391                                        dowrite=False
 9392                                        tabPro={'pet3d':None}
 9393                                        print(f"antspymmerror occurred while processing {overmodX}: {e}")
 9394                                        pass
 9395                                if tabPro['pet3d'] is not None and visualize:
 9396                                    maxslice = np.min( [21, tabPro['pet3d']['pet3d'].shape[2] ] )
 9397                                    ants.plot( tabPro['pet3d']['pet3d'],
 9398                                        axis=2, nslices=maxslice, ncol=7, crop=True, title='PET image', filename=mymm+mysep+"pet3d.png" )
 9399                            if ( mymod in ['DTILR', 'DTIRL', 'DTI_LR', 'DTI_RL', 'DTI'] ) and ishapelen == 4:
 9400                                bvalfn = re.sub( '.nii.gz', '.bval' , myimg )
 9401                                bvecfn = re.sub( '.nii.gz', '.bvec' , myimg )
 9402                                imgList = [ img ]
 9403                                bvalfnList = [ bvalfn ]
 9404                                bvecfnList = [ bvecfn ]
 9405                                missing_dti_data=False # bval, bvec or images
 9406                                if len( myimgsr ) == 2:  # find DTI_RL
 9407                                    dtilrfn = myimgsr[myimgcount+1]
 9408                                    if exists( dtilrfn ):
 9409                                        bvalfnRL = re.sub( '.nii.gz', '.bval' , dtilrfn )
 9410                                        bvecfnRL = re.sub( '.nii.gz', '.bvec' , dtilrfn )
 9411                                        imgRL = ants.image_read( dtilrfn )
 9412                                        imgList.append( imgRL )
 9413                                        bvalfnList.append( bvalfnRL )
 9414                                        bvecfnList.append( bvecfnRL )
 9415                                elif len( myimgsr ) == 3:  # find DTI_RL
 9416                                    print("DTI trinity")
 9417                                    dtilrfn = myimgsr[myimgcount+1]
 9418                                    dtilrfn2 = myimgsr[myimgcount+2]
 9419                                    if exists( dtilrfn ) and exists( dtilrfn2 ):
 9420                                        bvalfnRL = re.sub( '.nii.gz', '.bval' , dtilrfn )
 9421                                        bvecfnRL = re.sub( '.nii.gz', '.bvec' , dtilrfn )
 9422                                        bvalfnRL2 = re.sub( '.nii.gz', '.bval' , dtilrfn2 )
 9423                                        bvecfnRL2 = re.sub( '.nii.gz', '.bvec' , dtilrfn2 )
 9424                                        imgRL = ants.image_read( dtilrfn )
 9425                                        imgRL2 = ants.image_read( dtilrfn2 )
 9426                                        bvals, bvecs = read_bvals_bvecs( bvalfnRL , bvecfnRL  )
 9427                                        print( bvals.max() )
 9428                                        bvals2, bvecs2 = read_bvals_bvecs( bvalfnRL2 , bvecfnRL2  )
 9429                                        print( bvals2.max() )
 9430                                        temp = merge_dwi_data( imgRL, bvals, bvecs, imgRL2, bvals2, bvecs2  )
 9431                                        imgList.append( temp[0] )
 9432                                        bvalfnList.append( mymm+mysep+'joined.bval' )
 9433                                        bvecfnList.append( mymm+mysep+'joined.bvec' )
 9434                                        write_bvals_bvecs( temp[1], temp[2], mymm+mysep+'joined' )
 9435                                        bvalsX, bvecsX = read_bvals_bvecs( bvalfnRL2 , bvecfnRL2  )
 9436                                        print( bvalsX.max() )
 9437                                # check existence of all files expected ...
 9438                                for dtiex in bvalfnList+bvecfnList+myimgsr:
 9439                                    if not exists(dtiex):
 9440                                        print('mm_csv: missing dti data ' + dtiex )
 9441                                        missing_dti_data=True
 9442                                        dowrite=False
 9443                                if not missing_dti_data:
 9444                                    dowrite=True
 9445                                    srmodel_DTI_mdl=None
 9446                                    if srmodel_DTI is not None:
 9447                                        temp = ants.get_spacing(img)
 9448                                        dtspc=[temp[0],temp[1],temp[2]]
 9449                                        bestup = siq.optimize_upsampling_shape( dtspc, modality='DTI' )
 9450                                        mdlfn = re.sub( 'bestup', bestup, srmodel_DTI )
 9451                                        if isinstance( srmodel_DTI, str ):
 9452                                            srmodel_DTI = re.sub( "bestup", bestup, srmodel_DTI )
 9453                                            mdlfn = os.path.join( ex_pathmm, srmodel_DTI )
 9454                                        if exists( mdlfn ):
 9455                                            if verbose:
 9456                                                print(mdlfn)
 9457                                            srmodel_DTI_mdl = tf.keras.models.load_model( mdlfn, compile=False )
 9458                                        else:
 9459                                            print(mdlfn + " does not exist - wont use SR")
 9460                                    try:
 9461                                        tabPro, normPro = mm( t1, hier,
 9462                                            dw_image=imgList,
 9463                                            bvals = bvalfnList,
 9464                                            bvecs = bvecfnList,
 9465                                            srmodel=srmodel_DTI_mdl,
 9466                                            do_tractography=not test_run,
 9467                                            do_kk=False,
 9468                                            do_normalization=templateTx,
 9469                                            group_template = normalization_template,
 9470                                            group_transform = groupTx,
 9471                                            dti_motion_correct = dti_motion_correct,
 9472                                            dti_denoise = dti_denoise,
 9473                                            test_run=test_run,
 9474                                            verbose=True )
 9475                                    except Exception as e:
 9476                                            error_info = traceback.format_exc()
 9477                                            print(error_info)
 9478                                            visualize=False
 9479                                            dowrite=False
 9480                                            tabPro={'DTI':None}
 9481                                            print(f"antspymmerror occurred while processing {overmodX}: {e}")
 9482                                            pass
 9483                                    mydti = tabPro['DTI']
 9484                                    if visualize and tabPro['DTI'] is not None:
 9485                                        maxslice = np.min( [21, mydti['recon_fa'] ] )
 9486                                        ants.plot( mydti['recon_fa'],  axis=2, nslices=maxslice, ncol=7, crop=True, title='FA', filename=mymm+mysep+"FAbetter.png"  )
 9487                                        ants.plot( mydti['recon_fa'], mydti['jhu_labels'], axis=2, nslices=maxslice, ncol=7, crop=True, title='FA + JHU', filename=mymm+mysep+"FAJHU.png"  )
 9488                                        ants.plot( mydti['recon_md'],  axis=2, nslices=maxslice, ncol=7, crop=True, title='MD', filename=mymm+mysep+"MD.png"  )
 9489                            if dowrite:
 9490                                write_mm( output_prefix=mymm, mm=tabPro, mm_norm=normPro, t1wide=t1wide, separator=mysep )
 9491                                for mykey in normPro.keys():
 9492                                    if normPro[mykey] is not None and normPro[mykey].components == 1:
 9493                                        if visualize and False:
 9494                                            ants.plot( template, normPro[mykey], axis=2, nslices=21, ncol=7, crop=True, title=mykey, filename=mymm+mysep+mykey+".png"   )
 9495        if overmodX == nrg_modality_list[ len( nrg_modality_list ) - 1 ]:
 9496            return
 9497        if verbose:
 9498            print("done with " + overmodX )
 9499    if verbose:
 9500        print("mm_nrg complete.")
 9501    return
 9502
 9503def spec_taper(x, p=0.1):
 9504    from scipy import stats, signal, fft
 9505    from statsmodels.regression.linear_model import yule_walker
 9506    """
 9507    Computes a tapered version of x, with tapering p.
 9508
 9509    Adapted from R's stats::spec.taper at https://github.com/telmo-correa/time-series-analysis/blob/master/Python/spectrum.py
 9510
 9511    """
 9512
 9513    p = np.r_[p]
 9514    assert np.all((p >= 0) & (p < 0.5)), "'p' must be between 0 and 0.5"
 9515
 9516    x = np.r_[x].astype('float64')
 9517    original_shape = x.shape
 9518
 9519    assert len(original_shape) <= 2, "'x' must have at most 2 dimensions"
 9520    while len(x.shape) < 2:
 9521        x = np.expand_dims(x, axis=1)
 9522
 9523    nr, nc = x.shape
 9524    if len(p) == 1:
 9525        p = p * np.ones(nc)
 9526    else:
 9527        assert len(p) == nc, "length of 'p' must be 1 or equal the number of columns of 'x'"
 9528
 9529    for i in range(nc):
 9530        m = int(np.floor(nr * p[i]))
 9531        if m == 0:
 9532            continue
 9533        w = 0.5 * (1 - np.cos(np.pi * np.arange(1, 2 * m, step=2)/(2 * m)))
 9534        x[:, i] = np.r_[w, np.ones(nr - 2 * m), w[::-1]] * x[:, i]
 9535
 9536    x = np.reshape(x, original_shape)
 9537    return x
 9538
 9539def plot_spec(spec_res, coverage=None, ax=None, title=None):
 9540    import matplotlib.pyplot as plt
 9541    """Convenience plotting method, also includes confidence cross in the same style as R.
 9542
 9543    Note that the location of the cross is irrelevant; only width and height matter."""
 9544    f, Pxx = spec_res['freq'], spec_res['spec']
 9545
 9546    if coverage is not None:
 9547        ci = spec_ci(spec_res['df'], coverage=coverage)
 9548        conf_x = (max(spec_res['freq']) - spec_res['bandwidth']) + np.r_[-0.5, 0.5] * spec_res['bandwidth']
 9549        conf_y = max(spec_res['spec']) / ci[1]
 9550
 9551    if ax is None:
 9552        ax = plt.gca()
 9553
 9554    ax.plot(f, Pxx, color='C0')
 9555    ax.set_xlabel('Frequency')
 9556    ax.set_ylabel('Log Spectrum')
 9557    ax.set_yscale('log')
 9558    if coverage is not None:
 9559        ax.plot(np.mean(conf_x) * np.r_[1, 1], conf_y * ci, color='red')
 9560        ax.plot(conf_x, np.mean(conf_y) * np.r_[1, 1], color='red')
 9561
 9562    ax.set_title(spec_res['method'] if title is None else title)
 9563
 9564def spec_ci(df, coverage=0.95):
 9565    from scipy import stats, signal, fft
 9566    from statsmodels.regression.linear_model import yule_walker
 9567    """
 9568    Computes the confidence interval for a spectral fit, based on the number of degrees of freedom.
 9569
 9570    Adapted from R's stats::plot.spec at https://github.com/telmo-correa/time-series-analysis/blob/master/Python/spectrum.py
 9571
 9572    """
 9573
 9574    assert coverage >= 0 and coverage < 1, "coverage probability out of range [0, 1)"
 9575
 9576    tail = 1 - coverage
 9577
 9578    phi = stats.chi2.cdf(x=df, df=df)
 9579    upper_quantile = 1 - tail * (1 - phi)
 9580    lower_quantile = tail * phi
 9581
 9582    return df / stats.chi2.ppf([upper_quantile, lower_quantile], df=df)
 9583
 9584def spec_pgram(x, xfreq=1, spans=None, kernel=None, taper=0.1, pad=0, fast=True, demean=False, detrend=True,
 9585               plot=True, **kwargs):
 9586    """
 9587    Computes the spectral density estimate using a periodogram.  Optionally, it also:
 9588    - Uses a provided kernel window, or a sequence of spans for convoluted modified Daniell kernels.
 9589    - Tapers the start and end of the series to avoid end-of-signal effects.
 9590    - Pads the provided series before computation, adding pad*(length of series) zeros at the end.
 9591    - Pads the provided series before computation to speed up FFT calculation.
 9592    - Performs demeaning or detrending on the series.
 9593    - Plots results.
 9594
 9595    Implemented to ensure compatibility with R's spectral functions, as opposed to reusing scipy's periodogram.
 9596
 9597    Adapted from R's stats::spec.pgram at https://github.com/telmo-correa/time-series-analysis/blob/master/Python/spectrum.py
 9598
 9599    example:
 9600
 9601    import numpy as np
 9602    import antspymm
 9603    myx = np.random.rand(100,1)
 9604    myspec = antspymm.spec_pgram(myx,0.5)
 9605
 9606    """
 9607    from scipy import stats, signal, fft
 9608    from statsmodels.regression.linear_model import yule_walker
 9609    def daniell_window_modified(m):
 9610        """ Single-pass modified Daniell kernel window.
 9611
 9612        Weight is normalized to add up to 1, and all values are the same, other than the first and the
 9613        last, which are divided by 2.
 9614        """
 9615        def w(k):
 9616            return np.where(np.abs(k) < m, 1 / (2*m), np.where(np.abs(k) == m, 1/(4*m), 0))
 9617
 9618        return w(np.arange(-m, m+1))
 9619
 9620    def daniell_window_convolve(v):
 9621        """ Convolved version of multiple modified Daniell kernel windows.
 9622
 9623        Parameter v should be an iterable of m values.
 9624        """
 9625
 9626        if len(v) == 0:
 9627            return np.r_[1]
 9628
 9629        if len(v) == 1:
 9630            return daniell_window_modified(v[0])
 9631
 9632        return signal.convolve(daniell_window_modified(v[0]), daniell_window_convolve(v[1:]))
 9633
 9634    # Ensure we can store non-integers in x, and that it is a numpy object
 9635    x = np.r_[x].astype('float64')
 9636    original_shape = x.shape
 9637
 9638    # Ensure correct dimensions
 9639    assert len(original_shape) <= 2, "'x' must have at most 2 dimensions"
 9640    while len(x.shape) < 2:
 9641        x = np.expand_dims(x, axis=1)
 9642
 9643    N, nser = x.shape
 9644    N0 = N
 9645
 9646    # Ensure only one of spans, kernel is provided, and build the kernel window if needed
 9647    assert (spans is None) or (kernel is None), "must specify only one of 'spans' or 'kernel'"
 9648    if spans is not None:
 9649        kernel = daniell_window_convolve(np.floor_divide(np.r_[spans], 2))
 9650
 9651    # Detrend or demean the series
 9652    if detrend:
 9653        t = np.arange(N) - (N - 1)/2
 9654        sumt2 = N * (N**2 - 1)/12
 9655        x -= (np.repeat(np.expand_dims(np.mean(x, axis=0), 0), N, axis=0) + np.outer(np.sum(x.T * t, axis=1), t/sumt2).T)
 9656    elif demean:
 9657        x -= np.mean(x, axis=0)
 9658
 9659    # Compute taper and taper adjustment variables
 9660    x = spec_taper(x, taper)
 9661    u2 = (1 - (5/8) * taper * 2)
 9662    u4 = (1 - (93/128) * taper * 2)
 9663
 9664    # Pad the series with copies of the same shape, but filled with zeroes
 9665    if pad > 0:
 9666        x = np.r_[x, np.zeros((pad * x.shape[0], x.shape[1]))]
 9667        N = x.shape[0]
 9668
 9669    # Further pad the series to accelerate FFT computation
 9670    if fast:
 9671        newN = fft.next_fast_len(N, True)
 9672        x = np.r_[x, np.zeros((newN - N, x.shape[1]))]
 9673        N = newN
 9674
 9675    # Compute the Fourier frequencies (R's spec.pgram convention style)
 9676    Nspec = int(np.floor(N/2))
 9677    freq = (np.arange(Nspec) + 1) * xfreq / N
 9678
 9679    # Translations to keep same row / column convention as stats::mvfft
 9680    xfft = fft.fft(x.T).T
 9681
 9682    # Compute the periodogram for each i, j
 9683    pgram = np.empty((N, nser, nser), dtype='complex')
 9684    for i in range(nser):
 9685        for j in range(nser):
 9686            pgram[:, i, j] = xfft[:, i] * np.conj(xfft[:, j]) / (N0 * xfreq)
 9687            pgram[0, i, j] = 0.5 * (pgram[1, i, j] + pgram[-1, i, j])
 9688
 9689    if kernel is None:
 9690        # Values pre-adjustment
 9691        df = 2
 9692        bandwidth = np.sqrt(1 / 12)
 9693    else:
 9694        def conv_circular(signal, kernel):
 9695            """
 9696            Performs 1D circular convolution, in the same style as R::kernapply,
 9697            assuming the kernel window is centered at 0.
 9698            """
 9699            pad = len(signal) - len(kernel)
 9700            half_window = int((len(kernel) + 1) / 2)
 9701            indexes = range(-half_window, len(signal) - half_window)
 9702            orig_conv = np.real(fft.ifft(fft.fft(signal) * fft.fft(np.r_[np.zeros(pad), kernel])))
 9703            return orig_conv.take(indexes, mode='wrap')
 9704
 9705        # Convolve pgram with kernel with circular conv
 9706        for i in range(nser):
 9707            for j in range(nser):
 9708                pgram[:, i, j] = conv_circular(pgram[:, i, j], kernel)
 9709
 9710        df = 2 / np.sum(kernel**2)
 9711        m = (len(kernel) - 1)/2
 9712        k = np.arange(-m, m+1)
 9713        bandwidth = np.sqrt(np.sum((1/12 + k**2) * kernel))
 9714
 9715    df = df/(u4/u2**2)*(N0/N)
 9716    bandwidth = bandwidth * xfreq/N
 9717
 9718    # Remove padded results
 9719    pgram = pgram[1:(Nspec+1), :, :]
 9720
 9721    spec = np.empty((Nspec, nser))
 9722    for i in range(nser):
 9723        spec[:, i] = np.real(pgram[:, i, i])
 9724
 9725    if nser == 1:
 9726        coh = None
 9727        phase = None
 9728    else:
 9729        coh = np.empty((Nspec, int(nser * (nser - 1)/2)))
 9730        phase = np.empty((Nspec, int(nser * (nser - 1)/2)))
 9731        for i in range(nser):
 9732            for j in range(i+1, nser):
 9733                index = int(i + j*(j-1)/2)
 9734                coh[:, index] = np.abs(pgram[:, i, j])**2 / (spec[:, i] * spec[:, j])
 9735                phase[:, index] = np.angle(pgram[:, i, j])
 9736
 9737    spec = spec / u2
 9738    spec = spec.squeeze()
 9739
 9740    results = {
 9741        'freq': freq,
 9742        'spec': spec,
 9743        'coh': coh,
 9744        'phase': phase,
 9745        'kernel': kernel,
 9746        'df': df,
 9747        'bandwidth': bandwidth,
 9748        'n.used': N,
 9749        'orig.n': N0,
 9750        'taper': taper,
 9751        'pad': pad,
 9752        'detrend': detrend,
 9753        'demean': demean,
 9754        'method': 'Raw Periodogram' if kernel is None else 'Smoothed Periodogram'
 9755    }
 9756
 9757    if plot:
 9758        plot_spec(results, coverage=0.95, **kwargs)
 9759
 9760    return results
 9761
 9762def alffmap( x, flo=0.01, fhi=0.1, tr=1, detrend = True ):
 9763    """
 9764    Amplitude of Low Frequency Fluctuations (ALFF; Zang et al., 2007) and
 9765    fractional Amplitude of Low Frequency Fluctuations (f/ALFF; Zou et al., 2008)
 9766    are related measures that quantify the amplitude of low frequency
 9767    oscillations (LFOs).  This function outputs ALFF and fALFF for the input.
 9768    same function in ANTsR.
 9769
 9770    x input vector for the time series of interest
 9771    flo low frequency, typically 0.01
 9772    fhi high frequency, typically 0.1
 9773    tr the period associated with the vector x (inverse of frequency)
 9774    detrend detrend the input time series
 9775
 9776    return vector is output showing ALFF and fALFF values
 9777    """
 9778    temp = spec_pgram( x, xfreq=1.0/tr, demean=False, detrend=detrend, taper=0, fast=True, plot=False )
 9779    fselect = np.logical_and( temp['freq'] >= flo, temp['freq'] <= fhi )
 9780    denom = (temp['spec']).sum()
 9781    numer = (temp['spec'][fselect]).sum()
 9782    return {  'alff':numer, 'falff': numer/denom }
 9783
 9784
 9785def alff_image( x, mask, flo=0.01, fhi=0.1, nuisance=None ):
 9786    """
 9787    Amplitude of Low Frequency Fluctuations (ALFF; Zang et al., 2007) and
 9788    fractional Amplitude of Low Frequency Fluctuations (f/ALFF; Zou et al., 2008)
 9789    are related measures that quantify the amplitude of low frequency
 9790    oscillations (LFOs).  This function outputs ALFF and fALFF for the input.
 9791
 9792    x - input clean resting state fmri
 9793    mask - mask over which to compute f/alff
 9794    flo - low frequency, typically 0.01
 9795    fhi - high frequency, typically 0.1
 9796    nuisance - optional nuisance matrix
 9797
 9798    return dictionary with ALFF and fALFF images
 9799    """
 9800    xmat = ants.timeseries_to_matrix( x, mask )
 9801    if nuisance is not None:
 9802        xmat = ants.regress_components( xmat, nuisance )
 9803    alffvec = xmat[0,:]*0
 9804    falffvec = xmat[0,:]*0
 9805    mytr = ants.get_spacing( x )[3]
 9806    for n in range( xmat.shape[1] ):
 9807        temp = alffmap( xmat[:,n], flo=flo, fhi=fhi, tr=mytr )
 9808        alffvec[n]=temp['alff']
 9809        falffvec[n]=temp['falff']
 9810    alffi=ants.make_image( mask, alffvec )
 9811    falffi=ants.make_image( mask, falffvec )
 9812    alfftrimmedmean = calculate_trimmed_mean( alffvec, 0.01 )
 9813    falfftrimmedmean = calculate_trimmed_mean( falffvec, 0.01 )
 9814    alffi=alffi / alfftrimmedmean
 9815    falffi=falffi / falfftrimmedmean
 9816    return {  'alff': alffi, 'falff': falffi }
 9817
 9818
 9819def down2iso( x, interpolation='linear', takemin=False ):
 9820    """
 9821    will downsample an anisotropic image to an isotropic resolution
 9822
 9823    x: input image
 9824
 9825    interpolation: linear or nearestneighbor
 9826
 9827    takemin : boolean map to min space; otherwise max
 9828
 9829    return image downsampled to isotropic resolution
 9830    """
 9831    spc = ants.get_spacing( x )
 9832    if takemin:
 9833        newspc = np.asarray(spc).min()
 9834    else:
 9835        newspc = np.asarray(spc).max()
 9836    newspc = np.repeat( newspc, x.dimension )
 9837    if interpolation == 'linear':
 9838        xs = ants.resample_image( x, newspc, interp_type=0)
 9839    else:
 9840        xs = ants.resample_image( x, newspc, interp_type=1)
 9841    return xs
 9842
 9843
 9844def read_mm_csv( x, is_t1=False, colprefix=None, separator='-', verbose=False ):
 9845    splitter=os.path.basename(x).split( separator )
 9846    lensplit = len( splitter )-1
 9847    temp = os.path.basename(x)
 9848    temp = os.path.splitext(temp)[0]
 9849    temp = re.sub(separator+'mmwide','',temp)
 9850    idcols = ['u_hier_id','sid','visitdate','modality','mmimageuid','t1imageuid']
 9851    df = pd.DataFrame( columns = idcols, index=range(1) )
 9852    valstoadd = [temp] + splitter[1:(lensplit-1)]
 9853    if is_t1:
 9854        valstoadd = valstoadd + [splitter[(lensplit-1)],splitter[(lensplit-1)]]
 9855    else:
 9856        split2=splitter[(lensplit-1)].split( "_" )
 9857        if len(split2) == 1:
 9858            split2.append( split2[0] )
 9859        if len(valstoadd) == 3:
 9860            valstoadd = valstoadd + [split2[0]] + [math.nan] + [split2[1]]
 9861        else:
 9862            valstoadd = valstoadd + [split2[0],split2[1]]
 9863    if verbose:
 9864        print( valstoadd )
 9865    df.iloc[0] = valstoadd
 9866    if verbose:
 9867        print( "read xdf: " + x )
 9868    xdf = pd.read_csv( x )
 9869    df.reset_index()
 9870    xdf.reset_index(drop=True)
 9871    if "Unnamed: 0" in xdf.columns:
 9872        holder=xdf.pop( "Unnamed: 0" )
 9873    if "Unnamed: 1" in xdf.columns:
 9874        holder=xdf.pop( "Unnamed: 1" )
 9875    if "u_hier_id.1" in xdf.columns:
 9876        holder=xdf.pop( "u_hier_id.1" )
 9877    if "u_hier_id" in xdf.columns:
 9878        holder=xdf.pop( "u_hier_id" )
 9879    if not is_t1:
 9880        if 'resnetGrade' in xdf.columns:
 9881            index_no = xdf.columns.get_loc('resnetGrade')
 9882            xdf = xdf.drop( xdf.columns[range(index_no+1)] , axis=1)
 9883
 9884    if xdf.shape[0] == 2:
 9885        xdfcols = xdf.columns
 9886        xdf = xdf.iloc[1]
 9887        ddnum = xdf.to_numpy()
 9888        ddnum = ddnum.reshape([1,ddnum.shape[0]])
 9889        newcolnames = xdf.index.to_list()
 9890        if len(newcolnames) != ddnum.shape[1]:
 9891            print("Cannot Merge : Shape MisMatch " + str( len(newcolnames) ) + " " + str(ddnum.shape[1]))
 9892        else:
 9893            xdf = pd.DataFrame(ddnum, columns=xdfcols )
 9894    if xdf.shape[1] == 0:
 9895        return None
 9896    if colprefix is not None:
 9897        xdf.columns=colprefix + xdf.columns
 9898    return pd.concat( [df,xdf], axis=1, ignore_index=False )
 9899
 9900def merge_wides_to_study_dataframe( sdf, processing_dir, separator='-', sid_is_int=True, id_is_int=True, date_is_int=True, report_missing=False,
 9901progress=False, verbose=False ):
 9902    """
 9903    extend a study data frame with wide outputs
 9904
 9905    sdf : the input study dataframe from antspymm QC output
 9906
 9907    processing_dir:  the directory location of the processed data 
 9908
 9909    separator : string usually '-' or '_'
 9910
 9911    sid_is_int : boolean set to True to cast unique subject ids to int; can be useful if they are inadvertently stored as float by pandas
 9912
 9913    date_is_int : boolean set to True to cast date to int; can be useful if they are inadvertently stored as float by pandas
 9914
 9915    id_is_int : boolean set to True to cast unique image ids to int; can be useful if they are inadvertently stored as float by pandas
 9916
 9917    report_missing : boolean combined with verbose will report missing modalities
 9918
 9919    progress : integer reports percent progress modulo progress value 
 9920
 9921    verbose : boolean
 9922    """
 9923    from os.path import exists
 9924    musthavecols = ['projectID', 'subjectID','date','imageID']
 9925    for k in range(len(musthavecols)):
 9926        if not musthavecols[k] in sdf.keys():
 9927            raise ValueError('sdf is missing column ' +musthavecols[k] + ' in merge_wides_to_study_dataframe' )
 9928    possible_iids = [ 'imageID', 'imageID', 'imageID', 'flairid', 'dtid1', 'dtid2', 'rsfid1', 'rsfid2', 'nmid1', 'nmid2', 'nmid3', 'nmid4', 'nmid5', 'nmid6', 'nmid7', 'nmid8', 'nmid9', 'nmid10', 'perfid' ]
 9929    modality_ids = [ 'T1wHierarchical', 'T1wHierarchicalSR', 'T1w', 'T2Flair', 'DTI', 'DTI', 'rsfMRI', 'rsfMRI', 'NM2DMT', 'NM2DMT', 'NM2DMT', 'NM2DMT', 'NM2DMT', 'NM2DMT', 'NM2DMT', 'NM2DMT', 'NM2DMT', 'NM2DMT', 'perf']
 9930    alldf=pd.DataFrame()
 9931    for myk in sdf.index:
 9932        if progress > 0 and int(myk) % int(progress) == 0:
 9933            print( str( round( myk/sdf.shape[0]*100.0)) + "%...", end='', flush=True)
 9934        if verbose:
 9935            print( "DOROW " + str(myk) + ' of ' + str( sdf.shape[0] ) )
 9936        csvrow = sdf.loc[sdf.index == myk].dropna(axis=1)
 9937        ct=-1
 9938        for iidkey in possible_iids:
 9939            ct=ct+1
 9940            mod_name = modality_ids[ct]
 9941            if iidkey in csvrow.keys():
 9942                if id_is_int:
 9943                    iid = str( int( csvrow[iidkey].iloc[0] ) )
 9944                else:
 9945                    iid = str( csvrow[iidkey].iloc[0] )
 9946                if verbose:
 9947                    print( "iidkey " + iidkey + " modality " + mod_name + ' iid '+ iid )
 9948                pid=str(csvrow['projectID'].iloc[0] )
 9949                if sid_is_int:
 9950                    sid=str(int(csvrow['subjectID'].iloc[0] ))
 9951                else:
 9952                    sid=str(csvrow['subjectID'].iloc[0] )
 9953                if date_is_int:
 9954                    dt=str(int(csvrow['date'].iloc[0]))
 9955                else:
 9956                    dt=str(csvrow['date'].iloc[0])
 9957                if id_is_int:
 9958                    t1iid=str(int(csvrow['imageID'].iloc[0]))
 9959                else:
 9960                    t1iid=str(csvrow['imageID'].iloc[0])
 9961                if t1iid != iid:
 9962                    iidj=iid+"_"+t1iid
 9963                else:
 9964                    iidj=iid
 9965                rootid = pid +separator+ sid +separator+dt+separator+mod_name+separator+iidj
 9966                myext = rootid +separator+'mmwide.csv'
 9967                nrgwidefn=os.path.join( processing_dir, pid, sid, dt, mod_name, iid, myext )
 9968                moddersub = mod_name
 9969                is_t1=False
 9970                if mod_name == 'T1wHierarchical':
 9971                    is_t1=True
 9972                    moddersub='T1Hier'
 9973                elif mod_name == 'T1wHierarchicalSR':
 9974                    is_t1=True
 9975                    moddersub='T1HSR'
 9976                if exists( nrgwidefn ):
 9977                    if verbose:
 9978                        print( nrgwidefn + " exists")
 9979                    mm=read_mm_csv( nrgwidefn, colprefix=moddersub+'_', is_t1=is_t1, separator=separator, verbose=verbose )
 9980                    if mm is not None:
 9981                        if mod_name == 'T1wHierarchical':
 9982                            a=list( csvrow.keys() )
 9983                            b=list( mm.keys() )
 9984                            abintersect=list(set(b).intersection( set(a) ) )
 9985                            if len( abintersect  ) > 0 :
 9986                                for qq in abintersect:
 9987                                    mm.pop( qq )
 9988                        # mm.index=csvrow.index
 9989                        uidname = mod_name + '_mmwide_filename'
 9990                        mm[ uidname ] = rootid
 9991                        csvrow=pd.concat( [csvrow,mm], axis=1, ignore_index=False )
 9992                else:
 9993                    if verbose and report_missing:
 9994                        print( nrgwidefn + " absent")
 9995        if alldf.shape[0] == 0:
 9996            alldf = csvrow.copy()
 9997            alldf = alldf.loc[:,~alldf.columns.duplicated()]
 9998        else:
 9999            csvrow=csvrow.loc[:,~csvrow.columns.duplicated()]
10000            alldf = alldf.loc[:,~alldf.columns.duplicated()]
10001            alldf = pd.concat( [alldf, csvrow], axis=0, ignore_index=True )
10002    return alldf
10003
10004def assemble_modality_specific_dataframes( mm_wide_csvs, hierdfin, nrg_modality, separator='-', progress=None, verbose=False ):
10005    moddersub = re.sub( "[*]","",nrg_modality)
10006    nmdf=pd.DataFrame()
10007    for k in range( hierdfin.shape[0] ):
10008        if progress is not None:
10009            if k % progress == 0:
10010                progger = str( np.round( k / hierdfin.shape[0] * 100 ) )
10011                print( progger, end ="...", flush=True)
10012        temp = mm_wide_csvs[k]
10013        mypartsf = temp.split("T1wHierarchical")
10014        myparts = mypartsf[0]
10015        t1iid = str(mypartsf[1].split("/")[1])
10016        fnsnm = glob.glob(myparts+"/" + nrg_modality + "/*/*" + t1iid + "*wide.csv")
10017        if len( fnsnm ) > 0 :
10018            for y in fnsnm:
10019                temp=read_mm_csv( y, colprefix=moddersub+'_', is_t1=False, separator=separator, verbose=verbose )
10020                if temp is not None:
10021                    nmdf=pd.concat( [nmdf, temp], axis=0, ignore_index=False )
10022    return nmdf
10023
10024def bind_wide_mm_csvs( mm_wide_csvs, merge=True, separator='-', verbose = 0 ) :
10025    """
10026    will convert a list of t1w hierarchical csv filenames to a merged dataframe
10027
10028    returns a pair of data frames, the left side having all entries and the
10029        right side having row averaged entries i.e. unique values for each visit
10030
10031    set merge to False to return individual dataframes ( for debugging )
10032
10033    return alldata, row_averaged_data
10034    """
10035    mm_wide_csvs.sort()
10036    if not mm_wide_csvs:
10037        print("No files found with specified pattern")
10038        return
10039    # 1. row-bind the t1whier data
10040    # 2. same for each other modality
10041    # 3. merge the modalities by the keys
10042    hierdf = pd.DataFrame()
10043    for y in mm_wide_csvs:
10044        temp=read_mm_csv( y, colprefix='T1Hier_', separator=separator, is_t1=True )
10045        if temp is not None:
10046            hierdf=pd.concat( [hierdf, temp], axis=0, ignore_index=False )
10047    if verbose > 0:
10048        mypro=50
10049    else:
10050        mypro=None
10051    if verbose > 0:
10052        print("thickness")
10053    thkdf = assemble_modality_specific_dataframes( mm_wide_csvs, hierdf, 'T1w', progress=mypro, verbose=verbose==2)
10054    if verbose > 0:
10055        print("flair")
10056    flairdf = assemble_modality_specific_dataframes( mm_wide_csvs, hierdf, 'T2Flair', progress=mypro, verbose=verbose==2)
10057    if verbose > 0:
10058        print("NM")
10059    nmdf = assemble_modality_specific_dataframes( mm_wide_csvs, hierdf, 'NM2DMT', progress=mypro, verbose=verbose==2)
10060    if verbose > 0:
10061        print("rsf")
10062    rsfdf = assemble_modality_specific_dataframes( mm_wide_csvs, hierdf, 'rsfMRI*', progress=mypro, verbose=verbose==2)
10063    if verbose > 0:
10064        print("dti")
10065    dtidf = assemble_modality_specific_dataframes( mm_wide_csvs, hierdf, 'DTI*', progress=mypro, verbose=verbose==2 )
10066    if not merge:
10067        return hierdf, thkdf, flairdf, nmdf, rsfdf, dtidf
10068    hierdfmix = hierdf.copy()
10069    modality_df_suffixes = [
10070        (thkdf, "_thk"),
10071        (flairdf, "_flair"),
10072        (nmdf, "_nm"),
10073        (rsfdf, "_rsf"),
10074        (dtidf, "_dti"),
10075    ]
10076    for pair in modality_df_suffixes:
10077        hierdfmix = merge_mm_dataframe(hierdfmix, pair[0], pair[1])
10078    hierdfmix = hierdfmix.replace(r'^\s*$', np.nan, regex=True)
10079    return hierdfmix, hierdfmix.groupby("u_hier_id", as_index=False).mean(numeric_only=True)
10080
10081def merge_mm_dataframe(hierdf, mmdf, mm_suffix):
10082    try:
10083        hierdf = hierdf.merge(mmdf, on=['sid', 'visitdate', 't1imageuid'], suffixes=("",mm_suffix),how='left')
10084        return hierdf
10085    except KeyError:
10086        return hierdf
10087
10088def augment_image( x,  max_rot=10, nzsd=1 ):
10089    rRotGenerator = ants.contrib.RandomRotate3D( ( max_rot*(-1.0), max_rot ), reference=x )
10090    tx = rRotGenerator.transform()
10091    itx = ants.invert_ants_transform(tx)
10092    y = ants.apply_ants_transform_to_image( tx, x, x, interpolation='linear')
10093    y = ants.add_noise_to_image( y,'additivegaussian', [0,nzsd] )
10094    return y, tx, itx
10095
10096def boot_wmh( flair, t1, t1seg, mmfromconvexhull = 0.0, strict=True,
10097        probability_mask=None, prior_probability=None, n_simulations=8,
10098        random_seed = 42,
10099        verbose=False ) :
10100    import random
10101    random.seed( random_seed )
10102    if verbose and prior_probability is None:
10103        print("augmented flair")
10104    if verbose and prior_probability is not None:
10105        print("augmented flair with prior")
10106    wmh_sum_aug = 0
10107    wmh_sum_prior_aug = 0
10108    augprob = flair * 0.0
10109    augprob_prior = None
10110    if prior_probability is not None:
10111        augprob_prior = flair * 0.0
10112    for n in range(n_simulations):
10113        augflair, tx, itx = augment_image( ants.iMath(flair,"Normalize"), 5, 0.01 )
10114        locwmh = wmh( augflair, t1, t1seg, mmfromconvexhull = mmfromconvexhull,
10115            strict=strict, probability_mask=None, prior_probability=prior_probability )
10116        if verbose:
10117            print( "flair sim: " + str(n) + " vol: " + str( locwmh['wmh_mass'] )+ " vol-prior: " + str( locwmh['wmh_mass_prior'] )+ " snr: " + str( locwmh['wmh_SNR'] ) )
10118        wmh_sum_aug = wmh_sum_aug + locwmh['wmh_mass']
10119        wmh_sum_prior_aug = wmh_sum_prior_aug + locwmh['wmh_mass_prior']
10120        temp = locwmh['WMH_probability_map']
10121        augprob = augprob + ants.apply_ants_transform_to_image( itx, temp, flair, interpolation='linear')
10122        if prior_probability is not None:
10123            temp = locwmh['WMH_posterior_probability_map']
10124            augprob_prior = augprob_prior + ants.apply_ants_transform_to_image( itx, temp, flair, interpolation='linear')
10125    augprob = augprob * (1.0/float( n_simulations ))
10126    if prior_probability is not None:
10127        augprob_prior = augprob_prior * (1.0/float( n_simulations ))
10128    wmh_sum_aug = wmh_sum_aug / float( n_simulations )
10129    wmh_sum_prior_aug = wmh_sum_prior_aug / float( n_simulations )
10130    return{
10131      'flair' : ants.iMath(flair,"Normalize"),
10132      'WMH_probability_map' : augprob,
10133      'WMH_posterior_probability_map' : augprob_prior,
10134      'wmh_mass': wmh_sum_aug,
10135      'wmh_mass_prior': wmh_sum_prior_aug,
10136      'wmh_evr': locwmh['wmh_evr'],
10137      'wmh_SNR': locwmh['wmh_SNR']  }
10138
10139
10140def threaded_bind_wide_mm_csvs( mm_wide_csvs, n_workers ):
10141    from concurrent.futures import as_completed
10142    from concurrent import futures
10143    import concurrent.futures
10144    def chunks(l, n):
10145        """Yield n number of sequential chunks from l."""
10146        d, r = divmod(len(l), n)
10147        for i in range(n):
10148            si = (d+1)*(i if i < r else r) + d*(0 if i < r else i - r)
10149            yield l[si:si+(d+1 if i < r else d)]
10150    import numpy as np
10151    newx = list( chunks( mm_wide_csvs, n_workers ) )
10152    import pandas as pd
10153    alldf = pd.DataFrame()
10154    alldfavg = pd.DataFrame()
10155    with futures.ThreadPoolExecutor(max_workers=n_workers) as executor:
10156        to_do = []
10157        for group in range(len(newx)) :
10158            future = executor.submit(bind_wide_mm_csvs, newx[group] )
10159            to_do.append(future)
10160        results = []
10161        for future in futures.as_completed(to_do):
10162            res0, res1 = future.result()
10163            alldf=pd.concat(  [alldf, res0 ], axis=0, ignore_index=False )
10164            alldfavg=pd.concat(  [alldfavg, res1 ], axis=0, ignore_index=False )
10165    return alldf, alldfavg
10166
10167
10168def get_names_from_data_frame(x, demogIn, exclusions=None):
10169    """
10170    data = {'Name':['Tom', 'nick', 'krish', 'jack'], 'Age':[20, 21, 19, 18]}
10171    antspymm.get_names_from_data_frame( ['e'], df )
10172    antspymm.get_names_from_data_frame( ['a','e'], df )
10173    antspymm.get_names_from_data_frame( ['e'], df, exclusions='N' )
10174    """
10175    # Check if x is a string and convert it to a list
10176    if isinstance(x, str):
10177        x = [x]
10178    def get_unique( qq ):
10179        unique = []
10180        for number in qq:
10181            if number in unique:
10182                continue
10183            else:
10184                unique.append(number)
10185        return unique
10186    outnames = list(demogIn.columns[demogIn.columns.str.contains(x[0])])
10187    if len(x) > 1:
10188        for y in x[1:]:
10189            outnames = [i for i in outnames if y in i]
10190    outnames = get_unique( outnames )
10191    if exclusions is not None:
10192        toexclude = [name for name in outnames if exclusions[0] in name ]
10193        if len(exclusions) > 1:
10194            for zz in exclusions[1:]:
10195                toexclude.extend([name for name in outnames if zz in name ])
10196        if len(toexclude) > 0:
10197            outnames = [name for name in outnames if name not in toexclude]
10198    return outnames
10199
10200
10201def average_mm_df( jmm_in, diagnostic_n=25, corr_thresh=0.9, verbose=False ):
10202    """
10203    jmrowavg, jmmcolavg, diagnostics = antspymm.average_mm_df( jmm_in, verbose=True )
10204    """
10205
10206    jmm = jmm_in.copy()
10207    dxcols=['subjectid1','subjectid2','modalityid','joinid','correlation','distance']
10208    joinDiagnostics = pd.DataFrame( columns = dxcols )
10209    nanList=[math.nan]
10210    def rob(x, y=0.99):
10211        x[x > np.quantile(x, y, nan_policy="omit")] = np.nan
10212        return x
10213
10214    jmm = jmm.replace(r'^\s*$', np.nan, regex=True)
10215
10216    if verbose:
10217        print("do rsfMRI")
10218    # here - we first have to average within each row
10219    dt0 = get_names_from_data_frame(["rsfMRI"], jmm, exclusions=["Unnamed", "rsfMRI_LR", "rsfMRI_RL"])
10220    dt1 = get_names_from_data_frame(["rsfMRI_RL"], jmm, exclusions=["Unnamed"])
10221    if len( dt0 ) > 0 and len( dt1 ) > 0:
10222        flid = dt0[0]
10223        wrows = []
10224        for i in range(jmm.shape[0]):
10225            if not pd.isna(jmm[dt0[1]][i]) or not pd.isna(jmm[dt1[1]][i]) :
10226                wrows.append(i)
10227        for k in wrows:
10228            v1 = jmm.iloc[k][dt0[1:]].astype(float)
10229            v2 = jmm.iloc[k][dt1[1:]].astype(float)
10230            vvec = [v1[0], v2[0]]
10231            if any(~np.isnan(vvec)):
10232                mynna = [i for i, x in enumerate(vvec) if ~np.isnan(x)]
10233                jmm.iloc[k][dt0[0]] = 'rsfMRI'
10234                if len(mynna) == 1:
10235                    if mynna[0] == 0:
10236                        jmm.iloc[k][dt0[1:]] = v1
10237                    if mynna[0] == 1:
10238                        jmm.iloc[k][dt0[1:]] = v2
10239                elif len(mynna) > 1:
10240                    if len(v2) > diagnostic_n:
10241                        v1dx=v1[0:diagnostic_n]
10242                        v2dx=v2[0:diagnostic_n]
10243                    else :
10244                        v1dx=v1
10245                        v2dx=v2
10246                    joinDiagnosticsLoc = pd.DataFrame( columns = dxcols, index=range(1) )
10247                    mycorr = np.corrcoef( v1dx.values, v2dx.values )[0,1]
10248                    myerr=np.sqrt(np.mean((v1dx.values - v2dx.values)**2))
10249                    joinDiagnosticsLoc.iloc[0] = [jmm.loc[k,'u_hier_id'],math.nan,'rsfMRI','colavg',mycorr,myerr]
10250                    if mycorr > corr_thresh:
10251                        jmm.loc[k, dt0[1:]] = v1.values*0.5 + v2.values*0.5
10252                    else:
10253                        jmm.loc[k, dt0[1:]] = nanList * len(v1)
10254                    if verbose:
10255                        print( joinDiagnosticsLoc )
10256                    joinDiagnostics = pd.concat( [joinDiagnostics, joinDiagnosticsLoc], axis=0, ignore_index=False )
10257
10258    if verbose:
10259        print("do DTI")
10260    # here - we first have to average within each row
10261    dt0 = get_names_from_data_frame(["DTI"], jmm, exclusions=["Unnamed", "DTI_LR", "DTI_RL"])
10262    dt1 = get_names_from_data_frame(["DTI_LR"], jmm, exclusions=["Unnamed"])
10263    dt2 = get_names_from_data_frame( ["DTI_RL"], jmm, exclusions=["Unnamed"])
10264    flid = dt0[0]
10265    wrows = []
10266    for i in range(jmm.shape[0]):
10267        if not pd.isna(jmm[dt0[1]][i]) or not pd.isna(jmm[dt1[1]][i]) or not pd.isna(jmm[dt2[1]][i]):
10268            wrows.append(i)
10269    for k in wrows:
10270        v1 = jmm.loc[k, dt0[1:]].astype(float)
10271        v2 = jmm.loc[k, dt1[1:]].astype(float)
10272        v3 = jmm.loc[k, dt2[1:]].astype(float)
10273        checkcol = dt0[5]
10274        if not np.isnan(v1[checkcol]):
10275            if v1[checkcol] < 0.25:
10276                v1.replace(np.nan, inplace=True)
10277        checkcol = dt1[5]
10278        if not np.isnan(v2[checkcol]):
10279            if v2[checkcol] < 0.25:
10280                v2.replace(np.nan, inplace=True)
10281        checkcol = dt2[5]
10282        if not np.isnan(v3[checkcol]):
10283            if v3[checkcol] < 0.25:
10284                v3.replace(np.nan, inplace=True)
10285        vvec = [v1[0], v2[0], v3[0]]
10286        if any(~np.isnan(vvec)):
10287            mynna = [i for i, x in enumerate(vvec) if ~np.isnan(x)]
10288            jmm.loc[k, dt0[0]] = 'DTI'
10289            if len(mynna) == 1:
10290                if mynna[0] == 0:
10291                    jmm.loc[k, dt0[1:]] = v1
10292                if mynna[0] == 1:
10293                    jmm.loc[k, dt0[1:]] = v2
10294                if mynna[0] == 2:
10295                    jmm.loc[k, dt0[1:]] = v3
10296            elif len(mynna) > 1:
10297                if mynna[0] == 0:
10298                    jmm.loc[k, dt0[1:]] = v1
10299                else:
10300                    joinDiagnosticsLoc = pd.DataFrame( columns = dxcols, index=range(1) )
10301                    mycorr = np.corrcoef( v2[0:diagnostic_n].values, v3[0:diagnostic_n].values )[0,1]
10302                    myerr=np.sqrt(np.mean((v2[0:diagnostic_n].values - v3[0:diagnostic_n].values)**2))
10303                    joinDiagnosticsLoc.iloc[0] = [jmm.loc[k,'u_hier_id'],math.nan,'DTI','colavg',mycorr,myerr]
10304                    if mycorr > corr_thresh:
10305                        jmm.loc[k, dt0[1:]] = v2.values*0.5 + v3.values*0.5
10306                    else: #
10307                        jmm.loc[k, dt0[1:]] = nanList * len( dt0[1:] )
10308                    if verbose:
10309                        print( joinDiagnosticsLoc )
10310                    joinDiagnostics = pd.concat( [joinDiagnostics, joinDiagnosticsLoc], axis=0, ignore_index=False )
10311
10312
10313    # first task - sort by u_hier_id
10314    jmm = jmm.sort_values( "u_hier_id" )
10315    # get rid of junk columns
10316    badnames = get_names_from_data_frame( ['Unnamed'], jmm )
10317    jmm=jmm.drop(badnames, axis=1)
10318    jmm=jmm.set_index("u_hier_id",drop=False)
10319    # 2nd - get rid of duplicated u_hier_id
10320    jmmUniq = jmm.drop_duplicates( subset="u_hier_id" ) # fast and easy
10321    # for each modality, count which ids have more than one
10322    mod_names = get_valid_modalities()
10323    for mod_name in mod_names:
10324        fl_names = get_names_from_data_frame([mod_name], jmm,
10325            exclusions=['Unnamed',"DTI_LR","DTI_RL","rsfMRI_RL","rsfMRI_LR"])
10326        if len( fl_names ) > 1:
10327            if verbose:
10328                print(mod_name)
10329                print(fl_names)
10330            fl_id = fl_names[0]
10331            n_names = len(fl_names)
10332            locvec = jmm[fl_names[n_names-1]].astype(float)
10333            boolvec=~pd.isna(locvec)
10334            jmmsub = jmm[boolvec][ ['u_hier_id']+fl_names]
10335            my_tbl = Counter(jmmsub['u_hier_id'])
10336            gtoavg = [name for name in my_tbl.keys() if my_tbl[name] == 1]
10337            gtoavgG1 = [name for name in my_tbl.keys() if my_tbl[name] > 1]
10338            if verbose:
10339                print("Join 1")
10340            jmmsub1 = jmmsub.loc[jmmsub['u_hier_id'].isin(gtoavg)][['u_hier_id']+fl_names]
10341            for u in gtoavg:
10342                jmmUniq.loc[u][fl_names[1:]] = jmmsub1.loc[u][fl_names[1:]]
10343            if verbose and len(gtoavgG1) > 1:
10344                print("Join >1")
10345            jmmsubG1 = jmmsub.loc[jmmsub['u_hier_id'].isin(gtoavgG1)][['u_hier_id']+fl_names]
10346            for u in gtoavgG1:
10347                temp = jmmsubG1.loc[u][ ['u_hier_id']+fl_names ]
10348                dropnames = get_names_from_data_frame( ['MM.ID'], temp )
10349                tempVec = temp.drop(columns=dropnames)
10350                joinDiagnosticsLoc = pd.DataFrame( columns = dxcols, index=range(1) )
10351                id1=temp[fl_id].iloc[0]
10352                id2=temp[fl_id].iloc[1]
10353                v1=tempVec.iloc[0][1:].astype(float).to_numpy()
10354                v2=tempVec.iloc[1][1:].astype(float).to_numpy()
10355                if len(v2) > diagnostic_n:
10356                    v1=v1[0:diagnostic_n]
10357                    v2=v2[0:diagnostic_n]
10358                mycorr = np.corrcoef( v1, v2 )[0,1]
10359                # mycorr=temparr[np.triu_indices_from(temparr, k=1)].mean()
10360                myerr=np.sqrt(np.mean((v1 - v2)**2))
10361                joinDiagnosticsLoc.iloc[0] = [id1,id2,mod_name,'rowavg',mycorr,myerr]
10362                if verbose:
10363                    print( joinDiagnosticsLoc )
10364                temp = jmmsubG1.loc[u][fl_names[1:]].astype(float)
10365                if mycorr > corr_thresh or len( v1 ) < 10:
10366                    jmmUniq.loc[u][fl_names[1:]] = temp.mean(axis=0)
10367                else:
10368                    jmmUniq.loc[u][fl_names[1:]] = nanList * temp.shape[1]
10369                joinDiagnostics = pd.concat( [joinDiagnostics, joinDiagnosticsLoc], 
10370                                            axis=0, ignore_index=False )
10371
10372    return jmmUniq, jmm, joinDiagnostics
10373
10374
10375
10376def quick_viz_mm_nrg(
10377    sourcedir, # root folder
10378    projectid, # project name
10379    sid , # subject unique id
10380    dtid, # date
10381    extract_brain=True,
10382    slice_factor = 0.55,
10383    post = False,
10384    original_sourcedir = None,
10385    filename = None, # output path
10386    verbose = True
10387):
10388    """
10389    This function creates visualizations of brain images for a specific subject in a project using ANTsPy.
10390
10391    Args:
10392
10393    sourcedir (str): Root folder for original data (if post=False) or processed data (post=True)
10394    
10395    projectid (str): Project name.
10396    
10397    sid (str): Subject unique id.
10398    
10399    dtid (str): Date.
10400    
10401    extract_brain (bool): If True, the function extracts the brain from the T1w image. Default is True.
10402    
10403    slice_factor (float): The slice to be visualized is determined by multiplying the image size by this factor. Default is 0.55.
10404
10405    post ( bool ) : if True, will visualize example post-processing results.
10406    
10407    original_sourcedir (str): Root folder for original data (used if post=True)
10408    
10409    filename (str): Output path with extension (.png)
10410    
10411    verbose (bool): If True, information will be printed while running the function. Default is True.
10412
10413    Returns:
10414    None
10415
10416    """
10417    iid='*'
10418    import glob as glob
10419    from os.path import exists
10420    import ants
10421    temp = sourcedir.split( "/" )
10422    subjectrootpath = os.path.join(sourcedir, projectid, sid, dtid)
10423    if verbose:
10424        print( 'subjectrootpath' )
10425        print( subjectrootpath )
10426    t1_search_path = os.path.join(subjectrootpath, "T1w", "*", "*nii.gz")
10427    if verbose:
10428        print(f"t1 search path: {t1_search_path}")
10429    t1fn = glob.glob(t1_search_path)
10430    if len( t1fn ) < 1:
10431        raise ValueError('quick_viz_mm_nrg cannot find the T1w @ ' + subjectrootpath )
10432    vizlist=[]
10433    undlist=[]
10434    nrg_modality_list = [ 'T1w', 'DTI', 'rsfMRI', 'perf', 'T2Flair', 'NM2DMT' ]
10435    if post:
10436        nrg_modality_list = [ 'T1wHierarchical', 'DTI', 'rsfMRI', 'perf', 'T2Flair', 'NM2DMT' ]
10437    for nrgNum in [0,1,2,3,4,5]:
10438        underlay = None
10439        overmodX = nrg_modality_list[nrgNum]
10440        if  'T1w' in overmodX :
10441            mod_search_path = os.path.join(subjectrootpath, overmodX, iid, "*nii.gz")
10442            if post:
10443                mod_search_path = os.path.join(subjectrootpath, overmodX, iid, "*brain_n4_dnz.nii.gz")
10444                mod_search_path_ol = os.path.join(subjectrootpath, overmodX, iid, "*thickness_image.nii.gz" )
10445                mod_search_path_ol = re.sub( "T1wHierarchical","T1w",mod_search_path_ol)
10446                myol = glob.glob(mod_search_path_ol)
10447                if len( myol ) > 0:
10448                    temper = find_most_recent_file( myol )[0]
10449                    underlay = ants.image_read(  temper )
10450                    if verbose:
10451                        print("T1w overlay " + temper )
10452                    underlay = underlay * ants.threshold_image( underlay, 0.2, math.inf )
10453            myimgsr = glob.glob(mod_search_path)
10454            if len( myimgsr ) == 0:
10455                if verbose:
10456                    print("No t1 images: " + sid + dtid )
10457                return None
10458            myimgsr=find_most_recent_file( myimgsr )[0]
10459            vimg=ants.image_read( myimgsr )
10460        elif  'T2Flair' in overmodX :
10461            if verbose:
10462                print("search flair")
10463            mod_search_path = os.path.join(subjectrootpath, overmodX, iid, "*nii.gz")
10464            if post and original_sourcedir is not None:
10465                if verbose:
10466                    print("post in flair")
10467                mysubdir = os.path.join(original_sourcedir, projectid, sid, dtid)
10468                mod_search_path_under = os.path.join(mysubdir, overmodX, iid, "*T2Flair*.nii.gz")
10469                if verbose:
10470                    print("post in flair mod_search_path_under " + mod_search_path_under)
10471                mod_search_path = os.path.join(subjectrootpath, overmodX, iid, "*wmh.nii.gz")
10472                if verbose:
10473                    print("post in flair mod_search_path " + mod_search_path )
10474                myimgul = glob.glob(mod_search_path_under)
10475                if len( myimgul ) > 0:
10476                    myimgul = find_most_recent_file( myimgul )[0]
10477                    if verbose:
10478                        print("Flair  " + myimgul )
10479                    vimg = ants.image_read( myimgul )
10480                    myol = glob.glob(mod_search_path)
10481                    if len( myol ) == 0:
10482                        underlay = myimgsr * 0.0
10483                    else:
10484                        myol = find_most_recent_file( myol )[0]
10485                        if verbose:
10486                            print("Flair overlay " + myol )
10487                        underlay=ants.image_read( myol )
10488                        underlay=underlay*ants.threshold_image(underlay,0.05,math.inf)
10489                else:
10490                    vimg = noizimg.clone()
10491                    underlay = vimg * 0.0
10492            if original_sourcedir is None:
10493                myimgsr = glob.glob(mod_search_path)
10494                if len( myimgsr ) == 0:
10495                    vimg = noizimg.clone()
10496                else:
10497                    myimgsr=find_most_recent_file( myimgsr )[0]
10498                    vimg=ants.image_read( myimgsr )
10499        elif overmodX == 'DTI':
10500            mod_search_path = os.path.join(subjectrootpath, 'DTI*', "*", "*nii.gz")
10501            if post:
10502                mod_search_path = os.path.join(subjectrootpath, 'DTI*', "*", "*fa.nii.gz")
10503            myimgsr = glob.glob(mod_search_path)
10504            if len( myimgsr ) > 0:
10505                myimgsr=find_most_recent_file( myimgsr )[0]
10506                vimg=ants.image_read( myimgsr )
10507            else:
10508                if verbose:
10509                    print("No " + overmodX)
10510                vimg = noizimg.clone()
10511        elif overmodX == 'DTI2':
10512            mod_search_path = os.path.join(subjectrootpath, 'DTI*', "*", "*nii.gz")
10513            myimgsr = glob.glob(mod_search_path)
10514            if len( myimgsr ) > 0:
10515                myimgsr.sort()
10516                myimgsr=myimgsr[len(myimgsr)-1]
10517                vimg=ants.image_read( myimgsr )
10518            else:
10519                if verbose:
10520                    print("No " + overmodX)
10521                vimg = noizimg.clone()
10522        elif overmodX == 'NM2DMT':
10523            mod_search_path = os.path.join(subjectrootpath, overmodX, "*", "*nii.gz")
10524            if post:
10525                mod_search_path = os.path.join(subjectrootpath, overmodX, "*", "*NM_avg.nii.gz" )
10526            myimgsr = glob.glob(mod_search_path)
10527            if len( myimgsr ) > 0:
10528                myimgsr0=myimgsr[0]
10529                vimg=ants.image_read( myimgsr0 )
10530                for k in range(1,len(myimgsr)):
10531                    temp = ants.image_read( myimgsr[k])
10532                    vimg=vimg+ants.resample_image_to_target(temp,vimg)
10533            else:
10534                if verbose:
10535                    print("No " + overmodX)
10536                vimg = noizimg.clone()
10537        elif overmodX == 'rsfMRI':
10538            mod_search_path = os.path.join(subjectrootpath, 'rsfMRI*', "*", "*nii.gz")
10539            if post:
10540                mod_search_path = os.path.join(subjectrootpath, 'rsfMRI*', "*", "*fcnxpro122_meanBold.nii.gz" )
10541                mod_search_path_ol = os.path.join(subjectrootpath, 'rsfMRI*', "*", "*fcnxpro122_DefaultMode.nii.gz" )
10542                myol = glob.glob(mod_search_path_ol)
10543                if len( myol ) > 0:
10544                    myol = find_most_recent_file( myol )[0]
10545                    underlay = ants.image_read( myol )
10546                    if verbose:
10547                        print("BOLD overlay " + myol )
10548                    underlay = underlay * ants.threshold_image( underlay, 0.1, math.inf )
10549            myimgsr = glob.glob(mod_search_path)
10550            if len( myimgsr ) > 0:
10551                myimgsr=find_most_recent_file( myimgsr )[0]
10552                vimg=mm_read_to_3d( myimgsr )
10553            else:
10554                if verbose:
10555                    print("No " + overmodX)
10556                vimg = noizimg.clone()
10557        elif overmodX == 'perf':
10558            mod_search_path = os.path.join(subjectrootpath, 'perf*', "*", "*nii.gz")
10559            if post:
10560                mod_search_path = os.path.join(subjectrootpath, 'perf*', "*", "*cbf.nii.gz")
10561            myimgsr = glob.glob(mod_search_path)
10562            if len( myimgsr ) > 0:
10563                myimgsr=find_most_recent_file( myimgsr )[0]
10564                vimg=mm_read_to_3d( myimgsr )
10565            else:
10566                if verbose:
10567                    print("No " + overmodX)
10568                vimg = noizimg.clone()
10569        else :
10570            if verbose:
10571                print("Something else here")
10572            mod_search_path = os.path.join(subjectrootpath, overmodX, "*", "*nii.gz")
10573            myimgsr = glob.glob(mod_search_path)
10574            if post:
10575                myimgsr=[]
10576            if len( myimgsr ) > 0:
10577                myimgsr=find_most_recent_file( myimgsr )[0]
10578                vimg=ants.image_read( myimgsr )
10579            else:
10580                if verbose:
10581                    print("No " + overmodX)
10582                vimg = noizimg
10583        if True:
10584            if extract_brain and overmodX == 'T1w' and post == False:
10585                vimg = vimg * antspyt1w.brain_extraction(vimg)
10586            if verbose:
10587                print(f"modality search path: {myimgsr}" + " num: " + str(nrgNum))
10588            if vimg.dimension == 4 and ( overmodX == "DTI2"  ):
10589                ttb0, ttdw=get_average_dwi_b0(vimg)
10590                vimg = ttdw
10591            elif vimg.dimension == 4 and overmodX == "DTI":
10592                ttb0, ttdw=get_average_dwi_b0(vimg)
10593                vimg = ttb0
10594            elif vimg.dimension == 4 :
10595                vimg=ants.get_average_of_timeseries(vimg)
10596            msk=ants.get_mask(vimg)
10597            if overmodX == 'T2Flair':
10598                msk=vimg*0+1
10599            if underlay is not None:
10600                print( overmodX + " has underlay" )
10601            else:
10602                underlay = vimg * 0.0
10603            if nrgNum == 0:
10604                refimg=ants.image_clone( vimg )
10605                noizimg = ants.add_noise_to_image( refimg*0, 'additivegaussian', [100,1] )
10606                vizlist.append( vimg )
10607                undlist.append( underlay )
10608            else:
10609                vimg = ants.iMath( vimg, 'TruncateIntensity',0.01,0.98)
10610                vizlist.append( ants.iMath( vimg, 'Normalize' ) * 255 )
10611                undlist.append( underlay )
10612
10613    # mask & crop systematically ...
10614    msk = ants.get_mask( refimg )
10615    refimg = ants.crop_image( refimg, msk )
10616
10617    for jj in range(len(vizlist)):
10618        vizlist[jj]=ants.resample_image_to_target( vizlist[jj], refimg )
10619        undlist[jj]=ants.resample_image_to_target( undlist[jj], refimg )
10620        print( 'viz: ' + str( jj ) )
10621        print( vizlist[jj] )
10622        print( 'und: ' + str( jj ) )
10623        print( undlist[jj] )
10624
10625
10626    xyz = [None]*3
10627    for i in range(3):
10628        if xyz[i] is None:
10629            xyz[i] = int(refimg.shape[i] * slice_factor )
10630
10631    if verbose:
10632        print('slice positions')
10633        print( xyz )
10634
10635    ants.plot_ortho_stack( vizlist, overlays=undlist, crop=False, reorient=False, filename=filename, xyz=xyz, orient_labels=False )
10636    return
10637    # listlen = len( vizlist )
10638    # vizlist = np.asarray( vizlist )
10639    if show_it is not None:
10640        filenameout=None
10641        if verbose:
10642            print( show_it )
10643        for a in [0,1,2]:
10644            n=int(np.round( refimg.shape[a] * slice_factor ))
10645            slices=np.repeat( int(n), listlen  )
10646            if isinstance(show_it,str):
10647                filenameout=show_it+'_ax'+str(int(a))+'_sl'+str(n)+'.png'
10648                if verbose:
10649                    print( filenameout )
10650#            ants.plot_grid(vizlist.reshape(2,3), slices.reshape(2,3), title='MM Subject ' + sid + ' ' + dtid, rfacecolor='white', axes=a, filename=filenameout )
10651    if verbose:
10652        print("viz complete.")
10653    return vizlist
10654
10655
10656def blind_image_assessment(
10657    image,
10658    viz_filename=None,
10659    title=False,
10660    pull_rank=False,
10661    resample=None,
10662    n_to_skip = 10,
10663    verbose=False
10664):
10665    """
10666    quick blind image assessment and triplanar visualization of an image ... 4D input will be visualized and assessed in 3D.  produces a png and csv where csv contains:
10667
10668    * reflection error ( estimates asymmetry )
10669
10670    * brisq ( blind quality assessment )
10671
10672    * patch eigenvalue ratio ( blind quality assessment )
10673
10674    * PSNR and SSIM vs a smoothed reference (4D or 3D appropriate)
10675
10676    * mask volume ( estimates foreground object size )
10677
10678    * spacing
10679
10680    * dimension after cropping by mask
10681
10682    image : character or image object usually a nifti image
10683
10684    viz_filename : character for a png output image
10685
10686    title : display a summary title on the png
10687
10688    pull_rank : boolean
10689
10690    resample : None, numeric max or min, resamples image to isotropy
10691
10692    n_to_skip : 10 by default; samples time series every n_to_skip volume
10693
10694    verbose : boolean
10695
10696    """
10697    import glob as glob
10698    from os.path import exists
10699    import ants
10700    import matplotlib.pyplot as plt
10701    from PIL import Image
10702    from pathlib import Path
10703    import json
10704    import re
10705    from dipy.io.gradients import read_bvals_bvecs
10706    mystem=''
10707    if isinstance(image,list):
10708        isfilename=isinstance( image[0], str)
10709        image = image[0]
10710    else:
10711        isfilename=isinstance( image, str)
10712    outdf = pd.DataFrame()
10713    mymeta = None
10714    MagneticFieldStrength = None
10715    image_filename=''
10716    if isfilename:
10717        image_filename = image
10718        if isinstance(image,list):
10719            image_filename=image[0]
10720        json_name = re.sub(".nii.gz",".json",image_filename)
10721        if exists( json_name ):
10722            try:
10723                with open(json_name, 'r') as fcc_file:
10724                    mymeta = json.load(fcc_file)
10725                    if verbose:
10726                        print(json.dumps(mymeta, indent=4))
10727                    fcc_file.close()
10728            except:
10729                pass
10730        mystem=Path( image ).stem
10731        mystem=Path( mystem ).stem
10732        image_reference = ants.image_read( image )
10733        image = ants.image_read( image )
10734    else:
10735        image_reference = ants.image_clone( image )
10736    ntimepoints = 1
10737    bvalueMax=None
10738    bvecnorm=None
10739    if image_reference.dimension == 4:
10740        ntimepoints = image_reference.shape[3]
10741        if "DTI" in image_filename:
10742            myTSseg = segment_timeseries_by_meanvalue( image_reference )
10743            image_b0, image_dwi = get_average_dwi_b0( image_reference, fast=True )
10744            image_b0 = ants.iMath( image_b0, 'Normalize' )
10745            image_dwi = ants.iMath( image_dwi, 'Normalize' )
10746            bval_name = re.sub(".nii.gz",".bval",image_filename)
10747            bvec_name = re.sub(".nii.gz",".bvec",image_filename)
10748            if exists( bval_name ) and exists( bvec_name ):
10749                bvals, bvecs = read_bvals_bvecs( bval_name , bvec_name  )
10750                bvalueMax = bvals.max()
10751                bvecnorm = np.linalg.norm(bvecs,axis=1).reshape( bvecs.shape[0],1 )
10752                bvecnorm = bvecnorm.max()
10753        else:
10754            image_b0 = ants.get_average_of_timeseries( image_reference ).iMath("Normalize")
10755    else:
10756        image_compare = ants.smooth_image( image_reference, 3, sigma_in_physical_coordinates=False )
10757    for jjj in range(0,ntimepoints,n_to_skip):
10758        modality='unknown'
10759        if "rsfMRI" in image_filename:
10760            modality='rsfMRI'
10761        elif "perf" in image_filename:
10762            modality='perf'
10763        elif "DTI" in image_filename:
10764            modality='DTI'
10765        elif "T1w" in image_filename:
10766            modality='T1w'
10767        elif "T2Flair" in image_filename:
10768            modality='T2Flair'
10769        elif "NM2DMT" in image_filename:
10770            modality='NM2DMT'
10771        if image_reference.dimension == 4:
10772            image = ants.slice_image( image_reference, idx=int(jjj), axis=3 )
10773            if "DTI" in image_filename:
10774                if jjj in myTSseg['highermeans']:
10775                    image_compare = ants.image_clone( image_b0 )
10776                    modality='DTIb0'
10777                else:
10778                    image_compare = ants.image_clone( image_dwi )
10779                    modality='DTIdwi'
10780            else:
10781                image_compare = ants.image_clone( image_b0 )
10782        # image = ants.iMath( image, 'TruncateIntensity',0.01,0.995)
10783        minspc = np.min(ants.get_spacing(image))
10784        maxspc = np.max(ants.get_spacing(image))
10785        if resample is not None:
10786            if resample == 'min':
10787                if minspc < 1e-12:
10788                    minspc = np.max(ants.get_spacing(image))
10789                newspc = np.repeat( minspc, 3 )
10790            elif resample == 'max':
10791                newspc = np.repeat( maxspc, 3 )
10792            else:
10793                newspc = np.repeat( resample, 3 )
10794            image = ants.resample_image( image, newspc )
10795            image_compare = ants.resample_image( image_compare, newspc )
10796        else:
10797            # check for spc close to zero
10798            spc = list(ants.get_spacing(image))
10799            for spck in range(len(spc)):
10800                if spc[spck] < 1e-12:
10801                    spc[spck]=1
10802            ants.set_spacing( image, spc )
10803            ants.set_spacing( image_compare, spc )
10804        # if "NM2DMT" in image_filename or "FIXME" in image_filename or "SPECT" in image_filename or "UNKNOWN" in image_filename:
10805        minspc = np.min(ants.get_spacing(image))
10806        maxspc = np.max(ants.get_spacing(image))
10807        msk = ants.threshold_image( ants.iMath(image,'Normalize'), 0.15, 1.0 )
10808        # else:
10809        #    msk = ants.get_mask( image )
10810        msk = ants.morphology(msk, "close", 3 )
10811        bgmsk = msk*0+1-msk
10812        mskdil = ants.iMath(msk, "MD", 4 )
10813        # ants.plot_ortho( image, msk, crop=False )
10814        nvox = int( msk.sum() )
10815        spc = ants.get_spacing( image )
10816        org = ants.get_origin( image )
10817        if ( nvox > 0 ):
10818            image = ants.crop_image( image, mskdil ).iMath("Normalize")
10819            msk = ants.crop_image( msk, mskdil ).iMath("Normalize")
10820            bgmsk = ants.crop_image( bgmsk, mskdil ).iMath("Normalize")
10821            image_compare = ants.crop_image( image_compare, mskdil ).iMath("Normalize")           
10822            npatch = int( np.round(  0.1 * nvox ) )
10823            npatch = np.min(  [512,npatch ] )
10824            patch_shape = []
10825            for k in range( 3 ):
10826                p = int( 32.0 / ants.get_spacing( image  )[k] )
10827                if p > int( np.round( image.shape[k] * 0.5 ) ):
10828                    p = int( np.round( image.shape[k] * 0.5 ) )
10829                patch_shape.append( p )
10830            if verbose:
10831                print(image)
10832                print( patch_shape )
10833                print( npatch )
10834            myevr = math.nan # dont want to fail if something odd happens in patch extraction
10835            try:
10836                myevr = antspyt1w.patch_eigenvalue_ratio( image, npatch, patch_shape,
10837                    evdepth = 0.9, mask=msk )
10838            except:
10839                pass
10840            if pull_rank:
10841                image = ants.rank_intensity(image)
10842            imagereflect = ants.reflect_image(image, axis=0)
10843            asym_err = ( image - imagereflect ).abs().mean()
10844            # estimate noise by center cropping, denoizing and taking magnitude of difference
10845            nocrop=False
10846            if image.dimension == 3:
10847                if image.shape[2] == 1:
10848                    nocrop=True        
10849            if maxspc/minspc > 10:
10850                nocrop=True
10851            if nocrop:
10852                mycc = ants.image_clone( image )
10853            else:
10854                mycc = antspyt1w.special_crop( image,
10855                    ants.get_center_of_mass( msk *0 + 1 ), patch_shape )
10856            myccd = ants.denoise_image( mycc, p=1,r=1,noise_model='Gaussian' )
10857            noizlevel = ( mycc - myccd ).abs().mean()
10858    #        ants.plot_ortho( image, crop=False, filename=viz_filename, flat=True, xyz_lines=False, orient_labels=False, xyz_pad=0 )
10859    #        from brisque import BRISQUE
10860    #        obj = BRISQUE(url=False)
10861    #        mybrisq = obj.score( np.array( Image.open( viz_filename )) )
10862            msk_vol = msk.sum() * np.prod( spc )
10863            bgstd = image[ bgmsk == 1 ].std()
10864            fgmean = image[ msk == 1 ].mean()
10865            bgmean = image[ bgmsk == 1 ].mean()
10866            snrref = fgmean / bgstd
10867            cnrref = ( fgmean - bgmean ) / bgstd
10868            psnrref = antspynet.psnr(  image_compare, image  )
10869            ssimref = antspynet.ssim(  image_compare, image  )
10870            if nocrop:
10871                mymi = math.inf
10872            else:
10873                mymi = ants.image_mutual_information( image_compare, image )
10874        else:
10875            msk_vol = 0
10876            myevr = mymi = ssimref = psnrref = cnrref = asym_err = noizlevel = math.nan
10877            
10878        mriseries=None
10879        mrimfg=None
10880        mrimodel=None
10881        mriSAR=None
10882        BandwidthPerPixelPhaseEncode=None
10883        PixelBandwidth=None
10884        if mymeta is not None:
10885            # mriseries=mymeta['']
10886            try:
10887                mrimfg=mymeta['Manufacturer']
10888            except:
10889                pass
10890            try:
10891                mrimodel=mymeta['ManufacturersModelName']
10892            except:
10893                pass
10894            try:
10895                MagneticFieldStrength=mymeta['MagneticFieldStrength']
10896            except:
10897                pass
10898            try:
10899                PixelBandwidth=mymeta['PixelBandwidth']
10900            except:
10901                pass
10902            try:
10903                BandwidthPerPixelPhaseEncode=mymeta['BandwidthPerPixelPhaseEncode']
10904            except:
10905                pass
10906            try:
10907                mriSAR=mymeta['SAR']
10908            except:
10909                pass
10910        ttl=mystem + ' '
10911        ttl=''
10912        ttl=ttl + "NZ: " + "{:0.4f}".format(noizlevel) + " SNR: " + "{:0.4f}".format(snrref) + " CNR: " + "{:0.4f}".format(cnrref) + " PS: " + "{:0.4f}".format(psnrref)+ " SS: " + "{:0.4f}".format(ssimref) + " EVR: " + "{:0.4f}".format(myevr)+ " MI: " + "{:0.4f}".format(mymi)
10913        if viz_filename is not None and ( jjj == 0 or (jjj % 30 == 0) ) and image.shape[2] < 685:
10914            viz_filename_use = re.sub( ".png", "_slice"+str(jjj).zfill(4)+".png", viz_filename )
10915            ants.plot_ortho( image, crop=False, filename=viz_filename_use, flat=True, xyz_lines=False, orient_labels=False, xyz_pad=0,  title=ttl, titlefontsize=12, title_dy=-0.02,textfontcolor='red' )
10916        df = pd.DataFrame([[ 
10917            mystem, 
10918            image_reference.dimension, 
10919            noizlevel, snrref, cnrref, psnrref, ssimref, mymi, asym_err, myevr, msk_vol, 
10920            spc[0], spc[1], spc[2],org[0], org[1], org[2], 
10921            image.shape[0], image.shape[1], image.shape[2], ntimepoints, 
10922            jjj, modality, mriseries, mrimfg, mrimodel, MagneticFieldStrength, mriSAR, PixelBandwidth, BandwidthPerPixelPhaseEncode, bvalueMax, bvecnorm ]], 
10923            columns=[
10924                'filename', 
10925                'dimensionality',
10926                'noise', 'snr', 'cnr', 'psnr', 'ssim', 'mi', 'reflection_err', 'EVR', 'msk_vol', 'spc0','spc1','spc2','org0','org1','org2','dimx','dimy','dimz','dimt','slice','modality', 'mriseries', 'mrimfg', 'mrimodel', 'mriMagneticFieldStrength', 'mriSAR', 'mriPixelBandwidth', 'mriPixelBandwidthPE', 'dti_bvalueMax', 'dti_bvecnorm' ])
10927        outdf = pd.concat( [outdf, df ], axis=0, ignore_index=False )
10928        if verbose:
10929            print( outdf )
10930    if viz_filename is not None:
10931        csvfn = re.sub( "png", "csv", viz_filename )
10932        outdf.to_csv( csvfn )
10933    return outdf
10934
10935def remove_unwanted_columns(df):
10936    # Identify columns to drop: those named 'X' or starting with 'Unnamed'
10937    cols_to_drop = [col for col in df.columns if col == 'X' or col.startswith('Unnamed')]
10938    
10939    # Drop the identified columns from the DataFrame, if any
10940    df_cleaned = df.drop(columns=cols_to_drop, errors='ignore')
10941    
10942    return df_cleaned
10943
10944def process_dataframe_generalized(df, group_by_column):
10945    # Make sure the group_by_column is excluded from both numeric and other columns calculations
10946    numeric_cols = df.select_dtypes(include='number').columns.difference([group_by_column])
10947    other_cols = df.columns.difference(numeric_cols).difference([group_by_column])
10948    
10949    # Define aggregation functions: mean for numeric cols, mode for other cols
10950    # Update to handle empty mode results safely
10951    agg_dict = {col: 'mean' for col in numeric_cols}
10952    agg_dict.update({
10953        col: lambda x: pd.Series.mode(x).iloc[0] if not pd.Series.mode(x).empty else None for col in other_cols
10954    })    
10955    # Group by the specified column, applying different aggregation functions to different columns
10956    processed_df = df.groupby(group_by_column, as_index=False).agg(agg_dict)
10957    return processed_df
10958
10959def average_blind_qc_by_modality(qc_full,verbose=False):
10960    """
10961    Averages time series qc results to yield one entry per image. this also filters to "known" columns.
10962
10963    Args:
10964    qc_full: pandas dataframe containing the full qc data.
10965
10966    Returns:
10967    pandas dataframe containing the processed qc data.
10968    """
10969    qc_full = remove_unwanted_columns( qc_full )
10970    # Get unique modalities
10971    modalities = qc_full['modality'].unique()
10972    modalities = modalities[modalities != 'unknown']
10973    # Get unique ids
10974    uid = qc_full['filename']
10975    to_average = uid.unique()
10976    meta = pd.DataFrame(columns=qc_full.columns )
10977    # Process each unique id
10978    n = len(to_average)
10979    for k in range(n):
10980        if verbose:
10981            if k % 100 == 0:
10982                progger = str( np.round( k / n * 100 ) )
10983                print( progger, end ="...", flush=True)
10984        m1sel = uid == to_average[k]
10985        if sum(m1sel) > 1:
10986            # If more than one entry for id, take the average of continuous columns,
10987            # maximum of the slice column, and the first entry of the other columns
10988            mfsub = process_dataframe_generalized(qc_full[m1sel],'filename')
10989        else:
10990            mfsub = qc_full[m1sel]
10991        meta.loc[k] = mfsub.iloc[0]
10992    meta['modality'] = meta['modality'].replace(['DTIdwi', 'DTIb0'], 'DTI', regex=True)
10993    return meta
10994
10995def wmh( flair, t1, t1seg,
10996    mmfromconvexhull = 3.0,
10997    strict=True,
10998    probability_mask=None,
10999    prior_probability=None,
11000    model='sysu',
11001    verbose=False ) :
11002    """
11003    Outputs the WMH probability mask and a summary single measurement
11004
11005    Arguments
11006    ---------
11007    flair : ANTsImage
11008        input 3-D FLAIR brain image (not skull-stripped).
11009
11010    t1 : ANTsImage
11011        input 3-D T1 brain image (not skull-stripped).
11012
11013    t1seg : ANTsImage
11014        T1 segmentation image
11015
11016    mmfromconvexhull : float
11017        restrict WMH to regions that are WM or mmfromconvexhull mm away from the
11018        convex hull of the cerebrum.   we choose a default value based on
11019        Figure 4 from:
11020        https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6240579/pdf/fnagi-10-00339.pdf
11021
11022    strict: boolean - if True, only use convex hull distance
11023
11024    probability_mask : None - use to compute wmh just once - then this function
11025        just does refinement and summary
11026
11027    prior_probability : optional prior probability image in space of the input t1
11028
11029    model : either sysu or hyper
11030
11031    verbose : boolean
11032
11033    Returns
11034    ---------
11035    WMH probability map and a summary single measurement which is the sum of the WMH map
11036
11037    """
11038    import numpy as np
11039    import math
11040    t1_2_flair_reg = ants.registration(flair, t1, type_of_transform = 'antsRegistrationSyNRepro[r]') # Register T1 to Flair
11041    if probability_mask is None and model == 'sysu':
11042        if verbose:
11043            print('sysu')
11044        probability_mask = antspynet.sysu_media_wmh_segmentation( flair )
11045    elif probability_mask is None and model == 'hyper':
11046        if verbose:
11047            print('hyper')
11048        probability_mask = antspynet.hypermapp3r_segmentation( t1_2_flair_reg['warpedmovout'], flair )
11049    # t1_2_flair_reg = tra_initializer( flair, t1, n_simulations=4, max_rotation=5, transform=['rigid'], verbose=False )
11050    prior_probability_flair = None
11051    if prior_probability is not None:
11052        prior_probability_flair = ants.apply_transforms( flair, prior_probability,
11053            t1_2_flair_reg['fwdtransforms'] )
11054    wmseg_mask = ants.threshold_image( t1seg,
11055        low_thresh = 3, high_thresh = 3).iMath("FillHoles")
11056    wmseg_mask_use = ants.image_clone( wmseg_mask )
11057    distmask = None
11058    if mmfromconvexhull > 0:
11059            convexhull = ants.threshold_image( t1seg, 1, 4 )
11060            spc2vox = np.prod( ants.get_spacing( t1seg ) )
11061            voxdist = 0.0
11062            myspc = ants.get_spacing( t1seg )
11063            for k in range( t1seg.dimension ):
11064                voxdist = voxdist + myspc[k] * myspc[k]
11065            voxdist = math.sqrt( voxdist )
11066            nmorph = round( 2.0 / voxdist )
11067            convexhull = ants.morphology( convexhull, "close", nmorph ).iMath("FillHoles")
11068            dist = ants.iMath( convexhull, "MaurerDistance" ) * -1.0
11069            distmask = ants.threshold_image( dist, mmfromconvexhull, 1.e80 )
11070            wmseg_mask = wmseg_mask + distmask
11071            if strict:
11072                wmseg_mask_use = ants.threshold_image( wmseg_mask, 2, 2 )
11073            else:
11074                wmseg_mask_use = ants.threshold_image( wmseg_mask, 1, 2 )
11075    ##############################################################################
11076    wmseg_2_flair = ants.apply_transforms(flair, wmseg_mask_use,
11077        transformlist = t1_2_flair_reg['fwdtransforms'],
11078        interpolator = 'nearestNeighbor' )
11079    seg_2_flair = ants.apply_transforms(flair, t1seg,
11080        transformlist = t1_2_flair_reg['fwdtransforms'],
11081        interpolator = 'nearestNeighbor' )
11082    csfmask = ants.threshold_image(seg_2_flair,1,1)
11083    flairsnr = mask_snr( flair, csfmask, wmseg_2_flair, bias_correct = False )
11084    probability_mask_WM = wmseg_2_flair * probability_mask # Remove WMH signal outside of WM
11085    wmh_sum = np.prod( ants.get_spacing( flair ) ) * probability_mask_WM.sum()
11086    wmh_sum_prior = math.nan
11087    probability_mask_posterior = None
11088    if prior_probability_flair is not None:
11089        probability_mask_posterior = prior_probability_flair * probability_mask # use prior
11090        wmh_sum_prior = np.prod( ants.get_spacing(flair) ) * probability_mask_posterior.sum()
11091    if math.isnan( wmh_sum ):
11092        wmh_sum=0
11093    if math.isnan( wmh_sum_prior ):
11094        wmh_sum_prior=0
11095    flair_evr = antspyt1w.patch_eigenvalue_ratio( flair, 512, [16,16,16], evdepth = 0.9, mask=wmseg_2_flair )
11096    return{
11097        'WMH_probability_map_raw': probability_mask,
11098        'WMH_probability_map' : probability_mask_WM,
11099        'WMH_posterior_probability_map' : probability_mask_posterior,
11100        'wmh_mass': wmh_sum,
11101        'wmh_mass_prior': wmh_sum_prior,
11102        'wmh_evr' : flair_evr,
11103        'wmh_SNR' : flairsnr,
11104        'convexhull_mask': distmask }
11105
11106
11107def replace_elements_in_numpy_array(original_array, indices_to_replace, new_value):
11108    """
11109    Replace specified elements or rows in a numpy array with a new value.
11110
11111    Parameters:
11112    original_array (numpy.ndarray): A numpy array in which elements or rows are to be replaced.
11113    indices_to_replace (list or numpy.ndarray): Indices of elements or rows to be replaced.
11114    new_value: The new value to replace the specified elements or rows.
11115
11116    Returns:
11117    numpy.ndarray: A new numpy array with the specified elements or rows replaced. If the input array is None,
11118                   the function returns None.
11119    """
11120
11121    if original_array is None:
11122        return None
11123
11124    max_index = original_array.size if original_array.ndim == 1 else original_array.shape[0]
11125
11126    # Filter out invalid indices and check for any out-of-bounds indices
11127    valid_indices = []
11128    for idx in indices_to_replace:
11129        if idx < max_index:
11130            valid_indices.append(idx)
11131        else:
11132            warnings.warn(f"Warning: Index {idx} is out of bounds and will be ignored.")
11133
11134    if original_array.ndim == 1:
11135        # Replace elements in a 1D array
11136        original_array[valid_indices] = new_value
11137    elif original_array.ndim == 2:
11138        # Replace rows in a 2D array
11139        original_array[valid_indices, :] = new_value
11140    else:
11141        raise ValueError("original_array must be either 1D or 2D.")
11142
11143    return original_array
11144
11145
11146
11147def remove_elements_from_numpy_array(original_array, indices_to_remove):
11148    """
11149    Remove specified elements or rows from a numpy array.
11150
11151    Parameters:
11152    original_array (numpy.ndarray): A numpy array from which elements or rows are to be removed.
11153    indices_to_remove (list or numpy.ndarray): Indices of elements or rows to be removed.
11154
11155    Returns:
11156    numpy.ndarray: A new numpy array with the specified elements or rows removed. If the input array is None,
11157                   the function returns None.
11158    """
11159
11160    if original_array is None:
11161        return None
11162
11163    if original_array.ndim == 1:
11164        # Remove elements from a 1D array
11165        return np.delete(original_array, indices_to_remove)
11166    elif original_array.ndim == 2:
11167        # Remove rows from a 2D array
11168        return np.delete(original_array, indices_to_remove, axis=0)
11169    else:
11170        raise ValueError("original_array must be either 1D or 2D.")
11171
11172def remove_volumes_from_timeseries(time_series, volumes_to_remove):
11173    """
11174    Remove specified volumes from a time series.
11175
11176    :param time_series: ANTsImage representing the time series (4D image).
11177    :param volumes_to_remove: List of volume indices to remove.
11178    :return: ANTsImage with specified volumes removed.
11179    """
11180    if not isinstance(time_series, ants.core.ants_image.ANTsImage):
11181        raise ValueError("time_series must be an ANTsImage.")
11182
11183    if time_series.dimension != 4:
11184        raise ValueError("time_series must be a 4D image.")
11185
11186    # Create a boolean index for volumes to keep
11187    volumes_to_keep = [i for i in range(time_series.shape[3]) if i not in volumes_to_remove]
11188
11189    # Select the volumes to keep
11190    filtered_time_series = ants.from_numpy( time_series.numpy()[..., volumes_to_keep] )
11191
11192    return ants.copy_image_info( time_series, filtered_time_series )
11193
11194def remove_elements_from_list(original_list, elements_to_remove):
11195    """
11196    Remove specified elements from a list.
11197
11198    Parameters:
11199    original_list (list): The original list from which elements will be removed.
11200    elements_to_remove (list): A list of elements that need to be removed from the original list.
11201
11202    Returns:
11203    list: A new list with the specified elements removed.
11204    """
11205    return [element for element in original_list if element not in elements_to_remove]
11206
11207
11208def impute_timeseries(time_series, volumes_to_impute, method='linear', verbose=False):
11209    """
11210    Impute specified volumes from a time series with interpolated values.
11211
11212    :param time_series: ANTsImage representing the time series (4D image).
11213    :param volumes_to_impute: List of volume indices to impute.
11214    :param method: Interpolation method ('linear' or other methods if implemented).
11215    :param verbose: boolean
11216    :return: ANTsImage with specified volumes imputed.
11217    """
11218    if not isinstance(time_series, ants.core.ants_image.ANTsImage):
11219        raise ValueError("time_series must be an ANTsImage.")
11220
11221    if time_series.dimension != 4:
11222        raise ValueError("time_series must be a 4D image.")
11223
11224    # Convert time_series to numpy for manipulation
11225    time_series_np = time_series.numpy()
11226    total_volumes = time_series_np.shape[3]
11227
11228    # Create a complement list of volumes not to impute
11229    volumes_not_to_impute = [i for i in range(total_volumes) if i not in volumes_to_impute]
11230
11231    # Define the lower and upper bounds
11232    min_valid_index = min(volumes_not_to_impute)
11233    max_valid_index = max(volumes_not_to_impute)
11234
11235    for vol_idx in volumes_to_impute:
11236        # Ensure the volume index is within the valid range
11237        if vol_idx < 0 or vol_idx >= total_volumes:
11238            raise ValueError(f"Volume index {vol_idx} is out of bounds.")
11239
11240        # Find the nearest valid lower index within the bounds
11241        lower_candidates = [v for v in volumes_not_to_impute if v <= vol_idx]
11242        lower_idx = max(lower_candidates) if lower_candidates else min_valid_index
11243
11244        # Find the nearest valid upper index within the bounds
11245        upper_candidates = [v for v in volumes_not_to_impute if v >= vol_idx]
11246        upper_idx = min(upper_candidates) if upper_candidates else max_valid_index
11247
11248        if verbose:
11249            print(f"Imputing volume {vol_idx} using indices {lower_idx} and {upper_idx}")
11250
11251        if method == 'linear':
11252            # Linear interpolation between the two nearest volumes
11253            lower_volume = time_series_np[..., lower_idx]
11254            upper_volume = time_series_np[..., upper_idx]
11255            interpolated_volume = (lower_volume + upper_volume) / 2
11256        else:
11257            # Placeholder for other interpolation methods
11258            raise NotImplementedError("Currently, only linear interpolation is implemented.")
11259
11260        # Replace the specified volume with the interpolated volume
11261        time_series_np[..., vol_idx] = interpolated_volume
11262
11263    # Convert the numpy array back to ANTsImage
11264    imputed_time_series = ants.from_numpy(time_series_np)
11265    imputed_time_series = ants.copy_image_info(time_series, imputed_time_series)
11266
11267    return imputed_time_series
11268
11269def impute_dwi( dwi, threshold = 0.20, imputeb0=False, mask=None, verbose=False ):
11270    """
11271    Identify bad volumes in a dwi and impute them fully automatically.
11272
11273    :param dwi: ANTsImage representing the time series (4D image).
11274    :param threshold: threshold (0,1) for outlierness (lower means impute more data)
11275    :param imputeb0: boolean will impute the b0 with dwi if True
11276    :param mask: restricts to a region of interest
11277    :param verbose: boolean
11278    :return: ANTsImage automatically imputed.
11279    """
11280    list1 = segment_timeseries_by_meanvalue( dwi )['highermeans']
11281    if imputeb0:
11282        dwib = impute_timeseries( dwi, list1 ) # focus on the dwi - not the b0
11283        looped, list2 = loop_timeseries_censoring( dwib, threshold, mask )
11284    else:
11285        looped, list2 = loop_timeseries_censoring( dwi, threshold, mask )
11286    if verbose:
11287        print( list1 )
11288        print( list2 )
11289    complement = remove_elements_from_list( list2, list1 )
11290    if verbose:
11291        print( "Imputing:")
11292        print( complement )
11293    if len( complement ) == 0:
11294        return dwi
11295    return impute_timeseries( dwi, complement )
11296
11297def censor_dwi( dwi, bval, bvec, threshold = 0.20, imputeb0=False, mask=None, verbose=False ):
11298    """
11299    Identify bad volumes in a dwi and impute them fully automatically.
11300
11301    :param dwi: ANTsImage representing the time series (4D image).
11302    :param bval: bval array
11303    :param bvec: bvec array
11304    :param threshold: threshold (0,1) for outlierness (lower means impute more data)
11305    :param imputeb0: boolean will impute the b0 with dwi if True
11306    :param mask: restricts to a region of interest
11307    :param verbose: boolean
11308    :return: ANTsImage automatically imputed.
11309    """
11310    list1 = segment_timeseries_by_meanvalue( dwi )['highermeans']
11311    if imputeb0:
11312        dwib = impute_timeseries( dwi, list1 ) # focus on the dwi - not the b0
11313        looped, list2 = loop_timeseries_censoring( dwib, threshold, mask, verbose=verbose)
11314    else:
11315        looped, list2 = loop_timeseries_censoring( dwi, threshold, mask, verbose=verbose )
11316    if verbose:
11317        print( list1 )
11318        print( list2 )
11319    complement = remove_elements_from_list( list2, list1 )
11320    if verbose:
11321        print( "censoring:")
11322        print( complement )
11323    if len( complement ) == 0:
11324        return dwi, bval, bvec
11325    return remove_volumes_from_timeseries( dwi, complement ), remove_elements_from_numpy_array( bval, complement ), remove_elements_from_numpy_array( bvec, complement )
11326
11327
11328def flatten_time_series(time_series):
11329    """
11330    Flatten a 4D time series into a 2D array.
11331    
11332    :param time_series: A 4D numpy array where the last dimension is time.
11333    :return: A 2D numpy array where each row is a flattened volume.
11334    """
11335    n_volumes = time_series.shape[3]
11336    return time_series.reshape(-1, n_volumes).T
11337
11338def calculate_loop_scores_full(flattened_series, n_neighbors=20, verbose=True ):
11339    """
11340    Calculate Local Outlier Probabilities for each volume.
11341    
11342    :param flattened_series: A 2D numpy array from flatten_time_series.
11343    :param n_neighbors: Number of neighbors to use for calculating LOF scores.
11344    :param verbose: boolean
11345    :return: An array of LoOP scores.
11346    """
11347    from PyNomaly import loop
11348    from sklearn.neighbors import NearestNeighbors
11349    from sklearn.preprocessing import StandardScaler
11350    # replace nans with zero
11351    if verbose:
11352        print("loop: nan_to_num")
11353    flattened_series=np.nan_to_num(flattened_series, nan=0)
11354    scaler = StandardScaler()
11355    scaler.fit(flattened_series)
11356    data = scaler.transform(flattened_series)
11357    data=np.nan_to_num(data, nan=0)
11358    if n_neighbors > int(flattened_series.shape[0]/2.0):
11359        n_neighbors = int(flattened_series.shape[0]/2.0)
11360    if verbose:
11361        print("loop: nearest neighbors init")
11362    neigh = NearestNeighbors(n_neighbors=n_neighbors, metric='minkowski')
11363    if verbose:
11364        print("loop: nearest neighbors fit")
11365    neigh.fit(data)
11366    d, idx = neigh.kneighbors(data, return_distance=True)
11367    if verbose:
11368        print("loop: probability")
11369    m = loop.LocalOutlierProbability(distance_matrix=d, neighbor_matrix=idx, n_neighbors=n_neighbors).fit()
11370    return m.local_outlier_probabilities[:]
11371
11372
11373def calculate_loop_scores(
11374    flattened_series,
11375    n_neighbors=20,
11376    n_features_sample=0.02,
11377    n_feature_repeats=5,
11378    seed=42,
11379    use_approx_knn=True,
11380    verbose=True,
11381):
11382    """
11383    Memory-efficient and robust LoOP score estimation with optional approximate KNN
11384    and averaging over multiple random feature subsets.
11385
11386    Parameters:
11387        flattened_series (np.ndarray): 2D array (n_samples x n_features)
11388        n_neighbors (int): Number of neighbors for LoOP
11389        n_features_sample (int or float): Number or fraction of features to sample
11390        n_feature_repeats (int): How many independent feature subsets to sample and average over
11391        seed (int): Random seed
11392        use_approx_knn (bool): Whether to use fast approximate KNN (via pynndescent)
11393        verbose (bool): Verbose output
11394
11395    Returns:
11396        np.ndarray: Averaged local outlier probabilities (length n_samples)
11397    """
11398    import numpy as np
11399    from sklearn.preprocessing import StandardScaler
11400    from PyNomaly import loop
11401
11402    # Optional approximate nearest neighbors
11403    try:
11404        from pynndescent import NNDescent
11405        has_nn_descent = True
11406    except ImportError:
11407        has_nn_descent = False
11408
11409    rng = np.random.default_rng(seed)
11410    X = np.nan_to_num(flattened_series, nan=0).astype(np.float32)
11411    n_samples, n_features = X.shape
11412
11413    # Handle feature sampling
11414    if isinstance(n_features_sample, float):
11415        if 0 < n_features_sample <= 1.0:
11416            n_features_sample = max(1, int(n_features_sample * n_features))
11417        else:
11418            raise ValueError("If float, n_features_sample must be in (0, 1].")
11419
11420    n_features_sample = min(n_features, n_features_sample)
11421
11422    if n_neighbors >= n_samples:
11423        n_neighbors = max(1, n_samples // 2)
11424
11425    if verbose:
11426        print(f"[LoOP] Input shape: {X.shape}")
11427        print(f"[LoOP] Sampling {n_features_sample} features per repeat, {n_feature_repeats} repeats")
11428        print(f"[LoOP] Using {n_neighbors} neighbors")
11429
11430    loop_scores = []
11431
11432    for rep in range(n_feature_repeats):
11433        feature_idx = rng.choice(n_features, n_features_sample, replace=False)
11434        X_sub = X[:, feature_idx]
11435
11436        scaler = StandardScaler(copy=False)
11437        X_sub = scaler.fit_transform(X_sub)
11438        X_sub = np.nan_to_num(X_sub, nan=0)
11439
11440        # Approximate or exact KNN
11441        if use_approx_knn and has_nn_descent and n_samples > 1000:
11442            if verbose:
11443                print(f"  [Rep {rep+1}] Using NNDescent (approximate KNN)")
11444            ann = NNDescent(X_sub, n_neighbors=n_neighbors, random_state=seed + rep)
11445            indices, dists = ann.neighbor_graph
11446        else:
11447            from sklearn.neighbors import NearestNeighbors
11448            if verbose:
11449                print(f"  [Rep {rep+1}] Using NearestNeighbors (exact KNN)")
11450            nn = NearestNeighbors(n_neighbors=n_neighbors)
11451            nn.fit(X_sub)
11452            dists, indices = nn.kneighbors(X_sub)
11453
11454        # LoOP score for this repeat
11455        model = loop.LocalOutlierProbability(
11456            distance_matrix=dists,
11457            neighbor_matrix=indices,
11458            n_neighbors=n_neighbors
11459        ).fit()
11460        loop_scores.append(model.local_outlier_probabilities[:])
11461
11462    # Average over repeats
11463    loop_scores = np.stack(loop_scores)
11464    loop_scores_mean = loop_scores.mean(axis=0)
11465
11466    if verbose:
11467        print(f"[LoOP] Averaged over {n_feature_repeats} feature subsets. Final shape: {loop_scores_mean.shape}")
11468
11469    return loop_scores_mean
11470
11471
11472
11473def score_fmri_censoring(cbfts, csf_seg, gm_seg, wm_seg ):
11474    """
11475    Process CBF time series to remove high-leverage points.
11476    Derived from the SCORE algorithm by Sudipto Dolui et. al.
11477
11478    Parameters:
11479    cbfts (ANTsImage): 4D ANTsImage of CBF time series.
11480    csf_seg (ANTsImage): CSF binary map.
11481    gm_seg (ANTsImage): Gray matter binary map.
11482    wm_seg (ANTsImage): WM binary map.
11483
11484    Returns:
11485    ANTsImage: Processed CBF time series.
11486    ndarray: Index of removed volumes.
11487    """
11488    
11489    n_gm_voxels = np.sum(gm_seg.numpy()) - 1
11490    n_wm_voxels = np.sum(wm_seg.numpy()) - 1
11491    n_csf_voxels = np.sum(csf_seg.numpy()) - 1
11492    mask1img = gm_seg + wm_seg + csf_seg
11493    mask1 = (mask1img==1).numpy()
11494    
11495    cbfts_np = cbfts.numpy()
11496    gmbool = (gm_seg==1).numpy()
11497    csfbool = (csf_seg==1).numpy()
11498    wmbool = (wm_seg==1).numpy()
11499    gm_cbf_ts = ants.timeseries_to_matrix( cbfts, gm_seg )
11500    gm_cbf_ts = np.squeeze(np.mean(gm_cbf_ts, axis=1))
11501    
11502    median_gm_cbf = np.median(gm_cbf_ts)
11503    mad_gm_cbf = np.median(np.abs(gm_cbf_ts - median_gm_cbf)) / 0.675
11504    indx = np.abs(gm_cbf_ts - median_gm_cbf) > (2.5 * mad_gm_cbf)
11505    
11506    # the spatial mean
11507    spatmeannp = np.mean(cbfts_np[:, :, :, ~indx], axis=3)
11508    spatmean = ants.from_numpy( spatmeannp )
11509    V = (
11510        n_gm_voxels * np.var(spatmeannp[gmbool])
11511        + n_wm_voxels * np.var(spatmeannp[wmbool])
11512        + n_csf_voxels * np.var(spatmeannp[csfbool])
11513    )
11514    V1 = math.inf
11515    ct=0
11516    while V < V1:
11517        ct=ct+1
11518        V1 = V
11519        CC = np.zeros(cbfts_np.shape[3])
11520        for s in range(cbfts_np.shape[3]):
11521            if indx[s]:
11522                continue
11523            tmp1 = ants.from_numpy( cbfts_np[:, :, :, s] )
11524            CC[s] = ants.image_similarity( spatmean, tmp1, metric_type='Correlation', fixed_mask=mask1img )
11525        inx = np.argmin(CC)
11526        indx[inx] = True
11527        spatmeannp = np.mean(cbfts_np[:, :, :, ~indx], axis=3)
11528        spatmean = ants.from_numpy( spatmeannp )
11529        V = (
11530          n_gm_voxels * np.var(spatmeannp[gmbool]) + 
11531          n_wm_voxels * np.var(spatmeannp[wmbool]) + 
11532          n_csf_voxels * np.var(spatmeannp[csfbool])
11533        )
11534    cbfts_recon = cbfts_np[:, :, :, ~indx]
11535    cbfts_recon = np.nan_to_num(cbfts_recon)
11536    cbfts_recon_ants = ants.from_numpy(cbfts_recon)
11537    cbfts_recon_ants = ants.copy_image_info(cbfts, cbfts_recon_ants)
11538    return cbfts_recon_ants, indx
11539
11540def loop_timeseries_censoring(x, threshold=0.5, mask=None, n_features_sample=0.02, seed=42, verbose=True):
11541    """
11542    Censor high leverage volumes from a time series using Local Outlier Probabilities (LoOP).
11543
11544    Parameters:
11545    x (ANTsImage): A 4D time series image.
11546    threshold (float): Threshold for determining high leverage volumes based on LoOP scores.
11547    mask (antsImage): restricts to a ROI
11548    n_features_sample (int/float): feature sample size default 0.01; if less than one then this is interpreted as a percentage of the total features otherwise it sets the number of features to be used
11549    seed (int): random seed
11550    verbose (bool)
11551
11552    Returns:
11553    tuple: A tuple containing the censored time series (ANTsImage) and the indices of the high leverage volumes.
11554    """
11555    import warnings
11556    if x.shape[3] < 20: # just a guess at what we need here ...
11557        warnings.warn("Warning: the time dimension is < 20 - too few samples for loop. just return the original data.")
11558        return x, []
11559    if mask is None:
11560        flattened_series = flatten_time_series(x.numpy())
11561    else:
11562        flattened_series = ants.timeseries_to_matrix( x, mask )
11563    if verbose:
11564        print("loop_timeseries_censoring: flattened")
11565    loop_scores = calculate_loop_scores(flattened_series, n_features_sample=n_features_sample, seed=seed, verbose=verbose )
11566    high_leverage_volumes = np.where(loop_scores > threshold)[0]
11567    if verbose:
11568        print("loop_timeseries_censoring: High Leverage Volumes:", high_leverage_volumes)
11569    new_asl = remove_volumes_from_timeseries(x, high_leverage_volumes)
11570    return new_asl, high_leverage_volumes
11571
11572
11573def novelty_detection_ee(df_train, df_test, contamination=0.05):
11574    """
11575    This function performs novelty detection using Elliptic Envelope.
11576
11577    Parameters:
11578
11579    - df_train (pandas dataframe): training data used to fit the model
11580
11581    - df_test (pandas dataframe): test data used to predict novelties
11582
11583    - contamination (float): parameter controlling the proportion of outliers in the data (default: 0.05)
11584
11585    Returns:
11586
11587    predictions (pandas series): predicted labels for the test data (1 for novelties, 0 for inliers)
11588    """
11589    import pandas as pd
11590    from sklearn.covariance import EllipticEnvelope
11591    # Fit the model on the training data
11592    clf = EllipticEnvelope(contamination=contamination,support_fraction=1)
11593    df_train[ df_train == math.inf ] = 0
11594    df_test[ df_test == math.inf ] = 0
11595    from sklearn.preprocessing import StandardScaler
11596    scaler = StandardScaler()
11597    scaler.fit(df_train)
11598    clf.fit(scaler.transform(df_train))
11599    predictions = clf.predict(scaler.transform(df_test))
11600    predictions[predictions==1]=0
11601    predictions[predictions==-1]=1
11602    if str(type(df_train))=="<class 'pandas.core.frame.DataFrame'>":
11603        return pd.Series(predictions, index=df_test.index)
11604    else:
11605        return pd.Series(predictions)
11606
11607
11608
11609def novelty_detection_svm(df_train, df_test, nu=0.05, kernel='rbf'):
11610    """
11611    This function performs novelty detection using One-Class SVM.
11612
11613    Parameters:
11614
11615    - df_train (pandas dataframe): training data used to fit the model
11616
11617    - df_test (pandas dataframe): test data used to predict novelties
11618
11619    - nu (float): parameter controlling the fraction of training errors and the fraction of support vectors (default: 0.05)
11620
11621    - kernel (str): kernel type used in the SVM algorithm (default: 'rbf')
11622
11623    Returns:
11624
11625    predictions (pandas series): predicted labels for the test data (1 for novelties, 0 for inliers)
11626    """
11627    from sklearn.svm import OneClassSVM
11628    # Fit the model on the training data
11629    df_train[ df_train == math.inf ] = 0
11630    df_test[ df_test == math.inf ] = 0
11631    clf = OneClassSVM(nu=nu, kernel=kernel)
11632    from sklearn.preprocessing import StandardScaler
11633    scaler = StandardScaler()
11634    scaler.fit(df_train)
11635    clf.fit(scaler.transform(df_train))
11636    predictions = clf.predict(scaler.transform(df_test))
11637    predictions[predictions==1]=0
11638    predictions[predictions==-1]=1
11639    if str(type(df_train))=="<class 'pandas.core.frame.DataFrame'>":
11640        return pd.Series(predictions, index=df_test.index)
11641    else:
11642        return pd.Series(predictions)
11643
11644
11645
11646def novelty_detection_lof(df_train, df_test, n_neighbors=20):
11647    """
11648    This function performs novelty detection using Local Outlier Factor (LOF).
11649
11650    Parameters:
11651
11652    - df_train (pandas dataframe): training data used to fit the model
11653
11654    - df_test (pandas dataframe): test data used to predict novelties
11655
11656    - n_neighbors (int): number of neighbors used to compute the LOF (default: 20)
11657
11658    Returns:
11659
11660    - predictions (pandas series): predicted labels for the test data (1 for novelties, 0 for inliers)
11661
11662    """
11663    from sklearn.neighbors import LocalOutlierFactor
11664    # Fit the model on the training data
11665    df_train[ df_train == math.inf ] = 0
11666    df_test[ df_test == math.inf ] = 0
11667    clf = LocalOutlierFactor(n_neighbors=n_neighbors, algorithm='auto',contamination='auto', novelty=True)
11668    from sklearn.preprocessing import StandardScaler
11669    scaler = StandardScaler()
11670    scaler.fit(df_train)
11671    clf.fit(scaler.transform(df_train))
11672    predictions = clf.predict(scaler.transform(df_test))
11673    predictions[predictions==1]=0
11674    predictions[predictions==-1]=1
11675    if str(type(df_train))=="<class 'pandas.core.frame.DataFrame'>":
11676        return pd.Series(predictions, index=df_test.index)
11677    else:
11678        return pd.Series(predictions)
11679
11680
11681def novelty_detection_loop(df_train, df_test, n_neighbors=20, distance_metric='minkowski'):
11682    """
11683    This function performs novelty detection using Local Outlier Factor (LOF).
11684
11685    Parameters:
11686
11687    - df_train (pandas dataframe): training data used to fit the model
11688
11689    - df_test (pandas dataframe): test data used to predict novelties
11690
11691    - n_neighbors (int): number of neighbors used to compute the LOOP (default: 20)
11692
11693    - distance_metric : default minkowski
11694
11695    Returns:
11696
11697    - predictions (pandas series): predicted labels for the test data (1 for novelties, 0 for inliers)
11698
11699    """
11700    from PyNomaly import loop
11701    from sklearn.neighbors import NearestNeighbors
11702    from sklearn.preprocessing import StandardScaler
11703    scaler = StandardScaler()
11704    scaler.fit(df_train)
11705    data = np.vstack( [scaler.transform(df_test),scaler.transform(df_train)])
11706    neigh = NearestNeighbors(n_neighbors=n_neighbors, metric=distance_metric)
11707    neigh.fit(data)
11708    d, idx = neigh.kneighbors(data, return_distance=True)
11709    m = loop.LocalOutlierProbability(distance_matrix=d, neighbor_matrix=idx, n_neighbors=n_neighbors).fit()
11710    return m.local_outlier_probabilities[range(df_test.shape[0])]
11711
11712
11713
11714def novelty_detection_quantile(df_train, df_test):
11715    """
11716    This function performs novelty detection using quantiles for each column.
11717
11718    Parameters:
11719
11720    - df_train (pandas dataframe): training data used to fit the model
11721
11722    - df_test (pandas dataframe): test data used to predict novelties
11723
11724    Returns:
11725
11726    - quantiles for the test sample at each column where values range in [0,1]
11727        and higher values mean the column is closer to the edge of the distribution
11728
11729    """
11730    myqs = df_test.copy()
11731    n = df_train.shape[0]
11732    df_trainkeys = df_train.keys()
11733    for k in range( df_train.shape[1] ):
11734        mykey = df_trainkeys[k]
11735        temp = (myqs[mykey][0] >  df_train[mykey]).sum() / n
11736        myqs[mykey] = abs( temp - 0.5 ) / 0.5
11737    return myqs
11738
11739
11740
11741def shorten_pymm_names(x):
11742    """
11743    Shortens pmymm names by applying a series of regex substitutions.
11744    
11745    Parameters:
11746    x (str): The input string to be shortened
11747    
11748    Returns:
11749    str: The shortened string
11750    """
11751    xx = x.lower()
11752    xx = re.sub("_", ".", xx)  # Replace underscores with periods
11753    xx = re.sub("\.\.", ".", xx, flags=re.I)  # Replace double dots with single dot
11754    # Apply the following regex substitutions in order
11755    xx = re.sub("sagittal.stratum.include.inferior.longitidinal.fasciculus.and.inferior.fronto.occipital.fasciculus.","ilf.and.ifo", xx, flags=re.I)
11756    xx = re.sub(r"sagittal.stratum.include.inferior.longitidinal.fasciculus.and.inferior.fronto.occipital.fasciculus.", "ilf.and.ifo", xx, flags=re.I)
11757    xx = re.sub(r".cres.stria.terminalis.can.not.be.resolved.with.current.resolution.", "", 
11758xx, flags=re.I)
11759    xx = re.sub("_", ".", xx)  # Replace underscores with periods
11760    xx = re.sub(r"longitudinal.fasciculus", "l.fasc", xx, flags=re.I)
11761    xx = re.sub(r"corona.radiata", "cor.rad", xx, flags=re.I)
11762    xx = re.sub("central", "cent", xx, flags=re.I)
11763    xx = re.sub(r"deep.cit168", "dp.", xx, flags=re.I)
11764    xx = re.sub("cit168", "", xx, flags=re.I)
11765    xx = re.sub(".include", "", xx, flags=re.I)
11766    xx = re.sub("mtg.sn", "", xx, flags=re.I)
11767    xx = re.sub("brainstem", ".bst", xx, flags=re.I)
11768    xx = re.sub(r"rsfmri.", "rsf.", xx, flags=re.I)
11769    xx = re.sub(r"dti.mean.fa.", "dti.fa.", xx, flags=re.I)
11770    xx = re.sub("perf.cbf.mean.", "cbf.", xx, flags=re.I)
11771    xx = re.sub(".jhu.icbm.labels.1mm", "", xx, flags=re.I)
11772    xx = re.sub(".include.optic.radiation.", "", xx, flags=re.I)
11773    xx = re.sub("\.\.", ".", xx, flags=re.I)  # Replace double dots with single dot
11774    xx = re.sub("\.\.", ".", xx, flags=re.I)  # Replace double dots with single dot
11775    xx = re.sub("cerebellar.peduncle", "cereb.ped", xx, flags=re.I)
11776    xx = re.sub(r"anterior.limb.of.internal.capsule", "ant.int.cap", xx, flags=re.I)
11777    xx = re.sub(r"posterior.limb.of.internal.capsule", "post.int.cap", xx, flags=re.I)
11778    xx = re.sub("t1hier.", "t1.", xx, flags=re.I)
11779    xx = re.sub("anterior", "ant", xx, flags=re.I)
11780    xx = re.sub("posterior", "post", xx, flags=re.I)
11781    xx = re.sub("inferior", "inf", xx, flags=re.I)
11782    xx = re.sub("superior", "sup", xx, flags=re.I)
11783    xx = re.sub(r"dktcortex", ".ctx", xx, flags=re.I)
11784    xx = re.sub(".lravg", "", xx, flags=re.I)
11785    xx = re.sub("dti.mean.fa", "dti.fa", xx, flags=re.I)
11786    xx = re.sub(r"retrolenticular.part.of.internal", "rent.int.cap", xx, flags=re.I)
11787    xx = re.sub(r"iculus.could.be.a.part.of.ant.internal.capsule", "", xx, flags=re.I)  # Twice
11788    xx = re.sub(".fronto.occipital.", ".frnt.occ.", xx, flags=re.I)
11789    xx = re.sub(r".longitidinal.fasciculus.", ".long.fasc.", xx, flags=re.I)  # Twice
11790    xx = re.sub(".external.capsule", ".ext.cap", xx, flags=re.I)
11791    xx = re.sub("of.internal.capsule", ".int.cap", xx, flags=re.I)
11792    xx = re.sub("fornix.cres.stria.terminalis", "fornix.", xx, flags=re.I)
11793    xx = re.sub("capsule", "", xx, flags=re.I)
11794    xx = re.sub("and.inf.frnt.occ.fasciculus.", "", xx, flags=re.I)
11795    xx = re.sub("crossing.tract.a.part.of.mcp.", "", xx, flags=re.I)
11796    return xx[:40]  # Truncate to first 40 characters
11797
11798
11799def shorten_pymm_names2(x, verbose=False ):
11800    """
11801    Shortens pmymm names by applying a series of regex substitutions.
11802
11803    Parameters:
11804    x (str): The input string to be shortened
11805
11806    verbose (bool): explain the patterns and replacements and their impact
11807
11808    Returns:
11809    str: The shortened string
11810    """
11811    # Define substitution patterns as tuples
11812    substitutions = [
11813        ("_", "."),  
11814        ("\.\.", "."),
11815        ("sagittal.stratum.include.inferior.longitidinal.fasciculus.and.inferior.fronto.occipital.fasciculus.","ilf.and.ifo"),
11816        (r"sagittal.stratum.include.inferior.longitidinal.fasciculus.and.inferior.fronto.occipital.fasciculus.", "ilf.and.ifo"),
11817        (r".cres.stria.terminalis.can.not.be.resolved.with.current.resolution.", ""),
11818        ("_", "."),
11819        (r"longitudinal.fasciculus", "l.fasc"),
11820        (r"corona.radiata", "cor.rad"),
11821        ("central", "cent"),
11822        (r"deep.cit168", "dp."),
11823        ("cit168", ""),
11824        (".include", ""),
11825        ("mtg.sn", ""),
11826        ("brainstem", ".bst"),
11827        (r"rsfmri.", "rsf."),
11828        (r"dti.mean.fa.", "dti.fa."),
11829        ("perf.cbf.mean.", "cbf."),
11830        (".jhu.icbm.labels.1mm", ""),
11831        (".include.optic.radiation.", ""),
11832        ("\.\.", "."),  # Replace double dots with single dot
11833        ("\.\.", "."),  # Replace double dots with single dot
11834        ("cerebellar.peduncle", "cereb.ped"),
11835        (r"anterior.limb.of.internal.capsule", "ant.int.cap"),
11836        (r"posterior.limb.of.internal.capsule", "post.int.cap"),
11837        ("t1hier.", "t1."),
11838        ("anterior", "ant"),
11839        ("posterior", "post"),
11840        ("inferior", "inf"),
11841        ("superior", "sup"),
11842        (r"dktcortex", ".ctx"),
11843        (".lravg", ""),
11844        ("dti.mean.fa", "dti.fa"),
11845        (r"retrolenticular.part.of.internal", "rent.int.cap"),
11846        (r"iculus.could.be.a.part.of.ant.internal.capsule", ""),  # Twice
11847        (".fronto.occipital.", ".frnt.occ."),
11848        (r".longitidinal.fasciculus.", ".long.fasc."),  # Twice
11849        (".external.capsule", ".ext.cap"),
11850        ("of.internal.capsule", ".int.cap"),
11851        ("fornix.cres.stria.terminalis", "fornix."),
11852        ("capsule", ""),
11853        ("and.inf.frnt.occ.fasciculus.", ""),
11854        ("crossing.tract.a.part.of.mcp.", "")
11855      ]
11856
11857    # Apply substitutions in order
11858    for pattern, replacement in substitutions:
11859        if verbose:
11860            print("Pre " + x + " pattern "+pattern + " repl " + replacement )
11861        x = re.sub(pattern, replacement, x.lower(), flags=re.IGNORECASE)
11862        if verbose:
11863            print("Post " + x)
11864
11865    return x[:40]  # Truncate to first 40 characters
11866
11867
11868def brainmap_figure(statistical_df, data_dictionary, output_prefix, brain_image, overlay_cmap='bwr', nslices=21, ncol=7, edge_image_dilation = 0, black_bg=True, axes = [0,1,2], fixed_overlay_range=None, crop=5, verbose=0 ):
11869    """
11870    Create figures based on statistical data and an underlying brain image.
11871
11872    Assumes both ~/.antspyt1w and ~/.antspymm data is available
11873
11874    Parameters:
11875    - statistical_df (pandas dataframe): with 2 columns named anat and values
11876        the anat column should have names that meet *partial matching* criterion 
11877        with respect to regions that are measured in antspymm.   value will be 
11878        the value to be displayed.   if two examples of a given region exist in 
11879        statistical_df, then the largest absolute value will be taken for display.
11880    - data_dictionary (pandas dataframe): antspymm data dictionary.
11881    - output_prefix (str): Prefix for the output figure filenames.
11882    - brain_image (antsImage): the brain image on which results will overlay.
11883    - overlay_cmap (str): see matplotlib
11884    - nslices (int): number of slices to show
11885    - ncol (int): number of columns to show
11886    - edge_image_dilation (int): integer greater than or equal to zero
11887    - black_bg (bool): boolean
11888    - axes (list): integer list typically [0,1,2] sagittal coronal axial
11889    - fixed_overlay_range (list): scalar pair will try to keep a constant cbar and will truncate the overlay at these min/max values
11890    - crop (int): crops the image to display by the extent of the overlay; larger values dilate the masks more.
11891    - verbose (bool): boolean
11892
11893    Returns:
11894    an image with values mapped to the associated regions
11895    """
11896    import re
11897
11898    def is_bst_region(filename):
11899        return filename[-4:] == '.bst'
11900
11901    # Read the statistical file
11902    zz = statistical_df 
11903    
11904    # Read the data dictionary from a CSV file
11905    mydict = data_dictionary
11906    mydict = mydict[~mydict['Measurement'].str.contains("tractography-based connectivity", na=False)]
11907    mydict2=mydict.copy()
11908    mydict2['tidynames']=mydict2['tidynames'].str.replace(".left","")
11909    mydict2['tidynames']=mydict2['tidynames'].str.replace(".right","")
11910
11911    statistical_df['anat'] = statistical_df['anat'].str.replace("_", ".", regex=True)
11912
11913    # Load image and process it
11914    edgeimg = ants.iMath(brain_image,"Normalize")
11915    if edge_image_dilation > 0:
11916        edgeimg = ants.iMath( edgeimg, "MD", edge_image_dilation)
11917
11918    # Define lists and data frames
11919    postfix = ['bf', 'cit168lab', 'mtl', 'cerebellum', 'dkt_cortex','brainstem','JHU_wm','yeo']
11920    atlas = ['BF', 'CIT168', 'MTL', 'TustisonCobra', 'desikan-killiany-tourville','brainstem','JHU_wm','yeo']
11921    postdesc = ['nbm3CH13', 'CIT168_Reinf_Learn_v1_label_descriptions_pad', 'mtl_description', 'cerebellum', 'dkt','CIT168_T1w_700um_pad_adni_brainstem','FA_JHU_labels_edited','ppmi_template_500Parcels_Yeo2011_17Networks_2023_homotopic']
11922    templateprefix = '~/.antspymm/PPMI_template0_'
11923    # Iterate through columns and create figures
11924    col2viz = 'values'
11925    if True:
11926        anattoshow = zz['anat'].unique()
11927        if verbose > 0:
11928            print(col2viz)
11929            print(anattoshow)
11930        # Rest of your code for figure creation goes here...
11931        addem = edgeimg * 0
11932        for k in range(len(anattoshow)):
11933            if verbose > 0 :
11934                print(str(k) +  " " + anattoshow[k]  )
11935            mysub = zz[zz['anat'].str.contains(anattoshow[k])]
11936            anatsear=shorten_pymm_names( anattoshow[k] )
11937            anatsear=re.sub(r'[()]', '.', anatsear )
11938            anatsear=re.sub(r'\.\.', '.', anatsear )
11939            anatsear=re.sub("dti.mean.md.snc","md.snc",anatsear)
11940            anatsear=re.sub("dti.mean.fa.snc","fa.snc",anatsear)
11941            anatsear=re.sub("dti.mean.md.snr","md.snr",anatsear)
11942            anatsear=re.sub("dti.mean.fa.snr","fa.snr",anatsear)
11943            anatsear=re.sub("dti.mean.md.","",anatsear)
11944            anatsear=re.sub("dti.mean.fa.","",anatsear)
11945            anatsear=re.sub("dti.md.","",anatsear)
11946            anatsear=re.sub("dti.fa.","",anatsear)
11947            anatsear=re.sub("dti.md","",anatsear)
11948            anatsear=re.sub("dti.fa","",anatsear)
11949            anatsear=re.sub("cbf.","",anatsear)
11950            anatsear=re.sub("rsfmri.fcnxpro122.","",anatsear)
11951            anatsear=re.sub("rsfmri.fcnxpro129.","",anatsear)
11952            anatsear=re.sub("rsfmri.fcnxpro134.","",anatsear)
11953            anatsear=re.sub("t1hier.vollravg","",anatsear)
11954            anatsear=re.sub("t1hier.volasym","",anatsear)
11955            anatsear=re.sub("t1hier.thkasym","",anatsear)
11956            anatsear=re.sub("t1hier.areaasym","",anatsear)
11957            anatsear=re.sub("t1hier.vol.","",anatsear)
11958            anatsear=re.sub("t1hier.thk.","",anatsear)
11959            anatsear=re.sub("t1hier.area.","",anatsear)
11960            anatsear=re.sub("t1.volasym","",anatsear)
11961            anatsear=re.sub("t1.thkasym","",anatsear)
11962            anatsear=re.sub("t1.areaasym","",anatsear)
11963            anatsear=re.sub("t1.vol.","",anatsear)
11964            anatsear=re.sub("t1.thk.","",anatsear)
11965            anatsear=re.sub("t1.area.","",anatsear)
11966            anatsear=re.sub("asymdp.","",anatsear)
11967            anatsear=re.sub("asym.","",anatsear)
11968            anatsear=re.sub("asym","",anatsear)
11969            anatsear=re.sub("lravg.","",anatsear)
11970            anatsear=re.sub("lravg","",anatsear)
11971            anatsear=re.sub("dktcortex","",anatsear)
11972            anatsear=re.sub("dktregions","",anatsear)
11973            anatsear=re.sub("_",".",anatsear)
11974            anatsear=re.sub("superior","sup",anatsear)
11975            anatsear=re.sub("cerebellum","",anatsear)
11976            anatsear=re.sub("brainstem","",anatsear)
11977            anatsear=re.sub("t.limb.int","t.int",anatsear)
11978            anatsear=re.sub("paracentral","paracent",anatsear)
11979            anatsear=re.sub("precentral","precent",anatsear)
11980            anatsear=re.sub("postcentral","postcent",anatsear)
11981            anatsear=re.sub("sup.cerebellar.peduncle","sup.cereb.ped",anatsear)
11982            anatsear=re.sub("inferior.cerebellar.peduncle","inf.cereb.ped",anatsear)
11983            anatsear=re.sub(".crossing.tract.a.part.of.mcp.","",anatsear)
11984            anatsear=re.sub(".crossing.tract.a.part.of.","",anatsear)
11985            anatsear=re.sub(".column.and.body.of.fornix.","",anatsear)
11986            anatsear=re.sub("fronto.occipital.fasciculus.could.be.a.part.of.ant.internal.capsule","frnt.occ",anatsear)
11987            anatsear=re.sub("inferior.fronto.occipital.fasciculus.could.be.a.part.of.anterior.internal.capsule","inf.frnt.occ",anatsear)
11988            anatsear=re.sub("fornix.cres.stria.terminalis.can.not.be.resolved.with.current.resolution","fornix.column.and.body.of.fornix",anatsear)
11989            anatsear=re.sub("external.capsule","ext.cap",anatsear)
11990            anatsear=re.sub(".jhu.icbm.labels.1mm","",anatsear)
11991            anatsear=re.sub("dp.",".",anatsear)
11992            anatsear=re.sub(".mtg.sn.snc.",".snc.",anatsear)
11993            anatsear=re.sub(".mtg.sn.snr.",".snr.",anatsear)
11994            anatsear=re.sub("mtg.sn.snc.",".snc.",anatsear)
11995            anatsear=re.sub("mtg.sn.snr.",".snr.",anatsear)
11996            anatsear=re.sub("mtg.sn.snc",".snc.",anatsear)
11997            anatsear=re.sub("mtg.sn.snr",".snr.",anatsear)
11998            anatsear=re.sub("anterior.","ant.",anatsear)
11999            anatsear=re.sub("rsf.","",anatsear)
12000            anatsear=re.sub("fcnxpro122.","",anatsear)
12001            anatsear=re.sub("fcnxpro129.","",anatsear)
12002            anatsear=re.sub("fcnxpro134.","",anatsear)
12003            anatsear=re.sub("ant.corona.radiata","ant.cor.rad",anatsear)
12004            anatsear=re.sub("sup.corona.radiata","sup.cor.rad",anatsear)
12005            anatsear=re.sub("posterior.thalamic.radiation.include.optic.radiation","post.thalamic.radiation",anatsear)
12006            anatsear=re.sub("retrolenticular.part.of.internal.capsule","rent.int.cap",anatsear)
12007            anatsear=re.sub("post.limb.of.internal.capsule","post.int.cap",anatsear)
12008            anatsear=re.sub("ant.limb.of.internal.capsule","ant.int.cap",anatsear)
12009            anatsear=re.sub("sagittal.stratum.include.inferior.longitidinal.fasciculus.and.inferior.fronto.occipital.fasciculus","ilf.and.ifo",anatsear)
12010            anatsear=re.sub("post.thalamic.radiation.optic.rad","post.thalamic.radiation",anatsear)
12011            atlassearch = mydict['tidynames'].str.contains(anatsear)
12012            if atlassearch.sum() == 0:
12013                atlassearch = mydict2['tidynames'].str.contains(anatsear)
12014            if verbose > 0 :
12015                print( " anatsear " + anatsear + " atlassearch " )
12016            if atlassearch.sum() > 0:
12017                whichatlas = mydict[atlassearch]['Atlas'].iloc[0]
12018                oglabelname = mydict[atlassearch]['Label'].iloc[0]
12019                oglabelname=re.sub("_",".",oglabelname)
12020                oglabelname=re.sub(r'\.\.','.',oglabelname)
12021            else:
12022                print(anatsear)
12023                oglabelname='unknown'
12024                whichatlas=None
12025            if verbose > 0:
12026                print("oglabelname " + oglabelname + " whichatlas " + str(whichatlas) )
12027            vals2viz = mysub[col2viz].agg(['min', 'max'])
12028            vals2viz = vals2viz[abs(vals2viz).idxmax()]
12029            myext = None
12030            if anatsear == 'cingulum.hippocampus':
12031                myext = 'JHU_wm'
12032            elif 'dktcortex' in anattoshow[k] or whichatlas == 'desikan-killiany-tourville' or 'dtkregions' in anattoshow[k]  :
12033                myext = 'dkt_cortex'
12034            elif ('cit168' in anattoshow[k] or whichatlas == 'CIT168') and not 'brainstem' in anattoshow[k] and not is_bst_region(anatsear):
12035                myext = 'cit168lab'
12036            elif 'mtl' in anattoshow[k]:
12037                myext = 'mtl'
12038                oglabelname=re.sub('mtl', '',anatsear)
12039            elif 'cerebellum' in anattoshow[k]:
12040                myext = 'cerebellum'
12041                oglabelname=re.sub('cerebellum', '',anatsear)
12042                oglabelname=re.sub('t1.vo','',oglabelname)
12043                # oglabelname=oglabelname[2:]
12044            elif 'brainstem' in anattoshow[k] or is_bst_region(anatsear):
12045                myext = 'brainstem'
12046            elif any(item in anattoshow[k] for item in ['nbm', 'bf']):
12047                myext = 'bf'
12048                oglabelname=re.sub('bf', '',oglabelname)
12049#                oglabelname=re.sub(r'\.', '_',anatsear)
12050            elif whichatlas == 'johns hopkins white matter':
12051                myext = 'JHU_wm'
12052            elif whichatlas == 'desikan-killiany-tourville':
12053                myext = 'dkt_cortex'
12054            elif whichatlas == 'CIT168':
12055                myext = 'cit168lab'
12056            elif whichatlas == 'BF':
12057                myext = 'bf'
12058                oglabelname=re.sub('bf', '',oglabelname)
12059            elif whichatlas == 'yeo_homotopic':
12060                myext = 'yeo'
12061            if myext is None and verbose > 0 :
12062                if whichatlas is None:
12063                    whichatlas='None'
12064                if anattoshow[k] is None:
12065                    anattoshow[k]='None'
12066                print( "MYEXT " + anattoshow[k] + ' unfound ' + whichatlas )
12067            else:
12068                if verbose > 0 :
12069                    print( "MYEXT " + myext )
12070
12071            if myext == 'cit168lab':
12072                oglabelname=re.sub("cit168","",oglabelname)
12073            
12074            for j in postfix:
12075                if j == "dkt_cortex":
12076                    j = 'dktcortex'
12077                if j == "deep_cit168lab":
12078                    j = 'deep_cit168'
12079                anattoshow[k] = anattoshow[k].replace(j, "")
12080            if verbose > 0:
12081                print( anattoshow[k] + " " + str( vals2viz ) )
12082            correctdescript = postdesc[postfix.index(myext)]
12083            locfilename =  templateprefix + myext + '.nii.gz'
12084            if verbose > 0:
12085                print( locfilename )
12086            if myext == 'yeo':
12087                oglabelname=oglabelname.lower()
12088                oglabelname=re.sub("rsfmri_fcnxpro122_","",oglabelname)
12089                oglabelname=re.sub("rsfmri_fcnxpro129_","",oglabelname)
12090                oglabelname=re.sub("rsfmri_fcnxpro134_","",oglabelname)
12091                oglabelname=re.sub("rsfmri.fcnxpro122.","",oglabelname)
12092                oglabelname=re.sub("rsfmri.fcnxpro129.","",oglabelname)
12093                oglabelname=re.sub("rsfmri.fcnxpro134.","",oglabelname)
12094                oglabelname=re.sub("_",".",oglabelname)
12095                locfilename = "~/.antspymm/ppmi_template_500Parcels_Yeo2011_17Networks_2023_homotopic.nii.gz"
12096                atlasDescript = pd.read_csv(f"~/.antspymm/{correctdescript}.csv")
12097                atlasDescript.rename(columns={'SystemName': 'Description'}, inplace=True)
12098                atlasDescript.rename(columns={'ROI': 'Label'}, inplace=True)
12099                atlasDescript['Description'] = atlasDescript['Description'].str.lower()
12100            else:
12101                atlasDescript = pd.read_csv(f"~/.antspyt1w/{correctdescript}.csv")
12102                atlasDescript['Description'] = atlasDescript['Description'].str.lower()
12103                atlasDescript['Description'] = atlasDescript['Description'].str.replace(" ", "_")
12104                atlasDescript['Description'] = atlasDescript['Description'].str.replace("_left_", "_")
12105                atlasDescript['Description'] = atlasDescript['Description'].str.replace("_right_", "_")
12106                atlasDescript['Description'] = atlasDescript['Description'].str.replace("_left", "")
12107                atlasDescript['Description'] = atlasDescript['Description'].str.replace("_right", "")
12108                atlasDescript['Description'] = atlasDescript['Description'].str.replace("left_", "")
12109                atlasDescript['Description'] = atlasDescript['Description'].str.replace("right_", "")
12110                atlasDescript['Description'] = atlasDescript['Description'].str.replace("/",".")
12111                atlasDescript['Description'] = atlasDescript['Description'].str.replace("_",".")
12112                atlasDescript['Description'] = atlasDescript['Description'].str.replace(r'[()]', '', regex=True)
12113                atlasDescript['Description'] = atlasDescript['Description'].str.replace(r'\.\.', '.')
12114                if myext == 'JHU_wm':
12115                    atlasDescript['Description'] = atlasDescript['Description'].str.replace("-", ".")
12116                    atlasDescript['Description'] = atlasDescript['Description'].str.replace("jhu.icbm.labels.1mm", "")
12117                    atlasDescript['Description'] = atlasDescript['Description'].str.replace("fronto-occipital", "frnt.occ")
12118                    atlasDescript['Description'] = atlasDescript['Description'].str.replace("superior", "sup")
12119                    atlasDescript['Description'] = atlasDescript['Description'].str.replace("fa-", "")
12120                    atlasDescript['Description'] = atlasDescript['Description'].str.replace("-left-", "")
12121                    atlasDescript['Description'] = atlasDescript['Description'].str.replace("-right-", "")
12122                if myext == 'cerebellum':
12123                    atlasDescript['Description'] = atlasDescript['Description'].str.replace("l_", "")
12124                    atlasDescript['Description'] = atlasDescript['Description'].str.replace("r_", "")
12125                    atlasDescript['Description'] = atlasDescript['Description'].str.replace("l.", "")
12126                    atlasDescript['Description'] = atlasDescript['Description'].str.replace("r.", "")
12127                    atlasDescript['Description'] = atlasDescript['Description'].str.replace("_",".")
12128
12129            if verbose > 0:
12130                print( atlasDescript )
12131            oglabelname = oglabelname.lower()
12132            oglabelname = re.sub(" ", "_",oglabelname)
12133            oglabelname = re.sub("_left_", "_",oglabelname)
12134            oglabelname = re.sub("_right_", "_",oglabelname)
12135            oglabelname = re.sub("_left", "",oglabelname)
12136            oglabelname = re.sub("_right", "",oglabelname)
12137            oglabelname = re.sub("t1hier_vol_", "",oglabelname)
12138            oglabelname = re.sub("t1hier_area_", "",oglabelname)
12139            oglabelname = re.sub("t1hier_thk_", "",oglabelname)
12140            oglabelname = re.sub("dktregions", "",oglabelname)
12141            oglabelname = re.sub("dktcortex", "",oglabelname)
12142
12143            oglabelname = re.sub(" ", ".",oglabelname)
12144            oglabelname = re.sub(".left.", ".",oglabelname)
12145            oglabelname = re.sub(".right.", ".",oglabelname)
12146            oglabelname = re.sub(".left", "",oglabelname)
12147            oglabelname = re.sub(".right", "",oglabelname)
12148            oglabelname = re.sub("t1hier.vol.", "",oglabelname)
12149            oglabelname = re.sub("t1hier.area.", "",oglabelname)
12150            oglabelname = re.sub("t1hier.thk.", "",oglabelname)
12151            oglabelname = re.sub("dktregions", "",oglabelname)
12152            oglabelname = re.sub("dktcortex", "",oglabelname)
12153            oglabelname=re.sub("brainstem","",oglabelname)
12154            if myext == 'JHU_wm':
12155                oglabelname = re.sub("dti_mean_fa.", "",oglabelname)
12156                oglabelname = re.sub("dti_mean_md.", "",oglabelname)
12157                oglabelname = re.sub("dti.mean.fa.", "",oglabelname)
12158                oglabelname = re.sub("dti.mean.md.", "",oglabelname)
12159                oglabelname = re.sub(".left.", "",oglabelname)
12160                oglabelname = re.sub(".right.", "",oglabelname)
12161                oglabelname = re.sub(".lravg.", "",oglabelname)
12162                oglabelname = re.sub(".asym.", "",oglabelname)
12163                oglabelname = re.sub(".jhu.icbm.labels.1mm", "",oglabelname)
12164                oglabelname = re.sub("superior", "sup",oglabelname)
12165
12166            if verbose > 0:
12167                print("oglabelname " + oglabelname )
12168
12169            if myext == 'cerebellum':
12170                if not atlasDescript.empty and 'Description' in atlasDescript.columns:
12171                    atlasDescript['Description'] = atlasDescript['Description'].str.replace("l_", "")
12172                    atlasDescript['Description'] = atlasDescript['Description'].str.replace("r_", "")
12173                    oglabelname=re.sub("ravg","",oglabelname)
12174                    oglabelname=re.sub("lavg","",oglabelname)
12175                    whichindex = atlasDescript.index[atlasDescript['Description'] == oglabelname].values
12176                else:
12177                    if atlasDescript.empty:
12178                        print("The DataFrame 'atlasDescript' is empty.")
12179                    if 'Description' not in atlasDescript.columns:
12180                        print("The column 'Description' does not exist in 'atlasDescript'.")
12181            else:
12182                whichindex = atlasDescript.index[atlasDescript['Description'].str.contains(oglabelname)]
12183
12184            if type(whichindex) is np.int64:
12185                labelnums = atlasDescript.loc[whichindex, 'Label']
12186            else:
12187                labelnums = list(atlasDescript.loc[whichindex, 'Label'])
12188
12189            if myext == 'yeo':
12190                parts = re.findall(r'\D+', oglabelname)
12191                oglabelname = [part.replace('_', '') for part in parts if part.replace('_', '')]
12192                oglabelname = [part.replace('.', '') for part in parts if part.replace('.', '')]
12193                filtered_df = atlasDescript[atlasDescript['Description'].isin(oglabelname)]
12194                labelnums = filtered_df['Label'].tolist()
12195
12196            if not isinstance(labelnums, list):
12197                labelnums=[labelnums]
12198            addemiszero = ants.threshold_image(addem, 0, 0)
12199            temp = ants.image_read(locfilename)
12200            temp = ants.mask_image(temp, temp, level=labelnums, binarize=True)
12201            if verbose > 0:
12202                print("DEBUG")
12203                print(  temp.sum() ) 
12204                print( labelnums )
12205            temp[temp == 1] = (vals2viz)
12206            temp[addemiszero == 0] = 0
12207            addem = addem + temp
12208
12209        if verbose > 0:
12210            print('Done Adding')
12211        for axx in axes:
12212            figfn=output_prefix+f"fig{col2viz}ax{axx}_py.jpg"
12213            if crop > 0:
12214                cmask = ants.threshold_image( addem,1e-5, 1e9 ).iMath("MD",crop) + ants.threshold_image( addem,-1e9, -1e-5 ).iMath("MD",crop)
12215                addemC = ants.crop_image( addem, cmask )
12216                edgeimgC = ants.crop_image( edgeimg, cmask )
12217            else:
12218                addemC = addem
12219                edgeimgC = edgeimg
12220            if fixed_overlay_range is not None:
12221                addemC[0:3,0:3,0:3]=fixed_overlay_range[0]
12222                addemC[4:7,4:7,4:7]=fixed_overlay_range[1]
12223                addemC[ addemC <= fixed_overlay_range[0] ] = 0 # fixed_overlay_range[0]
12224                addemC[ addemC >= fixed_overlay_range[1] ] = fixed_overlay_range[1]
12225            ants.plot(edgeimgC, addemC, axis=axx, nslices=nslices, ncol=ncol,       
12226                overlay_cmap=overlay_cmap, resample=False, overlay_alpha=1.0,
12227                filename=figfn, cbar=axx==axes[0], crop=True, black_bg=black_bg )
12228        if verbose > 0:
12229            print(f"{col2viz} done")
12230    if verbose:
12231        print("DONE brain map figures")
12232    return addem
12233
12234def filter_df(indf, myprefix):
12235    """
12236    Process and filter a pandas DataFrame, removing certain columns, 
12237    filtering based on data types, computing the mean of numeric columns, 
12238    and adding a prefix to column names.
12239
12240    Parameters:
12241    indf (pandas.DataFrame): The input DataFrame to be processed.
12242    myprefix (str): A string prefix to be added to the column names 
12243                    of the processed DataFrame.
12244
12245    Steps:
12246    1. Removes columns with names containing 'Unnamed'.
12247    2. If the DataFrame has no rows, it returns the empty DataFrame.
12248    3. Filters out columns based on the type of the first element, 
12249       keeping those that are of type `object`, `int`, or `float`.
12250    4. Removes columns that are of `object` dtype.
12251    5. Calculates the mean of the remaining columns, skipping NaN values.
12252    6. Adds the specified `myprefix` to the column names.
12253
12254    Returns:
12255    pandas.DataFrame: A transformed DataFrame with a single row containing 
12256                      the mean values of the filtered columns, and with 
12257                      column names prefixed as specified.
12258    """
12259    indf = indf.loc[:, ~indf.columns.str.contains('Unnamed*', na=False, regex=True)]
12260    if indf.shape[0] == 0:
12261        return indf
12262    nums = [isinstance(indf[col].iloc[0], (object, int, float)) for col in indf.columns]
12263    indf = indf.loc[:, nums]
12264    indf = indf.loc[:, indf.dtypes != 'object']
12265    indf = pd.DataFrame(indf.mean(axis=0, skipna=True)).T
12266    indf = indf.add_prefix(myprefix)
12267    return indf
12268
12269
12270def aggregate_antspymm_results(input_csv, subject_col='subjectID', date_col='date', image_col='imageID', date_column='ses-1', base_path="./Processed/ANTsExpArt/", hiervariable='T1wHierarchical', valid_modalities=None, verbose=False ):
12271    """
12272    Aggregate ANTsPyMM results from the specified CSV file and save the aggregated results to a new CSV file.
12273
12274    Parameters:
12275    - input_csv (str): File path of the input CSV file containing ANTsPyMM QC results averaged and with outlier measurements.
12276    - subject_col (str): Name of the column to store subject IDs.
12277    - date_col (str): Name of the column to store date information.
12278    - image_col (str): Name of the column to store image IDs.
12279    - date_column (str): Name of the column representing the date information.
12280    - base_path (str): Base path for search paths. Defaults to "./Processed/ANTsExpArt/".
12281    - hiervariable (str) : the string variable denoting the Hierarchical output
12282    - valid_modalities (str array) : identifies for each modality; if None will be replaced by get_valid_modalities(long=True)
12283    - verbose : boolean
12284
12285    Note:
12286    This function is tested under limited circumstances. Use with caution.
12287
12288    Example usage:
12289    agg_df = aggregate_antspymm_results("qcdfaol.csv", subject_col='subjectID', date_col='date', image_col='imageID', date_column='ses-1', base_path="./Your/Custom/Path/")
12290
12291    Author:
12292    Avants and ChatGPT
12293    """
12294    import pandas as pd
12295    import numpy as np
12296    from glob import glob
12297
12298    def myread_csv(x, cnms):
12299        """
12300        Reads a CSV file and returns a DataFrame excluding specified columns.
12301
12302        Parameters:
12303        - x (str): File path of the input CSV file describing the blind QC output
12304        - cnms (list): List of column names to exclude from the DataFrame.
12305
12306        Returns:
12307        pd.DataFrame: DataFrame with specified columns excluded.
12308        """
12309        df = pd.read_csv(x)
12310        return df.loc[:, ~df.columns.isin(cnms)]
12311
12312    import warnings
12313    # Warning message for untested function
12314    warnings.warn("Warning: This function is not well tested. Use with caution.")
12315
12316    if valid_modalities is None:
12317        valid_modalities = get_valid_modalities('long')
12318
12319    # Read the input CSV file
12320    df = pd.read_csv(input_csv)
12321
12322    # Filter rows where modality is 'T1w'
12323    df = df[df['modality'] == 'T1w']
12324    badnames = get_names_from_data_frame( ['Unnamed'], df )
12325    df=df.drop(badnames, axis=1)
12326
12327    # Add new columns for subject ID, date, and image ID
12328    df[subject_col] = np.nan
12329    df[date_col] = date_column
12330    df[image_col] = np.nan
12331    df = df.astype({subject_col: str, date_col: str, image_col: str })
12332
12333#    if verbose:
12334#        print( df.shape )
12335#        print( df.dtypes )
12336
12337    # prefilter df for data that exists
12338    keep = np.tile( False, df.shape[0] )
12339    for x in range(df.shape[0]):
12340        temp = df['filename'].iloc[x].split("_")
12341        # Generalized search paths
12342        path_template = f"{base_path}{temp[0]}/{date_column}/*/*/*"
12343        hierfn = sorted(glob( path_template + "-" + hiervariable + "-*wide.csv" ) )
12344        if len( hierfn ) > 0:
12345            keep[x]=True
12346
12347    
12348    df=df[keep]
12349    
12350    if verbose:
12351        print( "original input had shape " + str( df.shape[0] ) + " (T1 only) and we find " + str( (keep).sum() ) + " with hierarchical output defined by variable: " + hiervariable )
12352        print( df.shape )
12353
12354    myct = 0
12355    for x in range( df.shape[0]):
12356        if verbose:
12357            print(f"{x}...")
12358        locind = df.index[x]
12359        temp = df['filename'].iloc[x].split("_")
12360        if verbose:
12361            print( temp )
12362        df[subject_col].iloc[x]=temp[0]
12363        df[date_col].iloc[x]=date_column
12364        df[image_col].iloc[x]=temp[1]
12365
12366        # Generalized search paths
12367        path_template = f"{base_path}{temp[0]}/{date_column}/*/*/*"
12368        if verbose:
12369            print(path_template)
12370        hierfn = sorted(glob( path_template + "-" + hiervariable + "-*wide.csv" ) )
12371        if len( hierfn ) > 0:
12372            hdf=t1df=dtdf=rsdf=perfdf=nmdf=flairdf=None
12373            if verbose:
12374                print(hierfn)
12375            hdf = pd.read_csv(hierfn[0])
12376            badnames = get_names_from_data_frame( ['Unnamed'], hdf )
12377            hdf=hdf.drop(badnames, axis=1)
12378            nums = [isinstance(hdf[col].iloc[0], (int, float)) for col in hdf.columns]
12379            corenames = list(np.array(hdf.columns)[nums])
12380            hdf.loc[:, nums] = hdf.loc[:, nums].add_prefix("T1Hier_")
12381            myct = myct + 1
12382            dflist = [hdf]
12383
12384            for mymod in valid_modalities:
12385                t1wfn = sorted(glob( path_template+ "-" + mymod + "-*wide.csv" ) )
12386                if len( t1wfn ) > 0 :
12387                    if verbose:
12388                        print(t1wfn)
12389                    t1df = myread_csv(t1wfn[0], corenames)
12390                    t1df = filter_df( t1df, mymod+'_')
12391                    dflist = dflist + [t1df]
12392                
12393            hdf = pd.concat( dflist, axis=1, ignore_index=False )
12394            if verbose:
12395                print( df.loc[locind,'filename'] )
12396            if myct == 1:
12397                subdf = df.iloc[[x]]
12398                hdf.index = subdf.index.copy()
12399                df = pd.concat( [df,hdf], axis=1, ignore_index=False )
12400            else:
12401                commcols = list(set(hdf.columns).intersection(df.columns))
12402                df.loc[locind, commcols] = hdf.loc[0, commcols]
12403    badnames = get_names_from_data_frame( ['Unnamed'], df )
12404    df=df.drop(badnames, axis=1)
12405    return( df )
12406
12407def find_most_recent_file(file_list):
12408    """
12409    Finds and returns the most recently modified file from a list of file paths.
12410    
12411    Parameters:
12412    - file_list: A list of strings, where each string is a path to a file.
12413    
12414    Returns:
12415    - The path to the most recently modified file in the list, or None if the list is empty or contains no valid files.
12416    """
12417    # Filter out items that are not files or do not exist
12418    valid_files = [f for f in file_list if os.path.isfile(f)]
12419    
12420    # Check if the filtered list is not empty
12421    if valid_files:
12422        # Find the file with the latest modification time
12423        most_recent_file = max(valid_files, key=os.path.getmtime)
12424        return [most_recent_file]
12425    else:
12426        return None
12427    
12428def aggregate_antspymm_results_sdf(
12429    study_df, 
12430    project_col='projectID',
12431    subject_col='subjectID', 
12432    date_col='date', 
12433    image_col='imageID', 
12434    base_path="./", 
12435    hiervariable='T1wHierarchical', 
12436    splitsep='-',
12437    idsep='-',
12438    wild_card_modality_id=False,
12439    second_split=False,
12440    verbose=False ):
12441    """
12442    Aggregate ANTsPyMM results from the specified study data frame and store the aggregated results in a new data frame.  This assumes data is organized on disk 
12443    as follows:  rootdir/projectID/subjectID/date/outputid/imageid/ where 
12444    outputid is modality-specific and created by ANTsPyMM processing.
12445
12446    Parameters:
12447    - study_df (pandas df): pandas data frame, output of generate_mm_dataframe.
12448    - project_col (str): Name of the column that stores the project ID
12449    - subject_col (str): Name of the column to store subject IDs.
12450    - date_col (str): Name of the column to store date information.
12451    - image_col (str): Name of the column to store image IDs.
12452    - base_path (str): Base path for searching for processing outputs of ANTsPyMM.
12453    - hiervariable (str) : the string variable denoting the Hierarchical output
12454    - splitsep (str):  the separator used to split the filename
12455    - idsep (str): the separator used to partition subjectid date and imageid 
12456        for example, if idsep is - then we have subjectid-date-imageid
12457    - wild_card_modality_id (bool): keep if False for safer execution
12458    - second_split (bool): this is a hack that will split the imageID by . and keep the first part of the split; may be needed when the input filenames contain .
12459    - verbose : boolean
12460
12461    Note:
12462    This function is tested under limited circumstances. Use with caution.
12463    One particular gotcha is if the imageID is stored as a numeric value in the dataframe 
12464    but is meant to be a string.  E.g. '000' (string) would be interpreted as 0 in the 
12465    file name glob.  This would miss the extant (on disk) csv.
12466
12467    Example usage:
12468    agg_df = aggregate_antspymm_results_sdf( studydf, subject_col='subjectID', date_col='date', image_col='imageID', base_path="./Your/Custom/Path/")
12469
12470    Author:
12471    Avants and ChatGPT
12472    """
12473    import pandas as pd
12474    import numpy as np
12475    from glob import glob
12476
12477    def progress_reporter(current_step, total_steps, width=50):
12478        # Calculate the proportion of progress
12479        progress = current_step / total_steps
12480        # Calculate the number of 'filled' characters in the progress bar
12481        filled_length = int(width * progress)
12482        # Create the progress bar string
12483        bar = '█' * filled_length + '-' * (width - filled_length)
12484        # Print the progress bar with percentage
12485        print(f'\rProgress: |{bar}| {int(100 * progress)}%', end='\r')
12486        # Print a new line when the progress is complete
12487        if current_step == total_steps:
12488            print()
12489
12490    def myread_csv(x, cnms):
12491        """
12492        Reads a CSV file and returns a DataFrame excluding specified columns.
12493
12494        Parameters:
12495        - x (str): File path of the input CSV file describing the blind QC output
12496        - cnms (list): List of column names to exclude from the DataFrame.
12497
12498        Returns:
12499        pd.DataFrame: DataFrame with specified columns excluded.
12500        """
12501        df = pd.read_csv(x)
12502        return df.loc[:, ~df.columns.isin(cnms)]
12503
12504    import warnings
12505    # Warning message for untested function
12506    warnings.warn("Warning: This function is not well tested. Use with caution.")
12507
12508    vmoddict = {}
12509    # Add key-value pairs
12510    vmoddict['imageID'] = 'T1w'
12511    vmoddict['flairid'] = 'T2Flair'
12512    vmoddict['perfid'] = 'perf'
12513    vmoddict['pet3did'] = 'pet3d'
12514    vmoddict['rsfid1'] = 'rsfMRI'
12515#    vmoddict['rsfid2'] = 'rsfMRI'
12516    vmoddict['dtid1'] = 'DTI'
12517#    vmoddict['dtid2'] = 'DTI'
12518    vmoddict['nmid1'] = 'NM2DMT'
12519#    vmoddict['nmid2'] = 'NM2DMT'
12520
12521    # Filter rows where modality is 'T1w'
12522    df = study_df[ study_df['modality'] == 'T1w']
12523    badnames = get_names_from_data_frame( ['Unnamed'], df )
12524    df=df.drop(badnames, axis=1)
12525    # prefilter df for data that exists
12526    keep = np.tile( False, df.shape[0] )
12527    for x in range(df.shape[0]):
12528        myfn = os.path.basename( df['filename'].iloc[x] )
12529        temp = myfn.split( splitsep )
12530        # Generalized search paths
12531        sid0 = str( temp[1] )
12532        sid = str( df[subject_col].iloc[x] )
12533        if sid0 != sid:
12534            warnings.warn("OUTER: the id derived from the filename " + sid0 + " does not match the id stored in the data frame " + sid )
12535            warnings.warn( "filename is : " +  myfn )
12536            warnings.warn( "sid is : " + sid )
12537            warnings.warn( "x is : " + str(x) )
12538        myproj = str(df[project_col].iloc[x])
12539        mydate = str(df[date_col].iloc[x])
12540        myid = str(df[image_col].iloc[x])
12541        if second_split:
12542            myid = myid.split(".")[0]
12543        path_template = base_path + "/" + myproj +  "/" + sid + "/" + mydate + '/' + hiervariable + '/' + str(myid) + "/"
12544        hierfn = sorted(glob( path_template + "*" + hiervariable + "*wide.csv" ) )
12545        if len( hierfn ) == 0:
12546            print( hierfn )
12547            print( path_template )
12548            print( myproj )
12549            print( sid )
12550            print( mydate ) 
12551            print( myid )
12552        if len( hierfn ) > 0:
12553            keep[x]=True
12554
12555    # df=df[keep]
12556    if df.shape[0] == 0:
12557        warnings.warn("input data frame shape is filtered down to zero")
12558        return df
12559
12560    if not df.index.is_unique:
12561        warnings.warn("data frame does not have unique indices.  we therefore reset the index to allow the function to continue on." )
12562        df = df.reset_index()
12563
12564    
12565    if verbose:
12566        print( "original input had shape " + str( df.shape[0] ) + " (T1 only) and we find " + str( (keep).sum() ) + " with hierarchical output defined by variable: " + hiervariable )
12567        print( df.shape )
12568
12569    dfout = pd.DataFrame()
12570    myct = 0
12571    for x in range( df.shape[0]):
12572        if verbose:
12573            print("\n\n-------------------------------------------------")
12574            print(f"{x}...")
12575        else:
12576            progress_reporter(x, df.shape[0], width=500)
12577        locind = df.index[x]
12578        myfn = os.path.basename( df['filename'].iloc[x] )
12579        sid = str( df[subject_col].iloc[x] )
12580        tempB = myfn.split( splitsep )
12581        sid0 = str(tempB[1])
12582        if sid0 != sid and verbose:
12583            warnings.warn("INNER: the id derived from the filename " + str(sid) + " does not match the id stored in the data frame " + str(sid0) )
12584            warnings.warn( "filename is : " +  str(myfn) )
12585            warnings.warn( "sid is : " + str(sid) )
12586            warnings.warn( "x is : " + str(x) )
12587            warnings.warn( "index is : " + str(locind) )
12588        myproj = str(df[project_col].iloc[x])
12589        mydate = str(df[date_col].iloc[x])
12590        myid = str(df[image_col].iloc[x])
12591        if second_split:
12592            myid = myid.split(".")[0]
12593        if verbose:
12594            print( myfn )
12595            print( temp )
12596            print( "id " + sid  )
12597        path_template = base_path + "/" + myproj +  "/" + sid + "/" + mydate + '/' + hiervariable + '/' + str(myid) + "/"
12598        searchhier = path_template + "*" + hiervariable + "*wide.csv"
12599        if verbose:
12600            print( searchhier )
12601        hierfn = sorted( glob( searchhier ) )
12602        if len( hierfn ) > 1:
12603            raise ValueError("there are " + str( len( hierfn ) ) + " number of hier fns with search path " + searchhier )
12604        if len( hierfn ) == 1:
12605            hdf=t1df=dtdf=rsdf=perfdf=nmdf=flairdf=None
12606            if verbose:
12607                print(hierfn)
12608            hdf = pd.read_csv(hierfn[0])
12609            if verbose:
12610                print( hdf['vol_hemisphere_lefthemispheres'] )
12611            badnames = get_names_from_data_frame( ['Unnamed'], hdf )
12612            hdf=hdf.drop(badnames, axis=1)
12613            nums = [isinstance(hdf[col].iloc[0], (int, float)) for col in hdf.columns]
12614            corenames = list(np.array(hdf.columns)[nums])
12615            # hdf.loc[:, nums] = hdf.loc[:, nums].add_prefix("T1Hier_")
12616            hdf = hdf.add_prefix("T1Hier_")
12617            myct = myct + 1
12618            dflist = [hdf]
12619
12620            for mymod in vmoddict.keys():
12621                if verbose:
12622                    print("\n\n************************* " + mymod + " *************************")
12623                modalityclass = vmoddict[ mymod ]
12624                if wild_card_modality_id:
12625                    mymodid = '*'
12626                else:
12627                    mymodid = str( df[mymod].iloc[x] )
12628                    if mymodid.lower() != "nan" and mymodid.lower() != "na":
12629                        mymodid = os.path.basename( mymodid )
12630                        mymodid = os.path.splitext( mymodid )[0]
12631                        mymodid = os.path.splitext( mymodid )[0]
12632                        temp = mymodid.split( idsep )
12633                        mymodid = temp[ len( temp )-1 ]
12634                    else:
12635                        if verbose:
12636                            print("missing")
12637                        continue
12638                if verbose:
12639                    print( "modality id is " + mymodid + " for modality " + modalityclass + ' modality specific subj ' + sid + ' modality specific id is ' + myid + " its date " +  mydate )
12640                modalityclasssearch = modalityclass
12641                if modalityclass in ['rsfMRI','DTI']:
12642                    modalityclasssearch=modalityclass+"*"
12643                path_template_m = base_path + "/" + myproj +  "/" + sid + "/" + mydate + '/' + modalityclasssearch + '/' + mymodid + "/"
12644                modsearch = path_template_m + "*" + modalityclasssearch + "*wide.csv"
12645                if verbose:
12646                    print( modsearch )
12647                t1wfn = sorted( glob( modsearch ) )
12648                if len( t1wfn ) > 1:
12649                    nlarge = len(t1wfn)
12650                    t1wfn = find_most_recent_file( t1wfn )
12651                    warnings.warn("there are " + str( nlarge ) + " number of wide fns with search path " + modsearch + " we take the most recent of these " + t1wfn[0] )
12652                if len( t1wfn ) == 1:
12653                    if verbose:
12654                        print(t1wfn)
12655                    t1df = myread_csv(t1wfn[0], corenames)
12656                    t1df = filter_df( t1df, modalityclass+'_')
12657                    dflist = dflist + [t1df]
12658                else:
12659                    if verbose:
12660                        print( " cannot find " + modsearch )
12661                
12662            hdf = pd.concat( dflist, axis=1, ignore_index=False)
12663            if verbose:
12664                print( "count: " + str( myct ) )
12665            subdf = df.iloc[[x]]
12666            hdf.index = subdf.index.copy()
12667            subdf = pd.concat( [subdf,hdf], axis=1, ignore_index=False)
12668            dfout = pd.concat( [dfout,subdf], axis=0, ignore_index=False )
12669
12670    if dfout.shape[0] > 0:
12671        badnames = get_names_from_data_frame( ['Unnamed'], dfout )
12672        dfout=dfout.drop(badnames, axis=1)
12673    return dfout
12674
12675def enantiomorphic_filling_without_mask( image, axis=0, intensity='low' ):
12676    """
12677    Perform an enantiomorphic lesion filling on an image without a lesion mask.
12678
12679    Args:
12680    image (antsImage): The ants image to flip and fill
12681    axis ( int ): the axis along which to reflect the image
12682    intensity ( str ) : low or high
12683
12684    Returns:
12685    ants.ANTsImage: The image after enantiomorphic filling.
12686    """
12687    imagen = ants.iMath( image, 'Normalize' )
12688    imagen = ants.iMath( imagen, "TruncateIntensity", 1e-6, 0.98 )
12689    imagen = ants.iMath( imagen, 'Normalize' )
12690    # Create a mirror image (flipping left and right)
12691    mirror_image = ants.reflect_image(imagen, axis=0, tx='antsRegistrationSyNQuickRepro[s]' )['warpedmovout']
12692
12693    # Create a symmetric version of the image by averaging the original and the mirror image
12694    symmetric_image = imagen * 0.5 + mirror_image * 0.5
12695
12696    # Identify potential lesion areas by finding differences between the original and symmetric image
12697    difference_image = image - symmetric_image
12698    diffseg = ants.threshold_image(difference_image, "Otsu", 3 )
12699    if intensity == 'low':
12700        likely_lesion = ants.threshold_image( diffseg, 1,  1)
12701    else:
12702        likely_lesion = ants.threshold_image( diffseg, 3,  3)
12703    likely_lesion = ants.smooth_image( likely_lesion, 3.0 ).iMath("Normalize")
12704    lesionneg = ( imagen*0+1.0 ) - likely_lesion
12705    filled_image = ants.image_clone(imagen)    
12706    filled_image = imagen * lesionneg + mirror_image * likely_lesion
12707
12708    return filled_image, diffseg
12709
12710
12711
12712def filter_image_files(image_paths, criteria='largest'):
12713    """
12714    Filters a list of image file paths based on specified criteria and returns 
12715    the path of the image that best matches that criteria (smallest, largest, or brightest).
12716
12717    Args:
12718    image_paths (list): A list of file paths to the images.
12719    criteria (str): Criteria for selecting the image ('smallest', 'largest', 'brightest').
12720
12721    Returns:
12722    str: The file path of the selected image, or None if no valid images are found.
12723    """
12724    import numpy as np
12725    if not image_paths:
12726        return None
12727
12728    selected_image_path = None
12729    if criteria == 'smallest' or criteria == 'largest':
12730        extreme_volume = None
12731
12732        for path in image_paths:
12733            try:
12734                image = ants.image_read(path)
12735                volume = np.prod(image.shape)
12736
12737                if criteria == 'largest':
12738                    if extreme_volume is None or volume > extreme_volume:
12739                        extreme_volume = volume
12740                        selected_image_path = path
12741                elif criteria == 'smallest':
12742                    if extreme_volume is None or volume < extreme_volume:
12743                        extreme_volume = volume
12744                        selected_image_path = path
12745
12746            except Exception as e:
12747                print(f"Error processing image {path}: {e}")
12748
12749    elif criteria == 'brightest':
12750        max_brightness = None
12751
12752        for path in image_paths:
12753            try:
12754                image = ants.image_read(path)
12755                brightness = np.mean(image.numpy())
12756
12757                if max_brightness is None or brightness > max_brightness:
12758                    max_brightness = brightness
12759                    selected_image_path = path
12760
12761            except Exception as e:
12762                print(f"Error processing image {path}: {e}")
12763
12764    else:
12765        raise ValueError("Criteria must be 'smallest', 'largest', or 'brightest'.")
12766
12767    return selected_image_path
12768
12769
12770
12771def mm_match_by_qc_scoring(df_a, df_b, match_column, criteria, prefix='matched_', exclude_columns=None):
12772    """
12773    Match each row in df_a to a row in df_b based on a matching column and criteria for selecting the best match,
12774    with options to prefix column names from df_b and exclude certain columns from the final output. Additionally,
12775    returns a DataFrame containing rows from df_b that were not matched to any row in df_a.
12776
12777    Parameters:
12778    - df_a: DataFrame A.
12779    - df_b: DataFrame B.
12780    - match_column: The column name on which rows should match between DataFrame A and B.
12781    - criteria: A dictionary where keys are column names and values are 'min' or 'max', indicating whether
12782                the column should be minimized or maximized for the best match.
12783    - prefix: A string prefix to add to column names from df_b in the final output to avoid duplication.
12784    - exclude_columns: A list of column names from df_b to exclude from the final output.
12785    
12786    Returns:
12787    - A tuple of two DataFrames: 
12788        1. A new DataFrame combining df_a with matched rows from df_b.
12789        2. A DataFrame containing rows from df_b that were not matched to df_a.
12790    """
12791    from scipy.stats import zscore
12792    df_a = df_a.loc[:, ~df_a.columns.str.startswith('Unnamed:')].copy()
12793    if df_b is not None:
12794        df_b = df_b.loc[:, ~df_b.columns.str.startswith('Unnamed:')].copy()
12795    else:
12796        return df_a, pd.DataFrame()
12797    
12798    # Normalize df_b based on criteria
12799    for col, crit in criteria.items():
12800        if crit == 'max':
12801            df_b.loc[df_b.index, f'score_{col}'] = zscore(-df_b[col])
12802        elif crit == 'min':
12803            df_b.loc[df_b.index, f'score_{col}'] = zscore(df_b[col])
12804
12805    # Calculate 'best_score' by summing all score columns
12806    score_columns = [f'score_{col}' for col in criteria.keys()]
12807    df_b['best_score'] = df_b[score_columns].sum(axis=1)
12808
12809    matched_indices = []  # Track indices of matched rows in df_b
12810
12811    # Match rows
12812    matched_rows = []
12813    for _, row_a in df_a.iterrows():
12814        matches = df_b[df_b[match_column] == row_a[match_column]]
12815        if not matches.empty:
12816            best_idx = matches['best_score'].idxmin()
12817            best_match = matches.loc[best_idx]
12818            matched_indices.append(best_idx)  # Track this index as matched
12819            matched_rows.append(best_match)
12820        else:
12821            matched_rows.append(pd.Series(dtype='float64'))
12822
12823    # Create a DataFrame from matched rows
12824    df_matched = pd.DataFrame(matched_rows).reset_index(drop=True)
12825    
12826    # Exclude specified columns and add prefix
12827    if exclude_columns is not None:
12828        df_matched = df_matched.drop(columns=exclude_columns, errors='ignore')
12829    df_matched = df_matched.rename(columns=lambda x: f"{prefix}{x}" if x != match_column and x in df_matched.columns else x)
12830
12831    # Combine df_a with matched rows from df_b
12832    result_df = pd.concat([df_a.reset_index(drop=True), df_matched], axis=1)
12833    
12834    # Extract unmatched rows from df_b
12835    unmatched_df_b = df_b.drop(index=matched_indices).reset_index(drop=True)
12836
12837    return result_df, unmatched_df_b
12838
12839
12840def fix_LR_RL_stuff(df, col1, col2, size_col1, size_col2, id1, id2 ):
12841    df_copy = df.copy()
12842    # Ensure columns contain strings for substring checks
12843    df_copy[col1] = df_copy[col1].astype(str)
12844    df_copy[col2] = df_copy[col2].astype(str)
12845    df_copy[id1] = df_copy[id1].astype(str)
12846    df_copy[id2] = df_copy[id2].astype(str)
12847    
12848    for index, row in df_copy.iterrows():
12849        col1_val = row[col1]
12850        col2_val = row[col2]
12851        size1 = row[size_col1]
12852        size2 = row[size_col2]
12853        
12854        # Check for 'RL' or 'LR' in each column and compare sizes
12855        if ('RL' in col1_val or 'LR' in col1_val) and ('RL' in col2_val or 'LR' in col2_val):
12856            continue
12857        elif 'RL' not in col1_val and 'LR' not in col1_val and 'RL' not in col2_val and 'LR' not in col2_val:
12858            if size1 < size2:
12859                df_copy.at[index, col1] = df_copy.at[index, col2]
12860                df_copy.at[index, size_col1] = df_copy.at[index, size_col2]
12861                df_copy.at[index, id1] = df_copy.at[index, id2]
12862                df_copy.at[index, size_col2] = 0
12863                df_copy.at[index, col2] = None
12864                df_copy.at[index, id2] = None
12865            else:
12866                df_copy.at[index, col2] = None
12867                df_copy.at[index, size_col2] = 0
12868                df_copy.at[index, id2] = None
12869        elif 'RL' in col1_val or 'LR' in col1_val:
12870            if size1 < size2:
12871                df_copy.at[index, col1] = df_copy.at[index, col2]
12872                df_copy.at[index, id1] = df_copy.at[index, id2]
12873                df_copy.at[index, size_col1] = df_copy.at[index, size_col2]
12874                df_copy.at[index, size_col2] = 0
12875                df_copy.at[index, col2] = None
12876                df_copy.at[index, id2] = None
12877            else:
12878                df_copy.at[index, col2] = None
12879                df_copy.at[index, id2] = None
12880                df_copy.at[index, size_col2] = 0
12881        elif 'RL' in col2_val or 'LR' in col2_val:
12882            if size2 < size1:
12883                df_copy.at[index, id2] = None
12884                df_copy.at[index, col2] = None
12885                df_copy.at[index, size_col2] = 0
12886            else:
12887                df_copy.at[index, col1] = df_copy.at[index, col2]
12888                df_copy.at[index, id1] = df_copy.at[index, id2]
12889                df_copy.at[index, size_col1] = df_copy.at[index, size_col2]
12890                df_copy.at[index, size_col2] = 0
12891                df_copy.at[index, col2] = None    
12892                df_copy.at[index, id2] = None    
12893    return df_copy
12894
12895
12896def renameit(df, old_col_name, new_col_name):
12897    """
12898    Renames a column in a pandas DataFrame in place. Raises an error if the specified old column name does not exist.
12899
12900    Parameters:
12901    - df: pandas.DataFrame
12902        The DataFrame in which the column is to be renamed.
12903    - old_col_name: str
12904        The current name of the column to be renamed.
12905    - new_col_name: str
12906        The new name for the column.
12907    
12908    Raises:
12909    - ValueError: If the old column name does not exist in the DataFrame.
12910    
12911    Returns:
12912    None
12913    """
12914    import warnings
12915    # Check if the old column name exists in the DataFrame
12916    if old_col_name not in df.columns:
12917        warnings.warn(f"The column '{old_col_name}' does not exist in the DataFrame.")
12918        return
12919    
12920    # Proceed with renaming the column if it exists
12921    df.rename(columns={old_col_name: new_col_name}, inplace=True)
12922
12923
12924def mm_match_by_qc_scoring_all( qc_dataframe, fix_LRRL=True, mysep='-', verbose=True ):
12925    """
12926    Processes a quality control (QC) DataFrame to perform modality-specific matching and filtering based
12927    on predefined criteria, optimizing for minimal outliers and noise, and maximal signal-to-noise ratio (SNR),
12928    expected value of randomness (EVR), and dimensionality time (dimt).
12929
12930    This function iteratively matches dataframes derived from the QC dataframe for different imaging modalities,
12931    applying a series of filters to select the best matches based on the QC metrics. Matches are made with
12932    consideration to minimize outlier loop and noise, while maximizing SNR, EVR, and dimt for each modality.
12933
12934    Parameters:
12935    ----------
12936    qc_dataframe : pandas.DataFrame
12937        The DataFrame containing QC metrics for different modalities and imaging data.
12938    fix_LRRL : bool, optional
12939    mysep : string, character such as - or _ the usual antspymm separator argument
12940
12941    verbose : bool, optional
12942        If True, prints the progress and the shape of the DataFrame being processed in each step.
12943
12944    Process:
12945    -------
12946    1. Standardizes modalities by merging DTI-related entries.
12947    2. Converts specific columns to appropriate data types for processing.
12948    3. Performs modality-specific matching and filtering based on the outlier column and criteria for each modality.
12949    4. Iteratively processes unmatched data for predefined modalities with specific prefixes to find further matches.
12950    
12951    Returns:
12952    -------
12953    pandas.DataFrame
12954        The matched and filtered DataFrame after applying all QC scoring and matching operations across specified modalities.
12955
12956    """
12957    qc_dataframe=remove_unwanted_columns( qc_dataframe )
12958    qc_dataframe['modality'] = qc_dataframe['modality'].replace(['DTIdwi', 'DTIb0'], 'DTI', regex=True)
12959    qc_dataframe['filename']=qc_dataframe['filename'].astype(str)
12960    qc_dataframe['ol_loop']=qc_dataframe['ol_loop'].astype(float)
12961    qc_dataframe['ol_lof']=qc_dataframe['ol_lof'].astype(float)
12962    qc_dataframe['ol_lof_decision']=qc_dataframe['ol_lof_decision'].astype(float)
12963    outlier_column='ol_loop'
12964    mmdf0 = best_mmm( qc_dataframe, 'T1w', outlier_column=outlier_column, mysep=mysep )['filt']
12965    fldf = best_mmm( qc_dataframe, 'T2Flair', outlier_column=outlier_column, mysep=mysep  )['filt']
12966    nmdf = best_mmm( qc_dataframe, 'NM2DMT', outlier_column=outlier_column, mysep=mysep  )['filt']
12967    rsdf = best_mmm( qc_dataframe, 'rsfMRI', outlier_column=outlier_column, mysep=mysep  )['filt']
12968    dtdf = best_mmm( qc_dataframe, 'DTI', outlier_column=outlier_column, mysep=mysep  )['filt']
12969    pfdf = best_mmm( qc_dataframe, 'perf', outlier_column=outlier_column, mysep=mysep  )['filt']
12970
12971    criteria = {'ol_loop': 'min', 'noise': 'min', 'snr': 'max', 'EVR': 'max', 'reflection_err':'min'}
12972    xcl = [ 'mrimfg', 'mrimodel','mriMagneticFieldStrength', 'dti_failed', 'rsf_failed', 'subjectID', 'date', 'subjectIDdate','repeat']
12973    # Assuming df_a and df_b are already loaded
12974    mmdf, undffl = mm_match_by_qc_scoring(mmdf0, fldf, 'subjectIDdate', criteria, 
12975                        prefix='T2Flair_', exclude_columns=xcl )
12976
12977    mmdf, undfpf = mm_match_by_qc_scoring(mmdf, pfdf, 'subjectIDdate', criteria, 
12978                        prefix='perf_', exclude_columns=xcl )
12979
12980    prefixes = ['NM1_', 'NM2_', 'NM3_', 'NM4_', 'NM5_', 'NM6_']  
12981    undfmod = nmdf  # Initialize 'undfmod' with 'nmdf' for the first iteration
12982    if undfmod is not None:
12983        if verbose:
12984            print('start NM')
12985            print( undfmod.shape )
12986        for prefix in prefixes:
12987            if undfmod.shape[0] > 50:
12988                mmdf, undfmod = mm_match_by_qc_scoring(mmdf, undfmod, 'subjectIDdate', criteria, prefix=prefix, exclude_columns=xcl)
12989                if verbose:
12990                    print( prefix )
12991                    print( undfmod.shape )
12992
12993    criteria = {'ol_loop': 'min', 'noise': 'min', 'snr': 'max', 'EVR': 'max', 'dimt':'max'}
12994    # higher bvalues lead to more noise ...
12995    criteria = {'ol_loop': 'min', 'noise': 'min',  'dti_bvalueMax':'min',  'dimt':'max'}
12996    prefixes = ['DTI1_', 'DTI2_', 'DTI3_']  # List of prefixes for each matching iteration
12997    undfmod = dtdf
12998    if undfmod is not None:
12999        if verbose:
13000            print('start DT')
13001            print( undfmod.shape )
13002        for prefix in prefixes:
13003            if undfmod.shape[0] > 50:
13004                mmdf, undfmod = mm_match_by_qc_scoring(mmdf, undfmod, 'subjectIDdate', criteria, prefix=prefix, exclude_columns=xcl)
13005                if verbose:
13006                    print( prefix )
13007                    print( undfmod.shape )
13008
13009    prefixes = ['rsf1_', 'rsf2_', 'rsf3_']  # List of prefixes for each matching iteration
13010    undfmod = rsdf  # Initialize 'undfmod' with 'nmdf' for the first iteration
13011    if undfmod is not None:
13012        if verbose:
13013            print('start rsf')
13014            print( undfmod.shape )
13015        for prefix in prefixes:
13016            if undfmod.shape[0] > 50:
13017                mmdf, undfmod = mm_match_by_qc_scoring(mmdf, undfmod, 'subjectIDdate', criteria, prefix=prefix, exclude_columns=xcl)
13018                if verbose:
13019                    print( prefix )
13020                    print( undfmod.shape )
13021    
13022    if fix_LRRL:
13023        #        mmdf=fix_LR_RL_stuff( mmdf, 'DTI1_filename', 'DTI2_filename', 'DTI1_dimt', 'DTI2_dimt')
13024        mmdf=fix_LR_RL_stuff( mmdf, 'rsf1_filename', 'rsf2_filename', 'rsf1_dimt', 'rsf2_dimt', 'rsf1_imageID', 'rsf2_imageID'  )
13025    else:
13026        import warnings
13027        warnings.warn("FIXME: should fix LR and RL situation for the DTI and rsfMRI")
13028
13029    # now do the necessary replacements
13030    
13031    renameit( mmdf, 'perf_imageID', 'perfid' )
13032    renameit( mmdf, 'perf_filename', 'perffn' )
13033    renameit( mmdf, 'T2Flair_imageID', 'flairid' )
13034    renameit( mmdf, 'T2Flair_filename', 'flairfn' )
13035    renameit( mmdf, 'rsf1_imageID', 'rsfid1' )
13036    renameit( mmdf, 'rsf2_imageID', 'rsfid2' )
13037    renameit( mmdf, 'rsf1_filename', 'rsffn1' )
13038    renameit( mmdf, 'rsf2_filename', 'rsffn2' )
13039    renameit( mmdf, 'DTI1_imageID', 'dtid1' )
13040    renameit( mmdf, 'DTI2_imageID', 'dtid2' )
13041    renameit( mmdf, 'DTI3_imageID', 'dtid3' )
13042    renameit( mmdf, 'DTI1_filename', 'dtfn1' )
13043    renameit( mmdf, 'DTI2_filename', 'dtfn2' )
13044    renameit( mmdf, 'DTI3_filename', 'dtfn3' )
13045    for x in range(1,6):
13046        temp0="NM"+str(x)+"_imageID"
13047        temp1="nmid"+str(x)
13048        renameit( mmdf, temp0, temp1 )
13049        temp0="NM"+str(x)+"_filename"
13050        temp1="nmfn"+str(x)
13051        renameit( mmdf, temp0, temp1 )
13052    return mmdf
13053
13054
13055def t1w_super_resolution_with_hemispheres(
13056    t1img,
13057    model,
13058    dilation_amount=8,
13059    truncation=[0.001, 0.999],
13060    target_range=[0, 1],
13061    poly_order="hist",
13062    min_spacing=0.8,
13063    verbose=True
13064):
13065    """
13066    Perform hemisphere-aware super-resolution on a T1-weighted image using a segmentation-guided DBPN model.
13067
13068    This function performs brain extraction, hemisphere labeling, and segmentation-aware
13069    super-resolution using the provided T1 image and model. If the resolution is sufficient,
13070    hemisphere labels guide targeted SR via the `siq.inference` function.
13071
13072    Parameters
13073    ----------
13074    t1img : ANTsImage
13075        Input T1-weighted image.
13076
13077    model : keras.Model
13078        Super-resolution model (e.g., from `siq.default_dbpn` or loaded from `.keras` file).
13079
13080    dilation_amount : int
13081        Amount of dilation to apply around labeled regions before SR.
13082
13083    truncation : list of float
13084        Percentile values used to truncate intensity before model inference.
13085
13086    target_range : list of float
13087        Range to normalize input intensities for model input.
13088
13089    poly_order : str or int
13090        Polynomial order or "hist" for histogram matching after SR.
13091
13092    min_spacing : float 
13093        if the minimum input image spacing is less than this value, 
13094        the function will return the original image.  Default 0.8.
13095
13096    verbose : bool
13097        If True, print progress updates.
13098
13099    Returns
13100    -------
13101    ANTsImage
13102        Super-resolved T1-weighted image.
13103    """
13104    if np.min(ants.get_spacing(t1img)) < min_spacing:
13105        if verbose:
13106            print("Image resolution too high — skipping SR.")
13107        return t1img
13108
13109    if verbose:
13110        print("Performing brain extraction...")
13111    brain_mask = antspyt1w.brain_extraction(t1img)
13112    brain = t1img * brain_mask
13113
13114    if verbose:
13115        print("Begin template loading")
13116    tlrfn = antspyt1w.get_data('T_template0_LR', target_extension='.nii.gz')
13117    tfn = antspyt1w.get_data('T_template0', target_extension='.nii.gz')
13118    template = ants.image_read(tfn)
13119    template = (template * antspynet.brain_extraction(template, 't1')).iMath("Normalize")
13120    template_lr = ants.image_read(tlrfn)
13121    if verbose:
13122        print("Done template loading")
13123
13124    if verbose:
13125        print("Labeling hemispheres...")
13126    hemi_seg = antspyt1w.label_hemispheres(brain, template, template_lr)
13127
13128    # Combine segmentation and brain mask — label values 1, 2 (hemi) → 3, 4
13129    hemisphere_mask = hemi_seg + 2.0 * brain_mask
13130
13131    if verbose:
13132        print("Starting segmentation-aware super-resolution...")
13133    sr_result = siq.inference(
13134        t1img,
13135        model,
13136        segmentation=hemisphere_mask,
13137        truncation=truncation,
13138        target_range=target_range,
13139        dilation_amount=dilation_amount,
13140        poly_order=poly_order,
13141        verbose=verbose
13142    )
13143
13144    sr_image = sr_result['super_resolution'] if isinstance(sr_result, dict) else sr_result
13145
13146    if verbose:
13147        print("Done super-resolution.")
13148    return sr_image
13149
13150
13151def map_idps_to_rois(
13152    idp_data_frame: pd.DataFrame,
13153    roi_image: ants.ANTsImage,
13154    idp_column: str,
13155    map_type: str = 'average'
13156) -> ants.ANTsImage:
13157    """
13158    Produces a new ANTsImage where each ROI is assigned a value based on IDP data
13159    from a DataFrame. ROIs are identified by integer labels in `roi_image`
13160    and values are linked via `idp_data_frame`.
13161
13162    Assumes `idp_data_frame` contains both 'Label' (integer ROI ID) and
13163    'Description' (string description for the ROI, e.g., 'left caudal anterior cingulate')
13164    columns, in addition to the specified `idp_column`.
13165
13166    Parameters:
13167    - idp_data_frame (pd.DataFrame): DataFrame containing IDP measurements.
13168      Must have 'Label', 'Description' (for hemisphere parsing), and `idp_column`.
13169    - roi_image (ants.ANTsImage): An ANTsImage where each voxel contains an integer
13170      label identifying an ROI.
13171    - idp_column (str): The name of the column in `idp_data_frame` whose values
13172      are to be mapped to the ROIs (e.g., 'VolumeInMillimeters').
13173    - map_type (str): Type of mapping to perform.
13174      - 'average': For identified paired left/right ROIs, their `idp_column` values are
13175                   averaged and this average is assigned to both the left and right
13176                   hemisphere ROIs in the output image. If only one side of a pair
13177                   is found, its raw value is used.
13178      - 'asymmetry': For identified paired left/right ROIs, the (Left - Right)
13179                     difference for `idp_column` is calculated and assigned only
13180                     to the left hemisphere ROI. Right hemisphere ROIs that are
13181                     part of a pair, and any unpaired ROIs, will be set to 0 in
13182                     the output image.
13183      - 'raw': Each ROI's original value from `idp_column` is mapped directly to
13184               its corresponding ROI in the output image.
13185      Default is 'average'.
13186
13187    Returns:
13188    - ants.ANTsImage: A new ANTsImage with the same header (origin, spacing,
13189                      direction, etc.) as `roi_image`, where ROI voxels are filled
13190                      with the mapped IDP values. Voxels not part of any described
13191                      ROI, or unmatched based on `map_type`, will be 0.
13192
13193    Raises:
13194    - ValueError: If required columns (`Label`, `Description`, `idp_column`) are missing
13195                  from `idp_data_frame`, `roi_image` is not an ANTsImage,
13196                  or `map_type` is invalid.
13197    """
13198    import logging
13199
13200    logging.info(f"Starting map_idps_to_rois (map_type='{map_type}', IDP column='{idp_column}')")
13201
13202    # --- 1. Input Validation ---
13203    required_idp_cols = ['Label', 'Description', idp_column]
13204    if not all(col in idp_data_frame.columns for col in required_idp_cols):
13205        raise ValueError(f"idp_data_frame must contain columns: {', '.join(required_idp_cols)}")
13206
13207    if not isinstance(roi_image, ants.ANTsImage):
13208        raise ValueError("roi_image must be an ants.ANTsImage object.")
13209
13210    valid_map_types = ['average', 'asymmetry', 'raw']
13211    if map_type not in valid_map_types:
13212        raise ValueError(f"Invalid map_type: '{map_type}'. Must be one of {valid_map_types}.")
13213
13214    # --- 2. Prepare Data (use idp_data_frame directly) ---
13215    # Select only the necessary columns from the input idp_data_frame
13216    processed_idp_df = idp_data_frame[['Label', 'Description', idp_column]].copy()
13217    processed_idp_df.rename(columns={idp_column: 'Value'}, inplace=True)
13218
13219    # Ensure 'Label' column is numeric and drop rows where conversion fails
13220    processed_idp_df['Label'] = pd.to_numeric(processed_idp_df['Label'], errors='coerce')
13221    processed_idp_df = processed_idp_df.dropna(subset=['Label']) # Drop rows where Label is NaN after coercion
13222    processed_idp_df['Label'] = processed_idp_df['Label'].astype(int) # Convert to integer labels
13223
13224    logging.info(f"Processed IDP data contains {len(processed_idp_df)} entries. "
13225                 f"{processed_idp_df['Value'].isnull().sum()} entries have no valid IDP value (NaN).")
13226
13227    # --- 3. Identify Hemispheres and Base Regions ---
13228    # This helper function parses the ROI description to determine its hemisphere
13229    # and a common base name for pairing (e.g., 'caudal anterior cingulate').
13230    def get_hemisphere_and_base(description):
13231        desc = str(description).strip()
13232
13233        # Pattern 1: FreeSurfer-like (e.g., "left caudal anterior cingulate")
13234        match_fs = re.match(r"^(left|right)\s(.+)$", desc, re.IGNORECASE)
13235        if match_fs:
13236            return match_fs.group(1).capitalize(), match_fs.group(2).strip()
13237
13238        # Pattern 2: BN_STR-like (e.g., "BN_STR_Pu_Left")
13239        match_bn = re.match(r"(.+)_(Left|Right)$", desc, re.IGNORECASE)
13240        if match_bn:
13241            return match_bn.group(2).capitalize(), match_bn.group(1).strip()
13242
13243        # No clear hemisphere identified (e.g., 'corpus callosum')
13244        return 'Unknown', desc
13245
13246    processed_idp_df[['Hemisphere', 'BaseRegion']] = processed_idp_df['Description'].apply(
13247        lambda x: pd.Series(get_hemisphere_and_base(x))
13248    )
13249
13250    # Dictionary to store the final computed value for each ROI Label
13251    label_value_map = {}
13252
13253    # --- 4. Process Values based on map_type ---
13254    if map_type == 'raw':
13255        logging.info("Mapping raw IDP values to ROIs.")
13256        # Directly map values where available
13257        for _, row in processed_idp_df.dropna(subset=['Value']).iterrows():
13258            label_value_map[row['Label']] = row['Value']
13259
13260    else: # 'average' or 'asymmetry' types which require pairing logic
13261        # Group by the 'BaseRegion' to find potential left/right pairs
13262        grouped = processed_idp_df.groupby('BaseRegion')
13263
13264        for base_region, group_df in grouped:
13265            left_roi_data = group_df[group_df['Hemisphere'] == 'Left'].dropna(subset=['Value'])
13266            right_roi_data = group_df[group_df['Hemisphere'] == 'Right'].dropna(subset=['Value'])
13267
13268            # Handle ROIs that are not clearly left/right (e.g., 'Bilateral' or 'Unknown')
13269            # For these, we include their raw value regardless of map_type.
13270            other_rois = group_df[ (group_df['Hemisphere'] != 'Left') & (group_df['Hemisphere'] != 'Right') ].dropna(subset=['Value'])
13271            for _, row in other_rois.iterrows():
13272                label_value_map[row['Label']] = row['Value']
13273                logging.debug(f"ROI: '{base_region}' (Label: {row['Label']}) - Not a pair, mapped raw value: {row['Value']:.2f}")
13274
13275            # Process paired regions if both left and right data are available
13276            if not left_roi_data.empty and not right_roi_data.empty:
13277                l_label = left_roi_data['Label'].iloc[0]
13278                l_value = left_roi_data['Value'].iloc[0]
13279                r_label = right_roi_data['Label'].iloc[0]
13280                r_value = right_roi_data['Value'].iloc[0]
13281
13282                if map_type == 'average':
13283                    avg_val = (l_value + r_value) / 2
13284                    label_value_map[l_label] = avg_val
13285                    label_value_map[r_label] = avg_val
13286                    logging.debug(f"ROI: '{base_region}' - Paired AVG: {avg_val:.2f} (L:{l_value:.2f}, R:{r_value:.2f})")
13287                elif map_type == 'asymmetry':
13288                    asym_val = l_value - r_value
13289                    label_value_map[l_label] = asym_val
13290                    # Right ROI is not assigned a value based on asymmetry, so it retains 0
13291                    logging.debug(f"ROI: '{base_region}' - Paired ASYM (L-R): {asym_val:.2f} (L:{l_value:.2f}, R:{r_value:.2f})")
13292            else:
13293                # If only one side of a pair (or neither) is found with a valid value
13294                if map_type == 'average':
13295                    if not left_roi_data.empty:
13296                        label_value_map[left_roi_data['Label'].iloc[0]] = left_roi_data['Value'].iloc[0]
13297                        logging.debug(f"ROI: '{base_region}' - L-only AVG (raw): {left_roi_data['Value'].iloc[0]:.2f}")
13298                    if not right_roi_data.empty:
13299                        label_value_map[right_roi_data['Label'].iloc[0]] = right_roi_data['Value'].iloc[0]
13300                        logging.debug(f"ROI: '{base_region}' - R-only AVG (raw): {right_roi_data['Value'].iloc[0]:.2f}")
13301                elif map_type == 'asymmetry':
13302                    # If only one hemisphere's data is available, asymmetry cannot be computed.
13303                    # For asymmetry map_type, any unpaired ROI (including left) gets 0.
13304                    if not left_roi_data.empty:
13305                        label_value_map[left_roi_data['Label'].iloc[0]] = 0.0
13306                        logging.debug(f"ROI: '{base_region}' - L-only ASYM: Set to 0.0 (no pair for computation).")
13307                    if not right_roi_data.empty:
13308                        logging.debug(f"ROI: '{base_region}' - R-only ASYM: Not assigned (relevant for left hemisphere only).")
13309
13310    # --- 5. Populate Output Image Array ---
13311    # Initialize the output NumPy array with zeros, using the robust float32 type
13312    output_numpy = np.zeros(roi_image.shape, dtype=np.float32)
13313    # Get the input ROI image's data as a NumPy array for fast lookups
13314    roi_image_numpy = roi_image.numpy()
13315
13316    total_voxels_mapped = 0
13317    unique_labels_mapped_in_image = set() # Track unique labels actually processed in the image
13318
13319    # Iterate through the `label_value_map` to assign values to the output image
13320    for label_id, value in label_value_map.items():
13321        # Only map if the value is not NaN (means it had data from IDP, valid conversion, etc.)
13322        if not np.isnan(value):
13323            # Find all voxels in `roi_image_numpy` that match the current `label_id`
13324            matching_indices = np.where(roi_image_numpy == int(label_id))
13325
13326            if matching_indices[0].size > 0: # Check if this label actually exists in roi_image
13327                output_numpy[matching_indices] = value
13328                total_voxels_mapped += matching_indices[0].size
13329                unique_labels_mapped_in_image.add(label_id)
13330
13331    logging.info(f"Mapped values for {len(unique_labels_mapped_in_image)} unique ROI labels found in `roi_image`, affecting {total_voxels_mapped} voxels.")
13332    logging.info("Unmapped ROIs in `roi_image` (not present in `idp_data_frame` or outside processing scope) retain value of 0.")
13333
13334    # --- 6. Create ANTsImage Output ---
13335    # Construct the final ANTsImage from the populated NumPy array,
13336    # preserving the spatial header information from the original `roi_image`.
13337    output_image = ants.from_numpy(
13338        output_numpy,
13339        origin=roi_image.origin,
13340        spacing=roi_image.spacing,
13341        direction=roi_image.direction
13342    )
13343
13344    logging.info("map_idps_to_rois completed successfully.")
13345    return output_image