myco.architectures

Deep learning model architectures and functions for validating their structure.

aug_layer()

Add a layer to randomy flip and rotate input features

Source code in myco/architectures.py

def aug_layer() -> Layer:
    """Add a layer to randomy flip and rotate input features"""
    data_augmentation_layer = keras.Sequential(
        [
            preprocessing.RandomFlip(),
            preprocessing.RandomRotation(np.random.choice([0.25, 0.5, 0.75, 1])),
        ],
        name="data_aug",
    )

    return data_augmentation_layer

channel_transform_layer(layer, dropout_rate=0.0, batch_normalize=False)

Apply a 1x1 (aka network-in-netowrk) convolution.

stats.stackexchange.com/questions/194142/what-does-1x1-convolution-mean-in-a-neural-network

Parameters:

Name	Type	Description	Default
layer	Layer	the input feature layer	required
dropout_rate	float	the fraction of random neurons to be set to zero	0.0
batch_normalize	bool	if batch normalization should be applied or not	False

Returns:

Type	Description
Layer	the transformed layer space

Source code in myco/architectures.py

def channel_transform_layer(
    layer: Layer, dropout_rate: float = 0.0, batch_normalize: bool = False
) -> Layer:
    """Apply a 1x1 (aka network-in-netowrk) convolution.

    https://stats.stackexchange.com/questions/194142/what-does-1x1-convolution-mean-in-a-neural-network

    Args:
        layer: the input feature layer
        dropout_rate: the fraction of random neurons to be set to zero
        batch_normalize: if batch normalization should be applied or not
    Returns:
        the transformed layer space
    """
    _, height, width, depth = list(layer.shape)
    n_transformed = depth**2
    encoder = Conv2D(filters=n_transformed, kernel_size=1, padding="valid")(layer)
    encoder = Conv2D(filters=depth, kernel_size=1, padding="valid")(encoder)
    if batch_normalize:
        output_layer = BatchNormalization()(encoder)
    if dropout_rate > 0:
        output_layer = Dropout(dropout_rate)(encoder)

    return encoder

compute_padding_dim(kernel_size, in_lyr=None, out_lyr=None)

Calculates the dimensions of tf.pad function to match the shape of tensors with the desired shape, in layers followed by updampling and in the last layer the in/out-lyr helps acquiring matching shapes

Parameters:

Name	Type	Description	Default
kernel_size	Tuple[int, int]	conv kernel size	required
in_lyr	Layer	input layer	None
out_lyr	Layer	expected layer dimensions	None

Returns:

Type	Description
	the dimensions of padding to attain the out_lyr dimensions

Source code in myco/architectures.py

def compute_padding_dim(
    kernel_size: Tuple[int, int], in_lyr: Layer = None, out_lyr: Layer = None
):
    """Calculates the dimensions of tf.pad function to match the shape of tensors with the desired shape, in layers
    followed by updampling and in the last layer the in/out-lyr helps acquiring matching shapes

    Args:
        kernel_size: conv kernel size
        in_lyr: input layer
        out_lyr: expected layer dimensions

    Returns:
        the dimensions of padding to attain the out_lyr dimensions
    """
    kernel_dim = kernel_size[0]
    if out_lyr is None or in_lyr is None:
        padding_side = kernel_dim - 1
    else:
        out_shp = tf.Tensor.get_shape(out_lyr)
        in_shp = tf.Tensor.get_shape(in_lyr)
        dim_chng = out_shp[1] - in_shp[1]
        padding_side = kernel_dim + dim_chng - 1

    return 2 * [padding_side]

conv2d_layer(layer, n_filters, kernel_size, padding, activation, activation_parameters, batch_normalize, dropout_rate, kernel_initializer, kernel_regularizer, bias_regularizer, activity_regularizer, regularization_type, regularization_lambda, padding_dim=None)

Create a 2D convolution layer with optional batch normalization.

Parameters:

Name	Type	Description	Default
layer	Layer	Input convolution layer.	required
n_filters	int	Filter count for initial convolutions.	required
kernel_size	Tuple[int, int]	Kernel size used for local neighborhood convolutions.	required
padding	str	One of ["valid", "same", "zeros", "symmetric", "reflect"]. "valid" means no padding.	required
activation	str	Activation function used between layers. See Keras docs for details.	required
activation_parameters	dict	Keywords to pass to non-standard activation functions.	required
batch_normalize	bool	Use batch normalization layers, which are typically applied after convolutions.	required
dropout_rate	float	The 0-1 proportion of layer weights to randomly set to 0 in each training step.	required
kernel_initializer	str	Method for initializing layer weights.	required
kernel_regularizer	bool	Apply regularization on the weights of kernel.	required
bias_regularizer	bool	Apply regularization on the bias values.	required
activity_regularizer	bool	Apply regularization on the layer output.	required
regularization_type	str	norm of regularizer ("l1", "l2" or "l1_l2")	required
regularization_lambda	float	regularization parameter is a scaler that defines the amount of regularization that is applied to the kernel weights, bias, and output. lambda reduces the variance of the estiamted parameters. Increasing this value encourages weights to towards smaller values. Very large lambda will prevent model from learning.	required
padding_dim	Tuple[int, int]	2D padding dimensions. Infers this value from kernel_size if unspecified.	None

Returns:

Type	Description
Layer	Keras layer for the start of the network.

Source code in myco/architectures.py

def conv2d_layer(
    layer: Layer,
    n_filters: int,
    kernel_size: Tuple[int, int],
    padding: str,
    activation: str,
    activation_parameters: dict,
    batch_normalize: bool,
    dropout_rate: float,
    kernel_initializer: str,
    kernel_regularizer: bool,
    bias_regularizer: bool,
    activity_regularizer: bool,
    regularization_type: str,
    regularization_lambda: float,
    padding_dim: Tuple[int, int] = None,
) -> Layer:
    """Create a 2D convolution layer with optional batch normalization.

    Args:
        layer: Input convolution layer.
        n_filters: Filter count for initial convolutions.
        kernel_size: Kernel size used for local neighborhood convolutions.
        padding: One of ["valid", "same", "zeros", "symmetric", "reflect"]. "valid" means no padding.
        activation: Activation function used between layers. See Keras docs for details.
        activation_parameters: Keywords to pass to non-standard activation functions.
        batch_normalize: Use batch normalization layers, which are typically applied after convolutions.
        dropout_rate: The 0-1 proportion of layer weights to randomly set to 0 in each training step.
        kernel_initializer: Method for initializing layer weights.
        kernel_regularizer: Apply regularization on the weights of kernel.
        bias_regularizer: Apply regularization on the bias values.
        activity_regularizer: Apply regularization on the layer output.
        regularization_type: norm of regularizer ("l1", "l2" or "l1_l2")
        regularization_lambda: regularization parameter is a scaler that defines the amount of regularization
            that is applied to the kernel weights, bias, and output. lambda reduces the variance of the
            estiamted parameters. Increasing this value encourages weights to towards smaller values.
            Very large lambda will prevent model from learning.
        padding_dim: 2D padding dimensions. Infers this value from kernel_size if unspecified.

    Returns:
        Keras layer for the start of the network.
    """
    if padding.lower() != "valid":
        padding_dim = padding_dim or compute_padding_dim(kernel_size)
        layer = pad_2d(layer, padding_dim, padding)

    output_layer = Conv2D(
        filters=n_filters,
        kernel_size=kernel_size,
        kernel_initializer=kernel_initializer,
    )(layer)

    # handle setting keywords for activation functions that aren't accessible via `Activation`
    if activation.lower() == "leakyrelu":
        output_layer = LeakyReLU(alpha=activation_parameters["alpha"])(output_layer)
    else:
        output_layer = Activation(activation)(output_layer)

    if batch_normalize:
        output_layer = BatchNormalization()(output_layer)

    if dropout_rate > 0:
        output_layer = Dropout(dropout_rate)(output_layer)

    if kernel_regularizer:
        output_layer.kernel_regularizer = get_regularizer(
            regularization_type, regularization_lambda
        )

    if bias_regularizer:
        output_layer.bias_regularizer = get_regularizer(
            regularization_type, regularization_lambda
        )

    if activity_regularizer:
        output_layer.activity_regularizer = get_regularizer(
            regularization_type, regularization_lambda
        )

    return output_layer

dense_2d_block(layer, opts, block_depth)

Create a dense 2D convolution layer.

Parameters:

Name	Type	Description	Default
layer	Layer	Input layer to the convolution.	required
opts	dict	All options to pass into the input convolution layer.	required
block_depth	int	Layer count for the dense block.	required

Returns:

Type	Description
Layer	Keras layer for the start of the network.

Source code in myco/architectures.py

def dense_2d_block(layer: Layer, opts: dict, block_depth: int) -> Layer:
    """Create a dense 2D convolution layer.

    Args:
        layer: Input layer to the convolution.
        opts: All options to pass into the input convolution layer.
        block_depth: Layer count for the dense block.

    Returns:
        Keras layer for the start of the network.
    """
    kernel_size = opts["kernel_size"]
    dense_layer = layer
    for _block_step in range(block_depth):
        intermediate_layer = conv2d_layer(
            dense_layer, **opts, padding_dim=compute_padding_dim(kernel_size)
        )
        dense_layer = Concatenate(axis=-1)([dense_layer, intermediate_layer])
    return dense_layer

dense_unet(input_shape, n_classes=dense_unet_config.n_classes, n_filters=dense_unet_config.n_filters, filter_growth=dense_unet_config.filter_growth, kernel_size=dense_unet_config.kernel_size, block_structure=dense_unet_config.block_structure, internal_activation=dense_unet_config.internal_activation, output_activation=dense_unet_config.output_activation, output_dtype=dense_unet_config.output_dtype, activation_parameters=unet_config.activation_parameters, padding=dense_unet_config.padding, pool_size=dense_unet_config.pool_size, batch_normalize=dense_unet_config.batch_normalize, dropout_rate=dense_unet_config.dropout_rate, kernel_initializer=dense_unet_config.kernel_initializer, augmentation_layer=dense_unet_config.augmentation_layer, feature_transform=dense_unet_config.feature_transform, kernel_regularizer=unet_config.kernel_regularizer, bias_regularizer=unet_config.bias_regularizer, activity_regularizer=unet_config.activity_regularizer, regularization_type=unet_config.regularization_type, regularization_lambda=unet_config.regularization_lambda)

Create a Dense U-Net architecture based on custom feature dimensions.

Parameters:

Name	Type	Description	Default
input_shape	Tuple[int, int, int]	Input feature dimensions, in [width, height, n_bands] order.	required
n_classes	int	Number of unique input classes. Set to 1 for regression models.	dense_unet_config.n_classes
n_filters	int	Filter count for initial convolutions. Increases if filter_growth=True.	dense_unet_config.n_filters
filter_growth	bool	Doubles the number of convolution filters.	dense_unet_config.filter_growth
kernel_size	Tuple[int, int]	Kernel size used for local neighborhood convolutions.	dense_unet_config.kernel_size
block_structure	List[int]	Layer count in each network block. The tuple length is the number of network blocks, and the value of each element is the number of layers in that block. block_structure=(2, 3, 4) configures a network where the first block has two layers, the second block has three layers, and the third block has four layers.	dense_unet_config.block_structure
internal_activation	str	Activation function used between layers. See Keras docs for details.	dense_unet_config.internal_activation
output_activation	str	Activation function for the final output layer.	dense_unet_config.output_activation
output_dtype	str	Data type of the final activation function.	dense_unet_config.output_dtype
activation_parameters	dict	Parameters to pass to activation functions with non-standard keywords (e.g. LeakyReLU accepts the alpha parameter, which you could set with activation_parameters = {'alpha': 0.2}).	unet_config.activation_parameters
padding	str	One of ["valid", "zeros", "symmetric", "reflect"]. valid means no padding. See www.tensorflow.org/api_docs/python/tf/pad for others.	dense_unet_config.padding
pool_size	Tuple[int, int]	Pooling and upsampling size during each downsampling/upsampling step.	dense_unet_config.pool_size
batch_normalize	bool	Use batch normalization layers, which are typically applied after convolutions.	dense_unet_config.batch_normalize
dropout_rate	float	The 0-1 proportion of layer weights to randomly set to 0 in each training step.	dense_unet_config.dropout_rate
kernel_initializer	str	Method for initializing weights. See keras.io/api/layers/initializers	dense_unet_config.kernel_initializer
augmentation_layer	bool	Whether the augmentation layer is on/off	dense_unet_config.augmentation_layer
feature_transform	bool	Apply 1x1 convolutions as the first model layer	dense_unet_config.feature_transform
kernel_initializer	str	Method for initializing layer weights (see keras.io/api/layers/initializers).	dense_unet_config.kernel_initializer
kernel_regularizer	bool	if the conv layer should apply regularization on the weights of kernel	unet_config.kernel_regularizer
bias_regularizer	bool	if the conv layer should apply regularization on the bias values	unet_config.bias_regularizer
activity_regularizer	bool	if the conv layer should apply regularization on the layer output	unet_config.activity_regularizer
regularization_type	bool	norm of regularizer (l1, l2 or l1_l2)	unet_config.regularization_type
regularization_lambda	bool	regularization parameter is a scaler that defines the amount of regularization that is applied to the kernel weights, bias, and output. lambda reduces the variance of the estiamted parameters. Increasing this value encourages weights to towards smaller values. Very large lambda will prevent model from learning.	unet_config.regularization_lambda

Returns:

Type	Description
keras.models.Model	Keras Dense U-Net model.

Source code in myco/architectures.py

def dense_unet(
    input_shape: Tuple[int, int, int],
    n_classes: int = dense_unet_config.n_classes,
    n_filters: int = dense_unet_config.n_filters,
    filter_growth: bool = dense_unet_config.filter_growth,
    kernel_size: Tuple[int, int] = dense_unet_config.kernel_size,
    block_structure: List[int] = dense_unet_config.block_structure,
    internal_activation: str = dense_unet_config.internal_activation,
    output_activation: str = dense_unet_config.output_activation,
    output_dtype: str = dense_unet_config.output_dtype,
    activation_parameters: dict = unet_config.activation_parameters,
    padding: str = dense_unet_config.padding,
    pool_size: Tuple[int, int] = dense_unet_config.pool_size,
    batch_normalize: bool = dense_unet_config.batch_normalize,
    dropout_rate: float = dense_unet_config.dropout_rate,
    kernel_initializer: str = dense_unet_config.kernel_initializer,
    augmentation_layer: bool = dense_unet_config.augmentation_layer,
    feature_transform: bool = dense_unet_config.feature_transform,
    kernel_regularizer: bool = unet_config.kernel_regularizer,
    bias_regularizer: bool = unet_config.bias_regularizer,
    activity_regularizer: bool = unet_config.activity_regularizer,
    regularization_type: bool = unet_config.regularization_type,
    regularization_lambda: bool = unet_config.regularization_lambda,
) -> keras.models.Model:
    """Create a Dense U-Net architecture based on custom feature dimensions.

    Args:
        input_shape: Input feature dimensions, in [width, height, n_bands] order.
        n_classes: Number of unique input classes. Set to 1 for regression models.
        n_filters: Filter count for initial convolutions. Increases if `filter_growth=True`.
        filter_growth: Doubles the number of convolution filters.
        kernel_size: Kernel size used for local neighborhood convolutions.
        block_structure: Layer count in each network block. The tuple length is the number of
            network blocks, and the value of each element is the number of layers in that block.
            `block_structure=(2, 3, 4)` configures a network where the first block has two layers,
            the second block has three layers, and the third block has four layers.
        internal_activation: Activation function used between layers. See Keras docs for details.
        output_activation: Activation function for the final output layer.
        output_dtype: Data type of the final activation function.
        activation_parameters: Parameters to pass to activation functions with non-standard keywords
            (e.g. `LeakyReLU` accepts the `alpha` parameter, which you could set with
            activation_parameters = {'alpha': 0.2}).
        padding: One of ["valid", "zeros", "symmetric", "reflect"]. valid means no padding.
            See https://www.tensorflow.org/api_docs/python/tf/pad for others.
        pool_size: Pooling and upsampling size during each downsampling/upsampling step.
        batch_normalize: Use batch normalization layers, which are typically applied after convolutions.
        dropout_rate: The 0-1 proportion of layer weights to randomly set to 0 in each training step.
        kernel_initializer: Method for initializing weights.
            See https://keras.io/api/layers/initializers
        augmentation_layer: Whether the augmentation layer is on/off
        feature_transform: Apply 1x1 convolutions as the first model layer
        kernel_initializer: Method for initializing layer weights (see https://keras.io/api/layers/initializers).
        kernel_regularizer: if the conv layer should apply regularization on the weights of kernel
        bias_regularizer: if the conv layer should apply regularization on the bias values
        activity_regularizer: if the conv layer should apply regularization on the layer output
        regularization_type: norm of regularizer (l1, l2 or l1_l2)
        regularization_lambda: regularization parameter is a scaler that defines the amount of regularization
            that is applied to the kernel weights, bias, and output. lambda reduces the variance of the
            estiamted parameters. Increasing this value encourages weights to towards smaller values.
            Very large lambda will prevent model from learning.

    Returns:
        Keras Dense U-Net model.
    """
    width, height, n_bands = input_shape

    # validate block structure based on input dimensions
    n_blocks = len(block_structure)
    minimum_width = width / 2 ** len(block_structure)
    assert minimum_width >= 2, (
        f"Convolution width in the last encoding block is less than 2."
        f"Reduce the number of blocks in block_structure (currently {n_blocks})."
    )

    # raise a warning if batch norm and dropouts are set
    if batch_normalize and dropout_rate > 0:
        raise ArchitectureWarning(
            "Batch normalization and dropouts set. Use one or the other."
        )

    # dict to easily and frequenly pass arguments to the conv2d layer as **kwargs
    conv2d_options = {
        "n_filters": n_filters,
        "kernel_size": kernel_size,
        "padding": padding,
        "activation": internal_activation,
        "activation_parameters": activation_parameters,
        "batch_normalize": batch_normalize,
        "dropout_rate": dropout_rate,
        "kernel_initializer": kernel_initializer,
        "kernel_regularizer": kernel_regularizer,
        "bias_regularizer": bias_regularizer,
        "activity_regularizer": activity_regularizer,
        "regularization_type": regularization_type,
        "regularization_lambda": regularization_lambda,
    }
    transition_options = conv2d_options.copy()
    transition_options["kernel_size"] = (1, 1)
    transition_options["dropout_rate"] = 0

    inlayer = keras.layers.Input(shape=input_shape)

    layers_pass_through = list()
    if augmentation_layer:
        layers_pass_through.append(aug_layer())

    if feature_transform:
        encoder = channel_transform_layer(
            inlayer, dropout_rate=dropout_rate, batch_normalize=batch_normalize
        )
    else:
        encoder = inlayer

    # create encoding layers, where each encoder block has a number of subblocks
    for num_sublayers in block_structure:

        # create a dense block
        encoder = dense_2d_block(encoder, conv2d_options, num_sublayers)
        layers_pass_through.append(encoder)

        # transition block
        encoder = conv2d_layer(encoder, **transition_options)

        # pooling
        encoder = MaxPooling2D(pool_size=pool_size)(encoder)

        if filter_growth:
            conv2d_options["n_filters"] *= 2
            transition_options["n_filters"] *= 2

    # encoder/decoder transition
    transition = dense_2d_block(encoder, conv2d_options, block_structure[-1])
    decoder = transition

    # set decoder layers options
    if augmentation_layer:
        decoder_layers = layers_pass_through[1:]
    else:
        decoder_layers = layers_pass_through

    # don't apply dropouts in the decoding layers
    conv2d_options["dropout_rate"] = 0

    # block-wise decoding
    for num_subblocks, layer_passed_through in zip(
        reversed(block_structure), reversed(decoder_layers)
    ):
        if filter_growth:
            conv2d_options["n_filters"] //= 2
            transition_options["n_filters"] //= 2

        # upsample
        decoder = UpSampling2D(size=pool_size, interpolation="bilinear")(decoder)

        # create a dense block and concatenate
        decoder = conv2d_layer(
            decoder,
            **conv2d_options,
            padding_dim=compute_padding_dim(kernel_size, decoder, layer_passed_through),
        )
        decoder = Concatenate()([layer_passed_through, decoder])

        # transition block
        decoder = dense_2d_block(decoder, transition_options, num_subblocks)

    # final output convolutions
    output_layer = decoder
    output_layer = conv2d_layer(
        output_layer,
        **conv2d_options,
        padding_dim=compute_padding_dim(kernel_size, output_layer, inlayer),
    )
    output_layer = Conv2D(
        filters=n_classes,
        kernel_size=(1, 1),
        padding="same",
    )(output_layer)
    output_layer = Activation(output_activation, dtype=output_dtype)(output_layer)
    model = keras.models.Model(inputs=[inlayer], outputs=[output_layer])

    return model

get_arch_config(name)

Return the configuration parameters for a specific architecture.

Parameters:

Name	Type	Description	Default
name	str	the model architecture. Get available options from get_arch_names().	required

Returns:

Type	Description
dict	Default architecture parameters as a dictionary.

Source code in myco/architectures.py

def get_arch_config(name: str) -> dict:
    """Return the configuration parameters for a specific architecture.

    Args:
        name: the model architecture. Get available options from get_arch_names().

    Returns:
        Default architecture parameters as a dictionary.
    """
    assert name in get_arch_names(), f"Invalid architecture: {name}"
    return SUPPORTED[name]["config"]

get_arch_model(name)

Return an un-compiled keras model for a specific architecture.

Parameters:

Name	Type	Description	Default
name	str	the model architecture. Get available options from get_arch_names().	required

Returns:

Type	Description
keras.models.Model	The corresponding keras model.

Source code in myco/architectures.py

def get_arch_model(name: str) -> keras.models.Model:
    """Return an un-compiled keras model for a specific architecture.

    Args:
        name: the model architecture. Get available options from get_arch_names().

    Returns:
        The corresponding keras model.
    """
    assert name in get_arch_names(), f"Invalid architecture: {name}"
    return SUPPORTED[name]["model"]

get_arch_names()

Return a list of valid model architecture names

Source code in myco/architectures.py

def get_arch_names() -> list:
    """Return a list of valid model architecture names"""
    return list(SUPPORTED.keys())

get_regularizer(norm='l1', l=1.0)

Add regularization to a convolution layer

Parameters:

Name	Type	Description	Default
norm	str	the type of normalization to apply choices: [l1, l2, l1_l2, both]	'l1'
l	float	the regularization lambda penalty to apply	1.0

Returns:

Type	Description
tf.keras.regularizers.Regularizer	beta coefficient regularization layer

Source code in myco/architectures.py

def get_regularizer(
    norm: str = "l1", l: float = 1.0
) -> tf.keras.regularizers.Regularizer:
    """Add regularization to a convolution layer

    Args:
        norm: the type of normalization to apply
            choices: [l1, l2, l1_l2, both]
        l: the regularization lambda penalty to apply

    Returns:
        beta coefficient regularization layer
    """
    if norm.lower() == "l1":
        return tf.keras.regularizers.l1(l=l)
    elif norm.lower() == "l2":
        return tf.keras.regularizers.l2(l=l)
    elif norm.lower() == "l1_l2" or norm.lower() == "both":
        return tf.keras.regularizers.l1_l2(l1=l, l2=l)

pad_2d(layer, padding_dim=[1, 1], padding_mode='CONSTANT')

Pads the input layer using the mode and padding dimensions

Parameters:

Name	Type	Description	Default
layer	Layer	input layer to be padded	required
padding_dim	Tuple[int, int]	dimensions of 2d padding including height and width	[1, 1]
padding_mode	str	padding mode using either symmetric, reflect or constant(zero padded)	'CONSTANT'

Returns:

Type	Description
	padded layer

Source code in myco/architectures.py

def pad_2d(
    layer: Layer, padding_dim: Tuple[int, int] = [1, 1], padding_mode: str = "CONSTANT"
):
    """Pads the input layer using the mode and padding dimensions

    Args:
        layer: input layer to be padded
        padding_dim: dimensions of 2d padding including height and width
        padding_mode: padding mode using either symmetric, reflect or constant(zero padded)

    Returns:
        padded layer
    """
    padding_mode = str.upper(padding_mode)

    assert padding_mode in [
        "SAME",
        "ZEROS",
        "SYMMETRIC",
        "REFLECT",
    ], "padding option not valid"

    if padding_mode in ["SAME", "ZEROS"]:
        padding_mode = "CONSTANT"

    return pad(
        layer,
        [[0, 0], [0, padding_dim[0]], [0, padding_dim[1]], [0, 0]],
        padding_mode,
    )

residuals_layer(input, building_block)

Add a residual estimation layer

openaccess.thecvf.com/content_cvpr_2016/papers/He_Deep_Residual_Learning_CVPR_2016_paper.pdf

Parameters:

Name	Type	Description	Default
input	Layer	the block's input layer prior to 2D convolutions	required
building_block	Layer	the post-2D convolutions layer	required

Returns:

Type	Description
Layer	the sum of the input layers

Source code in myco/architectures.py

def residuals_layer(input: Layer, building_block: Layer) -> Layer:
    """Add a residual estimation layer

    https://openaccess.thecvf.com/content_cvpr_2016/papers/He_Deep_Residual_Learning_CVPR_2016_paper.pdf

    Args:
        input: the block's input layer prior to 2D convolutions
        building_block: the post-2D convolutions layer

    Returns:
        the sum of the input layers
    """
    residual = Add()([input, building_block])
    return residual

unet(input_shape, n_classes=unet_config.n_classes, n_filters=unet_config.n_filters, filter_growth=unet_config.filter_growth, kernel_size=unet_config.kernel_size, block_structure=unet_config.block_structure, internal_activation=unet_config.internal_activation, output_activation=unet_config.output_activation, output_dtype=unet_config.output_dtype, activation_parameters=unet_config.activation_parameters, padding=unet_config.padding, pool_size=unet_config.pool_size, batch_normalize=unet_config.batch_normalize, dropout_rate=unet_config.dropout_rate, kernel_initializer=unet_config.kernel_initializer, augmentation_layer=unet_config.augmentation_layer, feature_transform=unet_config.feature_transform, residuals_transform=unet_config.residuals_transform, kernel_regularizer=unet_config.kernel_regularizer, bias_regularizer=unet_config.bias_regularizer, activity_regularizer=unet_config.activity_regularizer, regularization_type=unet_config.regularization_type, regularization_lambda=unet_config.regularization_lambda)

Create a U-Net architecture based on custom feature dimensions.

Parameters:

Name	Type	Description	Default
input_shape	Tuple[int, int, int]	Input feature dimensions, in [width, height, n_bands] order.	required
n_classes	int	Number of unique input classes. Set to 1 for regression models.	unet_config.n_classes
n_filters	int	Filter count for initial convolutions. Increases if filter_growth=True.	unet_config.n_filters
filter_growth	bool	Doubles the number of convolution filters.	unet_config.filter_growth
kernel_size	Tuple[int, int]	Kernel size used for local neighborhood convolutions.	unet_config.kernel_size
block_structure	List[int]	Layer count in each network block. The tuple length is the number of network blocks, and the value of each element is the number of layers in that block. block_structure=(2, 3, 4) configures a network where the first block has two layers, the second block has three layers, and the third block has four layers.	unet_config.block_structure
internal_activation	str	Activation function used between layers. See Keras docs for details.	unet_config.internal_activation
output_activation	str	Activation function for the final output layer.	unet_config.output_activation
output_dtype	str	Data type of the final activation function.	unet_config.output_dtype
activation_parameters	dict	Parameters to pass to activation functions with non-standard keywords (e.g. LeakyReLU accepts the alpha parameter, which you could set with activation_parameters = {'alpha': 0.2}).	unet_config.activation_parameters
padding	str	One of ["valid", "zeros", "symmetric", "reflect"]. valid means no padding. See www.tensorflow.org/api_docs/python/tf/pad for others.	unet_config.padding
pool_size	Tuple[int, int]	Pooling and upsampling size during each downsampling/upsampling step.	unet_config.pool_size
batch_normalize	bool	Use batch normalization layers, which are typically applied after convolutions.	unet_config.batch_normalize
dropout_rate	float	The 0-1 proportion of layer weights to randomly set to 0 in each training step.	unet_config.dropout_rate
kernel_initializer	str	Method for initializing weights. See keras.io/api/layers/initializers	unet_config.kernel_initializer
augmentation_layer	bool	Whether the augmentation layer is on/off	unet_config.augmentation_layer
feature_transform	bool	Apply 1x1 convolutions as the first model layer	unet_config.feature_transform
residuals_transform	bool	Apply a residuals shortcut layer	unet_config.residuals_transform
kernel_initializer	str	Method for initializing layer weights (see keras.io/api/layers/initializers).	unet_config.kernel_initializer
kernel_regularizer	bool	if the conv layer should apply regularization on the weights of kernel	unet_config.kernel_regularizer
bias_regularizer	bool	if the conv layer should apply regularization on the bias values	unet_config.bias_regularizer
activity_regularizer	bool	if the conv layer should apply regularization on the layer output	unet_config.activity_regularizer
regularization_type	bool	norm of regularizer (l1, l2 or l1_l2)	unet_config.regularization_type
regularization_lambda	bool	regularization parameter is a scaler that defines the amount of regularization that is applied to the kernel weights, bias, and output. lambda reduces the variance of the estiamted parameters. Increasing this value encourages weights to towards smaller values. Very large lambda will prevent model from learning.	unet_config.regularization_lambda

Returns:

Type	Description
keras.models.Model	Keras U-Net model.

Source code in myco/architectures.py

def unet(
    input_shape: Tuple[int, int, int],
    n_classes: int = unet_config.n_classes,
    n_filters: int = unet_config.n_filters,
    filter_growth: bool = unet_config.filter_growth,
    kernel_size: Tuple[int, int] = unet_config.kernel_size,
    block_structure: List[int] = unet_config.block_structure,
    internal_activation: str = unet_config.internal_activation,
    output_activation: str = unet_config.output_activation,
    output_dtype: str = unet_config.output_dtype,
    activation_parameters: dict = unet_config.activation_parameters,
    padding: str = unet_config.padding,
    pool_size: Tuple[int, int] = unet_config.pool_size,
    batch_normalize: bool = unet_config.batch_normalize,
    dropout_rate: float = unet_config.dropout_rate,
    kernel_initializer: str = unet_config.kernel_initializer,
    augmentation_layer: bool = unet_config.augmentation_layer,
    feature_transform: bool = unet_config.feature_transform,
    residuals_transform: bool = unet_config.residuals_transform,
    kernel_regularizer: bool = unet_config.kernel_regularizer,
    bias_regularizer: bool = unet_config.bias_regularizer,
    activity_regularizer: bool = unet_config.activity_regularizer,
    regularization_type: bool = unet_config.regularization_type,
    regularization_lambda: bool = unet_config.regularization_lambda,
) -> keras.models.Model:
    """Create a U-Net architecture based on custom feature dimensions.

    Args:
        input_shape: Input feature dimensions, in [width, height, n_bands] order.
        n_classes: Number of unique input classes. Set to 1 for regression models.
        n_filters: Filter count for initial convolutions. Increases if `filter_growth=True`.
        filter_growth: Doubles the number of convolution filters.
        kernel_size: Kernel size used for local neighborhood convolutions.
        block_structure: Layer count in each network block. The tuple length is the number of
            network blocks, and the value of each element is the number of layers in that block.
            `block_structure=(2, 3, 4)` configures a network where the first block has two layers,
            the second block has three layers, and the third block has four layers.
        internal_activation: Activation function used between layers. See Keras docs for details.
        output_activation: Activation function for the final output layer.
        output_dtype: Data type of the final activation function.
        activation_parameters: Parameters to pass to activation functions with non-standard keywords
            (e.g. `LeakyReLU` accepts the `alpha` parameter, which you could set with
            activation_parameters = {'alpha': 0.2}).
        padding: One of ["valid", "zeros", "symmetric", "reflect"]. valid means no padding.
            See https://www.tensorflow.org/api_docs/python/tf/pad for others.
        pool_size: Pooling and upsampling size during each downsampling/upsampling step.
        batch_normalize: Use batch normalization layers, which are typically applied after convolutions.
        dropout_rate: The 0-1 proportion of layer weights to randomly set to 0 in each training step.
        kernel_initializer: Method for initializing weights.
            See https://keras.io/api/layers/initializers
        augmentation_layer: Whether the augmentation layer is on/off
        feature_transform: Apply 1x1 convolutions as the first model layer
        residuals_transform: Apply a residuals shortcut layer
        kernel_initializer: Method for initializing layer weights (see https://keras.io/api/layers/initializers).
        kernel_regularizer: if the conv layer should apply regularization on the weights of kernel
        bias_regularizer: if the conv layer should apply regularization on the bias values
        activity_regularizer: if the conv layer should apply regularization on the layer output
        regularization_type: norm of regularizer (l1, l2 or l1_l2)
        regularization_lambda: regularization parameter is a scaler that defines the amount of regularization
            that is applied to the kernel weights, bias, and output. lambda reduces the variance of the
            estiamted parameters. Increasing this value encourages weights to towards smaller values.
            Very large lambda will prevent model from learning.

    Returns:
        Keras U-Net model.
    """
    width, height, n_bands = input_shape
    # validate block structure based on input dimensions
    n_blocks = len(block_structure)
    minimum_width = width / 2 ** len(block_structure)
    assert minimum_width >= 2, (
        f"Convolution width in the last encoding block is less than 2."
        f"Reduce the number of blocks in block_structure (currently {n_blocks})."
    )

    # raise a warning if batch norm and dropouts are set
    if batch_normalize and dropout_rate > 0:
        raise ArchitectureWarning(
            "Batch normalization and dropouts set. Use one or the other."
        )

    # dict to easily and frequently pass arguments to the conv2d layer as **kwargs
    conv2d_options = {
        "n_filters": n_filters,
        "kernel_size": kernel_size,
        "padding": padding,
        "activation": internal_activation,
        "activation_parameters": activation_parameters,
        "batch_normalize": batch_normalize,
        "dropout_rate": dropout_rate,
        "kernel_initializer": kernel_initializer,
        "kernel_regularizer": kernel_regularizer,
        "bias_regularizer": bias_regularizer,
        "activity_regularizer": activity_regularizer,
        "regularization_type": regularization_type,
        "regularization_lambda": regularization_lambda,
    }
    inlayer = keras.layers.Input(shape=input_shape)

    layers_pass_through = list()
    if augmentation_layer:
        layers_pass_through.append(aug_layer())

    if feature_transform:
        encoder = channel_transform_layer(
            inlayer, dropout_rate=dropout_rate, batch_normalize=batch_normalize
        )
    else:
        encoder = inlayer

    # transform the input layer to the same filter dimensions for the residual block
    if residuals_transform:
        encoder = conv2d_layer(encoder, **conv2d_options)

    # create encoding layers, where each encoder block has a number of subblocks
    for num_sublayers in block_structure:

        # index the first layer to pass to the residuals block
        entry_layer = encoder

        # each subblock applies series of convolutions
        for _sublayer in range(num_sublayers):
            encoder = conv2d_layer(encoder, **conv2d_options)

        # create the residual layer
        if residuals_transform:
            encoder = residuals_layer(entry_layer, encoder)

        # each encoder block passes its pre-pooled layers through to the decoder
        layers_pass_through.append(encoder)
        encoder = MaxPooling2D(pool_size=pool_size)(encoder)
        if filter_growth:
            conv2d_options["n_filters"] *= 2

    # create transition Layers
    transition = encoder
    for _sublayer in range(block_structure[-1]):
        transition = conv2d_layer(transition, **conv2d_options)

    # don't apply dropouts in the decoding layers
    conv2d_options["dropout_rate"] = 0

    # set decoder layers options
    if augmentation_layer:
        decoder_layers = layers_pass_through[1:]
    else:
        decoder_layers = layers_pass_through

    # create decoding layers, where each decoder block has subblocks in reverse order of encoder
    decoder = transition

    for num_subblocks, layer_passed_through in zip(
        reversed(block_structure), reversed(decoder_layers)
    ):
        if filter_growth:
            conv2d_options["n_filters"] //= 2
        decoder = UpSampling2D(size=pool_size, interpolation="bilinear")(decoder)
        decoder = conv2d_layer(
            decoder,
            **conv2d_options,
            padding_dim=compute_padding_dim(kernel_size, decoder, layer_passed_through),
        )
        decoder = Concatenate()([layer_passed_through, decoder])

        # index the concatenated layer to pass to the residuals block
        entry_layer = conv2d_layer(decoder, **conv2d_options)

        # each subblock has a series of convolutions
        for _sublayer in range(num_subblocks):
            decoder = conv2d_layer(decoder, **conv2d_options)

        # create the residual layer
        if residuals_transform:
            decoder = residuals_layer(entry_layer, decoder)

    # final output convolutions
    output_layer = decoder
    output_layer = conv2d_layer(
        output_layer,
        **conv2d_options,
        padding_dim=compute_padding_dim(kernel_size, output_layer, inlayer),
    )

    output_layer = Conv2D(
        filters=n_classes,
        kernel_size=(1, 1),
    )(output_layer)
    output_layer = Activation(output_activation, dtype=output_dtype)(output_layer)
    model = keras.models.Model(inputs=[inlayer], outputs=[output_layer])

    return model