Skip to content

mod_dense_layer_with_single_activations

Lines: 164 | Visibility: private

Implementation of a dense layer type for a neural network. author: Jonas Finkler, Alexander Knoll

Dependencies

Uses

module~~mod_dense_layer_with_single_activations~~UsesGraph module~mod_dense_layer_with_single_activations mod_dense_layer_with_single_activations module~mod_activationfunctions mod_activationfunctions module~mod_dense_layer_with_single_activations->module~mod_activationfunctions module~mod_decorations mod_decorations module~mod_dense_layer_with_single_activations->module~mod_decorations module~mod_dense_layer mod_dense_layer module~mod_dense_layer_with_single_activations->module~mod_dense_layer module~mod_global_warning mod_global_warning module~mod_dense_layer_with_single_activations->module~mod_global_warning module~mod_io mod_io module~mod_dense_layer_with_single_activations->module~mod_io module~mod_linalg mod_linalg module~mod_dense_layer_with_single_activations->module~mod_linalg module~mod_precision mod_precision module~mod_dense_layer_with_single_activations->module~mod_precision module~mod_throw mod_throw module~mod_dense_layer_with_single_activations->module~mod_throw module~mod_activationfunctions->module~mod_throw module~mod_activationfunction_abstract mod_activationfunction_abstract module~mod_activationfunctions->module~mod_activationfunction_abstract module~mod_activationfunction_linear mod_activationfunction_linear module~mod_activationfunctions->module~mod_activationfunction_linear module~mod_activationfunction_relu mod_activationfunction_relu module~mod_activationfunctions->module~mod_activationfunction_relu module~mod_activationfunction_sigmoid mod_activationfunction_sigmoid module~mod_activationfunctions->module~mod_activationfunction_sigmoid module~mod_activationfunction_softplus mod_activationfunction_softplus module~mod_activationfunctions->module~mod_activationfunction_softplus module~mod_activationfunction_square mod_activationfunction_square module~mod_activationfunctions->module~mod_activationfunction_square module~mod_activationfunction_stanh mod_activationfunction_stanh module~mod_activationfunctions->module~mod_activationfunction_stanh module~mod_activationfunction_tanh mod_activationfunction_tanh module~mod_activationfunctions->module~mod_activationfunction_tanh module~mod_decorations->module~mod_io module~mod_decorations->module~mod_precision module~mod_decorations->module~mod_throw module~mod_dense_layer->module~mod_activationfunctions module~mod_dense_layer->module~mod_decorations module~mod_dense_layer->module~mod_global_warning module~mod_dense_layer->module~mod_io module~mod_dense_layer->module~mod_linalg module~mod_dense_layer->module~mod_precision module~mod_dense_layer->module~mod_throw module~mod_constants mod_constants module~mod_dense_layer->module~mod_constants module~mod_initialize_array mod_initialize_array module~mod_dense_layer->module~mod_initialize_array module~mod_nan_check mod_nan_check module~mod_dense_layer->module~mod_nan_check module~mod_optional_args mod_optional_args module~mod_dense_layer->module~mod_optional_args module~mod_random mod_random module~mod_dense_layer->module~mod_random module~mod_global_warning->module~mod_io module~mod_global_warning->module~mod_throw module~mod_abstract_warning mod_abstract_warning module~mod_global_warning->module~mod_abstract_warning module~mod_io->module~mod_precision iso_fortran_env iso_fortran_env module~mod_io->iso_fortran_env module~mod_io->module~mod_optional_args module~mod_linalg->module~mod_precision module~mod_linalg->module~mod_throw module~cublas_wrapper cublas_wrapper module~mod_linalg->module~cublas_wrapper module~mod_settings mod_settings module~mod_linalg->module~mod_settings iso_c_binding iso_c_binding module~mod_precision->iso_c_binding module~mod_precision->iso_fortran_env module~mod_throw->module~mod_io module~mod_throw->module~mod_precision module~mod_throw->iso_c_binding module~mod_throw->iso_fortran_env module~mod_mpi mod_mpi module~mod_throw->module~mod_mpi module~mod_sort mod_sort module~mod_throw->module~mod_sort module~cublas_wrapper->iso_c_binding module~cublas_wrapper->module~mod_settings module~mod_activationfunction_linear->module~mod_precision module~mod_activationfunction_linear->module~mod_activationfunction_abstract module~mod_activationfunction_relu->module~mod_precision module~mod_activationfunction_relu->module~mod_activationfunction_abstract module~mod_activationfunction_sigmoid->module~mod_precision module~mod_activationfunction_sigmoid->module~mod_activationfunction_abstract module~mod_activationfunction_softplus->module~mod_precision module~mod_activationfunction_softplus->module~mod_activationfunction_abstract module~mod_activationfunction_square->module~mod_precision module~mod_activationfunction_square->module~mod_activationfunction_abstract module~mod_activationfunction_stanh->module~mod_precision module~mod_activationfunction_stanh->module~mod_activationfunction_abstract module~mod_activationfunction_tanh->module~mod_precision module~mod_activationfunction_tanh->module~mod_activationfunction_abstract module~mod_constants->module~mod_precision module~mod_constants->iso_c_binding module~mod_initialize_array->module~mod_io module~mod_initialize_array->module~mod_precision module~mod_initialize_array->module~mod_throw module~mod_initialize_array->module~mod_mpi module~mod_initialize_array->module~mod_random module~mod_parameter_file mod_parameter_file module~mod_initialize_array->module~mod_parameter_file module~mod_mpi->module~mod_precision mpi_f08 mpi_f08 module~mod_mpi->mpi_f08 module~mod_nan_check->module~mod_io module~mod_nan_check->module~mod_precision module~mod_nan_check->module~mod_throw ieee_arithmetic ieee_arithmetic module~mod_nan_check->ieee_arithmetic module~mod_random->module~mod_precision module~mod_random->module~mod_constants module~mod_settings->module~mod_io module~mod_settings->module~mod_precision module~mod_settings->module~mod_throw module~mod_settings->module~mod_constants module~mod_cpu_vendor mod_cpu_vendor module~mod_settings->module~mod_cpu_vendor module~mod_periodic_table mod_periodic_table module~mod_settings->module~mod_periodic_table module~mod_sort->module~mod_precision module~mod_cpu_vendor->iso_c_binding module~mod_parameter_file->module~mod_io module~mod_parameter_file->module~mod_precision module~mod_parameter_file->module~mod_throw module~mod_parameter_file->module~mod_nan_check module~mod_abstract_broadcast_file mod_abstract_broadcast_file module~mod_parameter_file->module~mod_abstract_broadcast_file module~mod_periodic_table->module~mod_io module~mod_periodic_table->module~mod_precision module~mod_periodic_table->module~mod_throw module~mod_periodic_table->module~mod_constants module~mod_array_helpers mod_array_helpers module~mod_periodic_table->module~mod_array_helpers module~mod_abstract_broadcast_file->module~mod_io module~mod_abstract_broadcast_file->module~mod_throw module~mod_abstract_broadcast_file->module~mod_mpi module~mod_genericfile mod_genericfile module~mod_abstract_broadcast_file->module~mod_genericfile module~mod_sanitize mod_sanitize module~mod_abstract_broadcast_file->module~mod_sanitize module~mod_array_helpers->module~mod_precision module~mod_array_helpers->module~mod_throw module~mod_genericfile->module~mod_io module~mod_genericfile->module~mod_throw module~mod_genericfile->module~mod_mpi module~mod_sanitize->module~mod_throw

Used By

module~~mod_dense_layer_with_single_activations~~UsedByGraph module~mod_dense_layer_with_single_activations mod_dense_layer_with_single_activations module~mod_layers mod_layers module~mod_layers->module~mod_dense_layer_with_single_activations module~mod_neural_network mod_neural_network module~mod_neural_network->module~mod_layers module~mod_hdnn mod_hdnn module~mod_hdnn->module~mod_neural_network module~mod_initialize mod_initialize module~mod_initialize->module~mod_hdnn module~mod_predict_mode mod_predict_mode module~mod_predict_mode->module~mod_initialize module~mod_write_outfiles mod_write_outfiles module~mod_predict_mode->module~mod_write_outfiles module~mod_runner_interface_bindings mod_runner_interface_bindings module~mod_runner_interface_bindings->module~mod_initialize module~mod_train_energy_multiple_structures mod_train_energy_multiple_structures module~mod_train_energy_multiple_structures->module~mod_initialize module~mod_train_forces_multiple_structures mod_train_forces_multiple_structures module~mod_train_forces_multiple_structures->module~mod_initialize module~mod_train_forces_multiple_structures->module~mod_train_energy_multiple_structures module~mod_train_loop_4g_charges mod_train_loop_4g_charges module~mod_train_loop_4g_charges->module~mod_initialize module~mod_cost_calculator mod_cost_calculator module~mod_train_loop_4g_charges->module~mod_cost_calculator module~mod_output_files_and_cost_4g_charges mod_output_files_and_cost_4g_charges module~mod_train_loop_4g_charges->module~mod_output_files_and_cost_4g_charges module~mod_train_loop_atomic_property mod_train_loop_atomic_property module~mod_train_loop_atomic_property->module~mod_initialize module~mod_train_loop_atomic_property->module~mod_cost_calculator module~mod_output_files_and_cost mod_output_files_and_cost module~mod_train_loop_atomic_property->module~mod_output_files_and_cost module~mod_train_loop_short_range mod_train_loop_short_range module~mod_train_loop_short_range->module~mod_initialize module~mod_train_loop_short_range->module~mod_train_energy_multiple_structures module~mod_train_loop_short_range->module~mod_train_forces_multiple_structures module~mod_train_loop_short_range->module~mod_cost_calculator module~mod_train_loop_short_range->module~mod_output_files_and_cost module~mod_output_files_and_cost_4g_short_range mod_output_files_and_cost_4g_short_range module~mod_train_loop_short_range->module~mod_output_files_and_cost_4g_short_range module~mod_train_mode mod_train_mode module~mod_train_mode->module~mod_initialize module~mod_train_mode->module~mod_train_loop_4g_charges module~mod_train_mode->module~mod_train_loop_atomic_property module~mod_train_mode->module~mod_train_loop_short_range module~mod_write_outfiles_training mod_write_outfiles_training module~mod_train_mode->module~mod_write_outfiles_training module~mod_write_outfiles->module~mod_initialize program~runner RuNNer program~runner->module~mod_initialize program~runner->module~mod_predict_mode program~runner->module~mod_train_mode module~mod_lammps_interface mod_lammps_interface module~mod_lammps_interface->module~mod_runner_interface_bindings module~mod_write_outfiles_training->module~mod_write_outfiles module~mod_cost_calculator->module~mod_write_outfiles_training module~mod_output_files_and_cost->module~mod_cost_calculator module~mod_output_files_and_cost_4g_charges->module~mod_cost_calculator module~mod_output_files_and_cost_4g_short_range->module~mod_cost_calculator

Derived Types Summary

Name Description
DenseLayerWithSingleActivations A dense (fully-connected) output layer for a feed-forward neural network. An output layer is...

Procedures Summary

Name Type Description
init_denselayer_with_single_act Subroutine Construct a new DenseLayerWithSingleActivations object.
print_denselayer_with_single_act Subroutine Print information about a DenseLayerWithSingleActivations object.
forward_denselayer_with_single_act Subroutine Calculate the output of the dense layer for a given batch of input samples. The output of a...
forward_gradient_denselayer_with_single_act Subroutine Perform a forward pass, storing the grad wrt the neural network input.
backward_denselayer_with_single_act Subroutine Perform a batchwise backward pass for a dense layer.
backward_gradient_denselayer_with_single_act Subroutine Perform a batchwise backward pass for a dense layer including the contributions of the input...

Derived Types Details

type DenseLayerWithSingleActivations

A dense (fully-connected) output layer for a feed-forward neural network.

An output layer is just a dense layer with separate activation functions for each node.

Components

  • num_parameters : integer– The total number of free parameters in the model. This should always be set by the `constructor` procedure of the derived type. – Inherited from DenseLayer
  • num_parameters_mask : integer– The number of parameters that are not kept frozen by `mask`. – Inherited from DenseLayer
  • mask : logical (dim: (:))– A logical array specifying, for each free parameter, whether it should be optimized or not. – Inherited from DenseLayer
  • batchsize : integer– The number of samples that can pass concurrently through the layer. – Inherited from DenseLayer
  • num_forward_passes : integer– Inherited from DenseLayer
  • num_backward_passes : integer– Inherited from DenseLayer
  • num_nodes_input : integer– The number of input nodes to this layer. – Inherited from DenseLayer
  • num_nodes : integer– The number of nodes in this layer. – Inherited from DenseLayer
  • idx : integer– Layer index. – Inherited from DenseLayer
  • weights : real(DP) (dim: (:,:))– Connections to preceding layer nodes. – Inherited from DenseLayer
  • biases : real(DP) (dim: (:))– Bias values for shifting each node's offset. – Inherited from DenseLayer
  • dw : real(DP) (dim: (:,:))– Gradients of `weights`. – Inherited from DenseLayer
  • db : real(DP) (dim: (:))– Gradients of `biases`. – Inherited from DenseLayer
  • z : real(DP) (dim: (:,:))– Node values before activation. Required for backpropagation. – Inherited from DenseLayer
  • dz_dq : real(DP) (dim: (:,:))– Derivatives of the unactivated node values wrt a single coordinate. – Inherited from DenseLayer
  • da_dq : real(DP) (dim: (:,:))– Derivatives of the activated node values wrt a single coordinate. – Inherited from DenseLayer
  • x : real(DP) (dim: (:,:))– The input to the layer. Dimensions are inputsize x batchsize. – Inherited from DenseLayer
  • a : real(DP) (dim: (:,:))– Layer output. – Inherited from DenseLayer
  • grad : real(DP) (dim: (:,:))– doutput/dinput. – Inherited from DenseLayer
  • dgrad_dq : real(DP) (dim: (:,:))– doutput/dinput/dx – Inherited from DenseLayer
  • f_act : class(AbstractActivationFunction)– Activation function. – Inherited from DenseLayer
  • lsaturation_check : logical– Inherited from DenseLayer
  • forward_ptr : procedure– Pointer for delegating the forward pass to the correct implementation. We have an OpenMP-parallel and a non-parallel forward pass implementation. The serial implementation is more efficient for small batchsizes. We align the pointer whenever reallocate_inputlayer is called, which is usually only once. – Inherited from DenseLayer
  • backward_ptr : procedure– Pointer for delegating the backward pass. See forward_ptr for details. – Inherited from DenseLayer
  • f_act_single : type(ActivationFunctionPtr) (dim: (:))– Activation functions for each single node.

Type-Bound Procedures

Name Attributes Binding
initialize public ⇒ init_denselayer
get_parameters public ⇒ get_parameters_denselayer
set_parameters public ⇒ set_parameters_denselayer
get_gradients_wrt_parameters public ⇒ get_gradients_wrt_parameters_denselayer
reset public ⇒ reset_denselayer
reset_prediction public ⇒ reset_prediction_denselayer
reallocate public ⇒ reallocate_denselayer
init_parameter_hidden_layer public ⇒ init_parameter_hidden_layer
init_parameter_last_layer public ⇒ init_parameter_last_layer
init_last_layer_precondition public ⇒ init_last_layer_precondition
initialize_single_act public ⇒ init_denselayer_with_single_act
print public ⇒ print_denselayer_with_single_act
forward public ⇒ forward_denselayer_with_single_act
forward_gradient public ⇒ forward_gradient_denselayer_with_single_act
backward public ⇒ backward_denselayer_with_single_act
backward_gradient public ⇒ backward_gradient_denselayer_with_single_act

Inherits

type~~denselayerwithsingleactivations~~InheritsGraph type~denselayerwithsingleactivations DenseLayerWithSingleActivations type~activationfunctionptr ActivationFunctionPtr type~denselayerwithsingleactivations->type~activationfunctionptr f_act_single type~denselayer DenseLayer type~denselayerwithsingleactivations->type~denselayer type~abstractactivationfunction AbstractActivationFunction type~activationfunctionptr->type~abstractactivationfunction ptr type~denselayer->type~abstractactivationfunction f_act

Procedure Details

subroutine init_denselayer_with_single_act

subroutine init_denselayer_with_single_act( &
    self, &
    num_nodes_input, &
    num_nodes, &
    activation_functions, &
    batchsize, &
    idx, &
    lsaturation_check, &
    mask &
)

Construct a new DenseLayerWithSingleActivations object.

Arguments

  • self : class(DenseLayerWithSingleActivations) [inout]– The constructed layer.
  • num_nodes_input : integer [in]– The number of nodes in the preceeding layer.
  • num_nodes : integer [in]– The number of requested nodes in this layer.
  • activation_functions : character (dim: (num_nodes)) [in]– The activation functions for all nodes.
  • batchsize : integer [in]
  • idx : integer [in]
  • lsaturation_check : logical [in]
  • mask : logical (dim: (:)) [in]– logical mask

Called By

proc~~init_denselayer_with_single_act~~CalledByGraph proc~init_denselayer_with_single_act mod_dense_layer_with_single_activations::DenseLayerWithSingleActivations%init_denselayer_with_single_act proc~initialize_nn_from_architecture mod_neural_network::NeuralNetwork%initialize_nn_from_architecture proc~initialize_nn_from_architecture->proc~init_denselayer_with_single_act none~initialize mod_neural_network::NeuralNetwork%initialize none~initialize->proc~initialize_nn_from_architecture proc~initialize_from_architecture mod_hdnn::HDNN%initialize_from_architecture proc~initialize_from_architecture->none~initialize proc~initialize_from_file_hdnn mod_hdnn::HDNN%initialize_from_file_hdnn proc~initialize_from_file_hdnn->proc~initialize_from_architecture

subroutine print_denselayer_with_single_act

subroutine print_denselayer_with_single_act( &
    self &
)

Print information about a DenseLayerWithSingleActivations object.

Arguments

subroutine forward_denselayer_with_single_act

subroutine forward_denselayer_with_single_act( &
    self, &
    x &
)

Calculate the output of the dense layer for a given batch of input samples.

The output of a dense layer is calculated according to output = activation(weights * input + biases)

Arguments

Called By

proc~~forward_denselayer_with_single_act~~CalledByGraph proc~forward_denselayer_with_single_act mod_dense_layer_with_single_activations::DenseLayerWithSingleActivations%forward_denselayer_with_single_act proc~forward_gradient_denselayer_with_single_act mod_dense_layer_with_single_activations::DenseLayerWithSingleActivations%forward_gradient_denselayer_with_single_act proc~forward_gradient_denselayer_with_single_act->proc~forward_denselayer_with_single_act

subroutine forward_gradient_denselayer_with_single_act

subroutine forward_gradient_denselayer_with_single_act( &
    self, &
    x, &
    dx_dq &
)

Perform a forward pass, storing the grad wrt the neural network input.

Arguments

  • self : class(DenseLayerWithSingleActivations) [inout]
  • x : real(DP) (dim: (self%num_nodes_input,self%batchsize)) [in]– The activations of the preceeding layer.
  • dx_dq : real(DP) (dim: (self%num_nodes_input,self%batchsize)) [in]– Derivative of the preceeding layer activations `x` with respect to some variable q.

subroutine backward_denselayer_with_single_act

subroutine backward_denselayer_with_single_act( &
    self, &
    grad_follow, &
    calc_gradient_wrt_parameters &
)

Perform a batchwise backward pass for a dense layer.

Arguments

  • self : class(DenseLayerWithSingleActivations) [inout]
  • grad_follow : real(DP) (dim: (self%num_nodes,self%batchsize)) [in]
  • calc_gradient_wrt_parameters : logical [in]

subroutine backward_gradient_denselayer_with_single_act

subroutine backward_gradient_denselayer_with_single_act( &
    self, &
    dx_dq, &
    grad_follow, &
    dgrad_dq_follow &
)

Perform a batchwise backward pass for a dense layer including the contributions of the input gradient. All math is documented at the relevant places. Notation: * = Hadamard (element-wise) product . = Dot product (matrix multiplication) = da2/db2 * da3/da2dx + da3/da2 * d^2a2/db2dx = da3/da2dx * f'(z2) + da3/da2 * f''(z2) * dz2/dx

    = (d^a3/da2dx f'(z2) + da3/da2 * f''(z2) * dz2/dx) . W12
    = d^2out/dbdx . W12







     =    (d^2a3/da2dx * f'(z2)      ) . a1
        + (da3/da2 * f''(z2) * dz2/dx) . a1
        + (da3/da2 * f'(z2)          ) . da1/dx

Arguments

  • self : class(DenseLayerWithSingleActivations) [inout]
  • dx_dq : real(DP) (dim: (self%num_nodes_input,self%batchsize)) [in]
  • grad_follow : real(DP) (dim: (self%num_nodes,self%batchsize)) [in]
  • dgrad_dq_follow : real(DP) (dim: (self%num_nodes_input,self%batchsize)) [in]