Skip to content

mod_adam

Lines: 139 | Visibility: private

Implementation of the Adam optimization algorithm.

Adam is an optimizer that is widely used throughout the machine learning community. It is a stochastic gradient descent method that is based on adaptive estimation of first-order and second-order moments. Adam was first published in 2017 here. author: Alexander Knoll, Moritz R. Schaefer

Dependencies

Uses

module~~mod_adam~~UsesGraph module~mod_adam mod_adam module~mod_abstractoptimizer mod_abstractoptimizer module~mod_adam->module~mod_abstractoptimizer module~mod_decorations mod_decorations module~mod_adam->module~mod_decorations module~mod_io mod_io module~mod_adam->module~mod_io module~mod_learning_rate mod_learning_rate module~mod_adam->module~mod_learning_rate module~mod_loss_mse mod_loss_mse module~mod_adam->module~mod_loss_mse module~mod_lossfunctions mod_lossfunctions module~mod_adam->module~mod_lossfunctions module~mod_precision mod_precision module~mod_adam->module~mod_precision module~mod_throw mod_throw module~mod_adam->module~mod_throw module~mod_abstractoptimizer->module~mod_io module~mod_abstractoptimizer->module~mod_precision module~mod_abstractoptimizer->module~mod_throw module~mod_global_warning mod_global_warning module~mod_abstractoptimizer->module~mod_global_warning module~mod_gradient_clipping mod_gradient_clipping module~mod_abstractoptimizer->module~mod_gradient_clipping module~mod_loss_abstract mod_loss_abstract module~mod_abstractoptimizer->module~mod_loss_abstract module~mod_mpi mod_mpi module~mod_abstractoptimizer->module~mod_mpi module~mod_nan_check mod_nan_check module~mod_abstractoptimizer->module~mod_nan_check module~mod_settings mod_settings module~mod_abstractoptimizer->module~mod_settings module~mod_decorations->module~mod_io module~mod_decorations->module~mod_precision module~mod_decorations->module~mod_throw module~mod_io->module~mod_precision iso_fortran_env iso_fortran_env module~mod_io->iso_fortran_env module~mod_optional_args mod_optional_args module~mod_io->module~mod_optional_args module~mod_learning_rate->module~mod_abstractoptimizer module~mod_learning_rate->module~mod_precision module~mod_loss_mse->module~mod_precision module~mod_loss_mse->module~mod_loss_abstract module~mod_lossfunctions->module~mod_io module~mod_lossfunctions->module~mod_loss_mse module~mod_lossfunctions->module~mod_throw module~mod_lossfunctions->module~mod_loss_abstract module~mod_loss_mae mod_loss_mae module~mod_lossfunctions->module~mod_loss_mae module~mod_loss_mse_torch mod_loss_mse_torch module~mod_lossfunctions->module~mod_loss_mse_torch module~mod_loss_rmse mod_loss_rmse module~mod_lossfunctions->module~mod_loss_rmse iso_c_binding iso_c_binding module~mod_precision->iso_c_binding module~mod_precision->iso_fortran_env module~mod_throw->module~mod_io module~mod_throw->module~mod_precision module~mod_throw->iso_c_binding module~mod_throw->iso_fortran_env module~mod_throw->module~mod_mpi module~mod_sort mod_sort module~mod_throw->module~mod_sort module~mod_global_warning->module~mod_io module~mod_global_warning->module~mod_throw module~mod_abstract_warning mod_abstract_warning module~mod_global_warning->module~mod_abstract_warning module~mod_gradient_clipping->module~mod_precision module~mod_gradient_clipping->module~mod_settings module~mod_loss_abstract->module~mod_precision module~mod_loss_abstract->module~mod_mpi module~mod_loss_mae->module~mod_precision module~mod_loss_mae->module~mod_loss_abstract module~mod_loss_mse_torch->module~mod_precision module~mod_loss_mse_torch->module~mod_loss_abstract module~mod_loss_rmse->module~mod_precision module~mod_loss_rmse->module~mod_loss_abstract module~mod_mpi->module~mod_precision mpi_f08 mpi_f08 module~mod_mpi->mpi_f08 module~mod_nan_check->module~mod_io module~mod_nan_check->module~mod_precision module~mod_nan_check->module~mod_throw ieee_arithmetic ieee_arithmetic module~mod_nan_check->ieee_arithmetic module~mod_settings->module~mod_io module~mod_settings->module~mod_precision module~mod_settings->module~mod_throw module~mod_constants mod_constants module~mod_settings->module~mod_constants module~mod_cpu_vendor mod_cpu_vendor module~mod_settings->module~mod_cpu_vendor module~mod_periodic_table mod_periodic_table module~mod_settings->module~mod_periodic_table module~mod_sort->module~mod_precision module~mod_constants->module~mod_precision module~mod_constants->iso_c_binding module~mod_cpu_vendor->iso_c_binding module~mod_periodic_table->module~mod_io module~mod_periodic_table->module~mod_precision module~mod_periodic_table->module~mod_throw module~mod_periodic_table->module~mod_constants module~mod_array_helpers mod_array_helpers module~mod_periodic_table->module~mod_array_helpers module~mod_array_helpers->module~mod_precision module~mod_array_helpers->module~mod_throw

Used By

module~~mod_adam~~UsedByGraph module~mod_adam mod_adam module~mod_optimizers mod_optimizers module~mod_optimizers->module~mod_adam module~mod_initialize mod_initialize module~mod_initialize->module~mod_optimizers module~mod_predict_mode mod_predict_mode module~mod_predict_mode->module~mod_initialize module~mod_write_outfiles mod_write_outfiles module~mod_predict_mode->module~mod_write_outfiles module~mod_runner_interface_bindings mod_runner_interface_bindings module~mod_runner_interface_bindings->module~mod_initialize module~mod_train_energy_multiple_structures mod_train_energy_multiple_structures module~mod_train_energy_multiple_structures->module~mod_initialize module~mod_train_forces_multiple_structures mod_train_forces_multiple_structures module~mod_train_forces_multiple_structures->module~mod_initialize module~mod_train_forces_multiple_structures->module~mod_train_energy_multiple_structures module~mod_train_loop_4g_charges mod_train_loop_4g_charges module~mod_train_loop_4g_charges->module~mod_initialize module~mod_cost_calculator mod_cost_calculator module~mod_train_loop_4g_charges->module~mod_cost_calculator module~mod_output_files_and_cost_4g_charges mod_output_files_and_cost_4g_charges module~mod_train_loop_4g_charges->module~mod_output_files_and_cost_4g_charges module~mod_train_loop_atomic_property mod_train_loop_atomic_property module~mod_train_loop_atomic_property->module~mod_initialize module~mod_train_loop_atomic_property->module~mod_cost_calculator module~mod_output_files_and_cost mod_output_files_and_cost module~mod_train_loop_atomic_property->module~mod_output_files_and_cost module~mod_train_loop_short_range mod_train_loop_short_range module~mod_train_loop_short_range->module~mod_initialize module~mod_train_loop_short_range->module~mod_train_energy_multiple_structures module~mod_train_loop_short_range->module~mod_train_forces_multiple_structures module~mod_train_loop_short_range->module~mod_cost_calculator module~mod_train_loop_short_range->module~mod_output_files_and_cost module~mod_output_files_and_cost_4g_short_range mod_output_files_and_cost_4g_short_range module~mod_train_loop_short_range->module~mod_output_files_and_cost_4g_short_range module~mod_train_mode mod_train_mode module~mod_train_mode->module~mod_initialize module~mod_train_mode->module~mod_train_loop_4g_charges module~mod_train_mode->module~mod_train_loop_atomic_property module~mod_train_mode->module~mod_train_loop_short_range module~mod_write_outfiles_training mod_write_outfiles_training module~mod_train_mode->module~mod_write_outfiles_training module~mod_write_outfiles->module~mod_initialize program~runner RuNNer program~runner->module~mod_initialize program~runner->module~mod_predict_mode program~runner->module~mod_train_mode module~mod_lammps_interface mod_lammps_interface module~mod_lammps_interface->module~mod_runner_interface_bindings module~mod_write_outfiles_training->module~mod_write_outfiles module~mod_cost_calculator->module~mod_write_outfiles_training module~mod_output_files_and_cost->module~mod_cost_calculator module~mod_output_files_and_cost_4g_charges->module~mod_cost_calculator module~mod_output_files_and_cost_4g_short_range->module~mod_cost_calculator

Derived Types Summary

Name Description
Adam Adam Optimizer.

Procedures Summary

Name Type Description
constructor_ptr_from_hyperparams_adam Function Initialize a new Adam optimizer object from a given set of (optional) hyperparameters.
constructor_ptr_from_file_adam Function Initialize a new Adam object from a string. See initialize_from_file for further details.
print_adam Subroutine Display information about the Adam object.
report_adam Subroutine Displays current status of the Adam object for verbosity >= 2.
set_num_parameters_adam Subroutine Prepare the optimizer for updating num_parameters free parameters. This allocated the first...
initialize_defaults_adam Subroutine Set the PyTorch default training parameters for Adam. See <a...
set_hyperparameter_adam Subroutine Update the value of the given hyperparameter.
validate_hyperparameters_adam Subroutine
update_adam Subroutine Update parameters according to their loss gradients.

Derived Types Details

type Adam

Adam Optimizer.

Components

  • name : character– Inherited from AbstractOptimizer
  • loss_function : class(AbstractLossFunction)– Inherited from AbstractOptimizer
  • num_parameters : integer– Inherited from AbstractOptimizer
  • loss_function_type : integer– Inherited from AbstractOptimizer
  • comm_idx : integer– Index of the committee member to which this optimizer is assigned to. – Inherited from AbstractOptimizer
  • opt_idx : integer– Index of the committee member to which this optimizer is assigned to. – Inherited from AbstractOptimizer
  • lupdate_hyperparameters : logical– Inherited from AbstractOptimizer
  • learning_rate : real(DP)– The initial step size for each gradient update. – Inherited from AbstractOptimizerLearningRate
  • beta1 : real(DP)– The exponential decay rate for the 1st and 2nd moment estimates.
  • beta2 : real(DP)– The exponential decay rate for the 1st and 2nd moment estimates.
  • m : real(DP) (dim: (:))– 1st and 2nd moment estimates.
  • v : real(DP) (dim: (:))– 1st and 2nd moment estimates.
  • epsilon : real(DP)– Small constant to ensure numerical stability in divisions.
  • iteration : integer– The amount of weight update iterations.

Type-Bound Procedures

Name Attributes Binding
weight_decay public ⇒ weight_decay
initialize_from_file public ⇒ initialize_from_file_abstract_optimizer
step public ⇒ step_abstractoptimizer_mpi
update_hyperparameters public ⇒ update_hyperparameters_abstractoptimizer
parameter_weight_decay public ⇒ parameter_weight_decay
update public ⇒ update_adam
print public ⇒ print_adam
report public ⇒ report_adam
set_num_parameters public ⇒ set_num_parameters_adam
validate public ⇒ validate_hyperparameters_adam
set_hyperparameter public ⇒ set_hyperparameter_adam
initialize_defaults public ⇒ initialize_defaults_adam

Inherits

type~~adam~~InheritsGraph type~adam Adam type~abstractoptimizerlearningrate AbstractOptimizerLearningRate type~adam->type~abstractoptimizerlearningrate type~abstractoptimizer AbstractOptimizer type~abstractoptimizerlearningrate->type~abstractoptimizer type~abstractlossfunction AbstractLossFunction type~abstractoptimizer->type~abstractlossfunction loss_function

Interface Details

interface Adam

Procedure Details

function constructor_ptr_from_hyperparams_adam

function constructor_ptr_from_hyperparams_adam( &
    learning_rate, &
    loss_function_type, &
    beta1, &
    beta2, &
    epsilon, &
    icomm &
)

Initialize a new Adam optimizer object from a given set of (optional) hyperparameters.

Arguments

  • learning_rate : real(DP) [in]– The step size for each gradient update.
  • loss_function_type : integer [in]
  • beta1 : real(DP) [in]
  • beta2 : real(DP) [in]
  • epsilon : real(DP) [in]
  • icomm : integer [in]– Index of the committee member to which this optimizer is assigned to.

Called By

proc~~constructor_ptr_from_hyperparams_adam~~CalledByGraph proc~constructor_ptr_from_hyperparams_adam mod_adam::constructor_ptr_from_hyperparams_adam interface~adam mod_adam::Adam interface~adam->proc~constructor_ptr_from_hyperparams_adam

function constructor_ptr_from_file_adam

function constructor_ptr_from_file_adam( &
    line, &
    icomm &
)

Initialize a new Adam object from a string. See initialize_from_file for further details.

Arguments

  • line : character [in]
  • icomm : integer [in]– Committee index to which this `Adam` object is assigned to.

Called By

proc~~constructor_ptr_from_file_adam~~CalledByGraph proc~constructor_ptr_from_file_adam mod_adam::constructor_ptr_from_file_adam interface~adam mod_adam::Adam interface~adam->proc~constructor_ptr_from_file_adam

subroutine print_adam

subroutine print_adam( &
    self &
)

Display information about the Adam object.

Arguments

  • self : class(Adam) [in]

subroutine report_adam

subroutine report_adam( &
    self, &
    epoch &
)

Displays current status of the Adam object for verbosity >= 2.

Arguments

  • self : class(Adam) [in]
  • epoch : integer [in]

subroutine set_num_parameters_adam

subroutine set_num_parameters_adam( &
    self, &
    num_parameters &
)

Prepare the optimizer for updating num_parameters free parameters. This allocated the first and second momentum vectors.

Arguments

  • self : class(Adam) [inout]
  • num_parameters : integer [in]

subroutine initialize_defaults_adam

subroutine initialize_defaults_adam( &
    self &
)

Set the PyTorch default training parameters for Adam. See here.

Arguments

  • self : class(Adam) [inout]

Called By

proc~~initialize_defaults_adam~~CalledByGraph proc~initialize_defaults_adam mod_adam::Adam%initialize_defaults_adam proc~constructor_ptr_from_hyperparams_adam mod_adam::constructor_ptr_from_hyperparams_adam proc~constructor_ptr_from_hyperparams_adam->proc~initialize_defaults_adam interface~adam mod_adam::Adam interface~adam->proc~constructor_ptr_from_hyperparams_adam

subroutine set_hyperparameter_adam

subroutine set_hyperparameter_adam( &
    self, &
    name, &
    value_str &
)

Update the value of the given hyperparameter.

Arguments

  • self : class(Adam) [inout]
  • name : character [in]
  • value_str : character [in]

subroutine validate_hyperparameters_adam

subroutine validate_hyperparameters_adam( &
    self &
)

Arguments

  • self : class(Adam) [in]

Called By

proc~~validate_hyperparameters_adam~~CalledByGraph proc~validate_hyperparameters_adam mod_adam::Adam%validate_hyperparameters_adam proc~constructor_ptr_from_hyperparams_adam mod_adam::constructor_ptr_from_hyperparams_adam proc~constructor_ptr_from_hyperparams_adam->proc~validate_hyperparameters_adam interface~adam mod_adam::Adam interface~adam->proc~constructor_ptr_from_hyperparams_adam

subroutine update_adam

subroutine update_adam( &
    self, &
    parameters, &
    gradients, &
    losses, &
    num_streams &
)

Update parameters according to their loss gradients.

Arguments

  • self : class(Adam) [inout]
  • parameters : real(DP) (dim: (self%num_parameters)) [inout]– A list of all free parameters in the `Model`.
  • gradients : real(DP) (dim: (self%num_parameters,num_streams)) [inout]– Gradients of the `Model` output wrt its parameters.
  • losses : real(DP) (dim: (num_streams)) [in]– Loss of the `Model` output (prediction - target).
  • num_streams : integer [in]– The number of streams participating in the update.