Skip to content

mod_stochastic_gradient_descent

Lines: 114 | Visibility: private

Definition of the stochastic gradient descent optimization algorithm.

In SGD, the size of one parameter update along the gradient is varied according to the size of the previous update step. author: Alexander Knoll, Moritz R. Schaefer

Dependencies

Uses

module~~mod_stochastic_gradient_descent~~UsesGraph module~mod_stochastic_gradient_descent mod_stochastic_gradient_descent module~mod_abstractoptimizer mod_abstractoptimizer module~mod_stochastic_gradient_descent->module~mod_abstractoptimizer module~mod_decorations mod_decorations module~mod_stochastic_gradient_descent->module~mod_decorations module~mod_io mod_io module~mod_stochastic_gradient_descent->module~mod_io module~mod_learning_rate mod_learning_rate module~mod_stochastic_gradient_descent->module~mod_learning_rate module~mod_loss_mse mod_loss_mse module~mod_stochastic_gradient_descent->module~mod_loss_mse module~mod_lossfunctions mod_lossfunctions module~mod_stochastic_gradient_descent->module~mod_lossfunctions module~mod_precision mod_precision module~mod_stochastic_gradient_descent->module~mod_precision module~mod_throw mod_throw module~mod_stochastic_gradient_descent->module~mod_throw module~mod_abstractoptimizer->module~mod_io module~mod_abstractoptimizer->module~mod_precision module~mod_abstractoptimizer->module~mod_throw module~mod_global_warning mod_global_warning module~mod_abstractoptimizer->module~mod_global_warning module~mod_gradient_clipping mod_gradient_clipping module~mod_abstractoptimizer->module~mod_gradient_clipping module~mod_loss_abstract mod_loss_abstract module~mod_abstractoptimizer->module~mod_loss_abstract module~mod_mpi mod_mpi module~mod_abstractoptimizer->module~mod_mpi module~mod_nan_check mod_nan_check module~mod_abstractoptimizer->module~mod_nan_check module~mod_settings mod_settings module~mod_abstractoptimizer->module~mod_settings module~mod_decorations->module~mod_io module~mod_decorations->module~mod_precision module~mod_decorations->module~mod_throw module~mod_io->module~mod_precision iso_fortran_env iso_fortran_env module~mod_io->iso_fortran_env module~mod_optional_args mod_optional_args module~mod_io->module~mod_optional_args module~mod_learning_rate->module~mod_abstractoptimizer module~mod_learning_rate->module~mod_precision module~mod_loss_mse->module~mod_precision module~mod_loss_mse->module~mod_loss_abstract module~mod_lossfunctions->module~mod_io module~mod_lossfunctions->module~mod_loss_mse module~mod_lossfunctions->module~mod_throw module~mod_lossfunctions->module~mod_loss_abstract module~mod_loss_mae mod_loss_mae module~mod_lossfunctions->module~mod_loss_mae module~mod_loss_mse_torch mod_loss_mse_torch module~mod_lossfunctions->module~mod_loss_mse_torch module~mod_loss_rmse mod_loss_rmse module~mod_lossfunctions->module~mod_loss_rmse iso_c_binding iso_c_binding module~mod_precision->iso_c_binding module~mod_precision->iso_fortran_env module~mod_throw->module~mod_io module~mod_throw->module~mod_precision module~mod_throw->iso_c_binding module~mod_throw->iso_fortran_env module~mod_throw->module~mod_mpi module~mod_sort mod_sort module~mod_throw->module~mod_sort module~mod_global_warning->module~mod_io module~mod_global_warning->module~mod_throw module~mod_abstract_warning mod_abstract_warning module~mod_global_warning->module~mod_abstract_warning module~mod_gradient_clipping->module~mod_precision module~mod_gradient_clipping->module~mod_settings module~mod_loss_abstract->module~mod_precision module~mod_loss_abstract->module~mod_mpi module~mod_loss_mae->module~mod_precision module~mod_loss_mae->module~mod_loss_abstract module~mod_loss_mse_torch->module~mod_precision module~mod_loss_mse_torch->module~mod_loss_abstract module~mod_loss_rmse->module~mod_precision module~mod_loss_rmse->module~mod_loss_abstract module~mod_mpi->module~mod_precision mpi_f08 mpi_f08 module~mod_mpi->mpi_f08 module~mod_nan_check->module~mod_io module~mod_nan_check->module~mod_precision module~mod_nan_check->module~mod_throw ieee_arithmetic ieee_arithmetic module~mod_nan_check->ieee_arithmetic module~mod_settings->module~mod_io module~mod_settings->module~mod_precision module~mod_settings->module~mod_throw module~mod_constants mod_constants module~mod_settings->module~mod_constants module~mod_cpu_vendor mod_cpu_vendor module~mod_settings->module~mod_cpu_vendor module~mod_periodic_table mod_periodic_table module~mod_settings->module~mod_periodic_table module~mod_sort->module~mod_precision module~mod_constants->module~mod_precision module~mod_constants->iso_c_binding module~mod_cpu_vendor->iso_c_binding module~mod_periodic_table->module~mod_io module~mod_periodic_table->module~mod_precision module~mod_periodic_table->module~mod_throw module~mod_periodic_table->module~mod_constants module~mod_array_helpers mod_array_helpers module~mod_periodic_table->module~mod_array_helpers module~mod_array_helpers->module~mod_precision module~mod_array_helpers->module~mod_throw

Used By

module~~mod_stochastic_gradient_descent~~UsedByGraph module~mod_stochastic_gradient_descent mod_stochastic_gradient_descent module~mod_optimizers mod_optimizers module~mod_optimizers->module~mod_stochastic_gradient_descent module~mod_initialize mod_initialize module~mod_initialize->module~mod_optimizers module~mod_predict_mode mod_predict_mode module~mod_predict_mode->module~mod_initialize module~mod_write_outfiles mod_write_outfiles module~mod_predict_mode->module~mod_write_outfiles module~mod_runner_interface_bindings mod_runner_interface_bindings module~mod_runner_interface_bindings->module~mod_initialize module~mod_train_energy_multiple_structures mod_train_energy_multiple_structures module~mod_train_energy_multiple_structures->module~mod_initialize module~mod_train_forces_multiple_structures mod_train_forces_multiple_structures module~mod_train_forces_multiple_structures->module~mod_initialize module~mod_train_forces_multiple_structures->module~mod_train_energy_multiple_structures module~mod_train_loop_4g_charges mod_train_loop_4g_charges module~mod_train_loop_4g_charges->module~mod_initialize module~mod_cost_calculator mod_cost_calculator module~mod_train_loop_4g_charges->module~mod_cost_calculator module~mod_output_files_and_cost_4g_charges mod_output_files_and_cost_4g_charges module~mod_train_loop_4g_charges->module~mod_output_files_and_cost_4g_charges module~mod_train_loop_atomic_property mod_train_loop_atomic_property module~mod_train_loop_atomic_property->module~mod_initialize module~mod_train_loop_atomic_property->module~mod_cost_calculator module~mod_output_files_and_cost mod_output_files_and_cost module~mod_train_loop_atomic_property->module~mod_output_files_and_cost module~mod_train_loop_short_range mod_train_loop_short_range module~mod_train_loop_short_range->module~mod_initialize module~mod_train_loop_short_range->module~mod_train_energy_multiple_structures module~mod_train_loop_short_range->module~mod_train_forces_multiple_structures module~mod_train_loop_short_range->module~mod_cost_calculator module~mod_train_loop_short_range->module~mod_output_files_and_cost module~mod_output_files_and_cost_4g_short_range mod_output_files_and_cost_4g_short_range module~mod_train_loop_short_range->module~mod_output_files_and_cost_4g_short_range module~mod_train_mode mod_train_mode module~mod_train_mode->module~mod_initialize module~mod_train_mode->module~mod_train_loop_4g_charges module~mod_train_mode->module~mod_train_loop_atomic_property module~mod_train_mode->module~mod_train_loop_short_range module~mod_write_outfiles_training mod_write_outfiles_training module~mod_train_mode->module~mod_write_outfiles_training module~mod_write_outfiles->module~mod_initialize program~runner RuNNer program~runner->module~mod_initialize program~runner->module~mod_predict_mode program~runner->module~mod_train_mode module~mod_lammps_interface mod_lammps_interface module~mod_lammps_interface->module~mod_runner_interface_bindings module~mod_write_outfiles_training->module~mod_write_outfiles module~mod_cost_calculator->module~mod_write_outfiles_training module~mod_output_files_and_cost->module~mod_cost_calculator module~mod_output_files_and_cost_4g_charges->module~mod_cost_calculator module~mod_output_files_and_cost_4g_short_range->module~mod_cost_calculator

Derived Types Summary

Name Description
StochasticGradientDescent Stochastic Gradient Descent (SGD) Optimizer. The initial step size for each gradient update.

Procedures Summary

Name Type Description
constructor_ptr_from_hyperparams_sgd Function Initialize a new StochasticGradientDescent optimizer object based on given hyperparameters.
constructor_ptr_from_file_sgd Function
set_num_parameters_sgd Subroutine
print_sgd Subroutine Display information about the StochasticGradientDescent object.
report_sgd Subroutine Displays current status of the StochasticGradientDescent object for verbosity >= 2.
initialize_defaults_sgd Subroutine
validate_hyperparameters_sgd Subroutine
set_hyperparameter_sgd Subroutine Update the value of the given hyperparameter.
update_sgd Subroutine Update parameters according to their loss gradients.

Derived Types Details

type StochasticGradientDescent

Stochastic Gradient Descent (SGD) Optimizer.

The initial step size for each gradient update.

Components

  • name : character– Inherited from AbstractOptimizer
  • loss_function : class(AbstractLossFunction)– Inherited from AbstractOptimizer
  • num_parameters : integer– Inherited from AbstractOptimizer
  • loss_function_type : integer– Inherited from AbstractOptimizer
  • comm_idx : integer– Index of the committee member to which this optimizer is assigned to. – Inherited from AbstractOptimizer
  • opt_idx : integer– Index of the committee member to which this optimizer is assigned to. – Inherited from AbstractOptimizer
  • lupdate_hyperparameters : logical– Inherited from AbstractOptimizer
  • learning_rate : real(DP)– The initial step size for each gradient update. – Inherited from AbstractOptimizerLearningRate
  • momentum : real(DP)– The acceleration of gradient descent in the relevant directions. Dampens oscillations.
  • velocity : real(DP) (dim: (:))– Storage array for the stepsize of the previous parameter update.

Type-Bound Procedures

Name Attributes Binding
weight_decay public ⇒ weight_decay
initialize_from_file public ⇒ initialize_from_file_abstract_optimizer
step public ⇒ step_abstractoptimizer_mpi
update_hyperparameters public ⇒ update_hyperparameters_abstractoptimizer
parameter_weight_decay public ⇒ parameter_weight_decay
update public ⇒ update_sgd
print public ⇒ print_sgd
report public ⇒ report_sgd
set_num_parameters public ⇒ set_num_parameters_sgd
initialize_defaults public ⇒ initialize_defaults_sgd
set_hyperparameter public ⇒ set_hyperparameter_sgd
validate public ⇒ validate_hyperparameters_sgd

Inherits

type~~stochasticgradientdescent~~InheritsGraph type~stochasticgradientdescent StochasticGradientDescent type~abstractoptimizerlearningrate AbstractOptimizerLearningRate type~stochasticgradientdescent->type~abstractoptimizerlearningrate type~abstractoptimizer AbstractOptimizer type~abstractoptimizerlearningrate->type~abstractoptimizer type~abstractlossfunction AbstractLossFunction type~abstractoptimizer->type~abstractlossfunction loss_function

Interface Details

interface StochasticGradientDescent

Procedure Details

function constructor_ptr_from_hyperparams_sgd

function constructor_ptr_from_hyperparams_sgd( &
    learning_rate, &
    loss_function_type, &
    momentum, &
    icomm &
)

Initialize a new StochasticGradientDescent optimizer object based on given hyperparameters.

Arguments

  • learning_rate : real(DP) [in]– The step size for each gradient update.
  • loss_function_type : integer [in]
  • momentum : real(DP) [in]
  • icomm : integer [in]– Index of the committee member to which this optimizer is assigned to.

Called By

proc~~constructor_ptr_from_hyperparams_sgd~~CalledByGraph proc~constructor_ptr_from_hyperparams_sgd mod_stochastic_gradient_descent::constructor_ptr_from_hyperparams_sgd interface~stochasticgradientdescent mod_stochastic_gradient_descent::StochasticGradientDescent interface~stochasticgradientdescent->proc~constructor_ptr_from_hyperparams_sgd

function constructor_ptr_from_file_sgd

function constructor_ptr_from_file_sgd( &
    line, &
    icomm &
)

Arguments

  • line : character [in]
  • icomm : integer [in]– Committee index to which this `StochasticGradientDescent` object is assigned to.

Called By

proc~~constructor_ptr_from_file_sgd~~CalledByGraph proc~constructor_ptr_from_file_sgd mod_stochastic_gradient_descent::constructor_ptr_from_file_sgd interface~stochasticgradientdescent mod_stochastic_gradient_descent::StochasticGradientDescent interface~stochasticgradientdescent->proc~constructor_ptr_from_file_sgd

subroutine set_num_parameters_sgd

subroutine set_num_parameters_sgd( &
    self, &
    num_parameters &
)

Arguments

subroutine print_sgd

subroutine print_sgd( &
    self &
)

Display information about the StochasticGradientDescent object.

Arguments

subroutine report_sgd

subroutine report_sgd( &
    self, &
    epoch &
)

Displays current status of the StochasticGradientDescent object for verbosity >= 2.

Arguments

subroutine initialize_defaults_sgd

subroutine initialize_defaults_sgd( &
    self &
)

Arguments

Called By

proc~~initialize_defaults_sgd~~CalledByGraph proc~initialize_defaults_sgd mod_stochastic_gradient_descent::StochasticGradientDescent%initialize_defaults_sgd proc~constructor_ptr_from_hyperparams_sgd mod_stochastic_gradient_descent::constructor_ptr_from_hyperparams_sgd proc~constructor_ptr_from_hyperparams_sgd->proc~initialize_defaults_sgd interface~stochasticgradientdescent mod_stochastic_gradient_descent::StochasticGradientDescent interface~stochasticgradientdescent->proc~constructor_ptr_from_hyperparams_sgd

subroutine validate_hyperparameters_sgd

subroutine validate_hyperparameters_sgd( &
    self &
)

Arguments

Called By

proc~~validate_hyperparameters_sgd~~CalledByGraph proc~validate_hyperparameters_sgd mod_stochastic_gradient_descent::StochasticGradientDescent%validate_hyperparameters_sgd proc~constructor_ptr_from_hyperparams_sgd mod_stochastic_gradient_descent::constructor_ptr_from_hyperparams_sgd proc~constructor_ptr_from_hyperparams_sgd->proc~validate_hyperparameters_sgd interface~stochasticgradientdescent mod_stochastic_gradient_descent::StochasticGradientDescent interface~stochasticgradientdescent->proc~constructor_ptr_from_hyperparams_sgd

subroutine set_hyperparameter_sgd

subroutine set_hyperparameter_sgd( &
    self, &
    name, &
    value_str &
)

Update the value of the given hyperparameter.

Arguments

subroutine update_sgd

subroutine update_sgd( &
    self, &
    parameters, &
    gradients, &
    losses, &
    num_streams &
)

Update parameters according to their loss gradients.

Arguments

  • self : class(StochasticGradientDescent) [inout]
  • parameters : real(DP) (dim: (self%num_parameters)) [inout]– A list of all free parameters in the `Model`.
  • gradients : real(DP) (dim: (self%num_parameters,num_streams)) [inout]– Gradients of the `Model` output wrt its parameters.
  • losses : real(DP) (dim: (num_streams)) [in]– Loss of the `Model` output (prediction - target).
  • num_streams : integer [in]– The number of streams participating in the update.