Skip to content

mod_dataloader_file

Lines: 353 | Visibility: private

Dataloader for loading multiple structures from a file. author: Alexander Knoll

Dependencies

Uses

module~~mod_dataloader_file~~UsesGraph module~mod_dataloader_file mod_dataloader_file module~mod_abstract_structure_data_file mod_abstract_structure_data_file module~mod_dataloader_file->module~mod_abstract_structure_data_file module~mod_cutoff_function mod_cutoff_function module~mod_dataloader_file->module~mod_cutoff_function module~mod_databatcher mod_databatcher module~mod_dataloader_file->module~mod_databatcher module~mod_feature_calculator mod_feature_calculator module~mod_dataloader_file->module~mod_feature_calculator module~mod_feature_map_neighborlist mod_feature_map_neighborlist module~mod_dataloader_file->module~mod_feature_map_neighborlist module~mod_io mod_io module~mod_dataloader_file->module~mod_io module~mod_memory_estimation mod_memory_estimation module~mod_dataloader_file->module~mod_memory_estimation module~mod_mpi mod_mpi module~mod_dataloader_file->module~mod_mpi module~mod_optional_args mod_optional_args module~mod_dataloader_file->module~mod_optional_args module~mod_periodic_table mod_periodic_table module~mod_dataloader_file->module~mod_periodic_table module~mod_precision mod_precision module~mod_dataloader_file->module~mod_precision module~mod_split_and_assign mod_split_and_assign module~mod_dataloader_file->module~mod_split_and_assign module~mod_structures mod_structures module~mod_dataloader_file->module~mod_structures module~mod_throw mod_throw module~mod_dataloader_file->module~mod_throw module~mod_abstract_structure_data_file->module~mod_mpi module~mod_abstract_structure_data_file->module~mod_periodic_table module~mod_abstract_structure_data_file->module~mod_structures module~mod_dataset_storage mod_dataset_storage module~mod_abstract_structure_data_file->module~mod_dataset_storage module~mod_genericfile mod_genericfile module~mod_abstract_structure_data_file->module~mod_genericfile module~mod_cutoff_function->module~mod_io module~mod_cutoff_function->module~mod_precision module~mod_cutoff_function->module~mod_throw module~mod_abstract_cutoff_function mod_abstract_cutoff_function module~mod_cutoff_function->module~mod_abstract_cutoff_function module~mod_cutoff_function_cosine mod_cutoff_function_cosine module~mod_cutoff_function->module~mod_cutoff_function_cosine module~mod_cutoff_function_hard mod_cutoff_function_hard module~mod_cutoff_function->module~mod_cutoff_function_hard module~mod_cutoff_function_hypertangent mod_cutoff_function_hypertangent module~mod_cutoff_function->module~mod_cutoff_function_hypertangent module~mod_cutoff_function_hypertangent_approx mod_cutoff_function_hypertangent_approx module~mod_cutoff_function->module~mod_cutoff_function_hypertangent_approx module~mod_cutoff_function_null mod_cutoff_function_null module~mod_cutoff_function->module~mod_cutoff_function_null module~mod_cutoff_function_polynomial mod_cutoff_function_polynomial module~mod_cutoff_function->module~mod_cutoff_function_polynomial module~mod_cutoff_function_vdw mod_cutoff_function_vdw module~mod_cutoff_function->module~mod_cutoff_function_vdw module~mod_databatcher->module~mod_mpi module~mod_databatcher->module~mod_precision module~mod_databatcher->module~mod_throw module~mod_array_helpers mod_array_helpers module~mod_databatcher->module~mod_array_helpers module~mod_random mod_random module~mod_databatcher->module~mod_random module~mod_feature_calculator->module~mod_cutoff_function module~mod_feature_calculator->module~mod_feature_map_neighborlist module~mod_feature_calculator->module~mod_io module~mod_feature_calculator->module~mod_mpi module~mod_feature_calculator->module~mod_optional_args module~mod_feature_calculator->module~mod_periodic_table module~mod_feature_calculator->module~mod_precision module~mod_feature_calculator->module~mod_structures module~mod_feature_calculator->module~mod_throw module~mod_angular_symmetry_function_3 mod_angular_symmetry_function_3 module~mod_feature_calculator->module~mod_angular_symmetry_function_3 module~mod_angular_symmetry_function_8 mod_angular_symmetry_function_8 module~mod_feature_calculator->module~mod_angular_symmetry_function_8 module~mod_angular_symmetry_function_9 mod_angular_symmetry_function_9 module~mod_feature_calculator->module~mod_angular_symmetry_function_9 module~mod_feature_calculator->module~mod_array_helpers module~mod_atomic_descriptor mod_atomic_descriptor module~mod_feature_calculator->module~mod_atomic_descriptor module~mod_data_container mod_data_container module~mod_feature_calculator->module~mod_data_container module~mod_element mod_element module~mod_feature_calculator->module~mod_element module~mod_extrapolation_warning mod_extrapolation_warning module~mod_feature_calculator->module~mod_extrapolation_warning module~mod_feature_map mod_feature_map module~mod_feature_calculator->module~mod_feature_map module~mod_overlap_matrix mod_overlap_matrix module~mod_feature_calculator->module~mod_overlap_matrix module~mod_path mod_path module~mod_feature_calculator->module~mod_path module~mod_radial_symmetry_function_1 mod_radial_symmetry_function_1 module~mod_feature_calculator->module~mod_radial_symmetry_function_1 module~mod_radial_symmetry_function_2 mod_radial_symmetry_function_2 module~mod_feature_calculator->module~mod_radial_symmetry_function_2 module~mod_set_integer mod_set_integer module~mod_feature_calculator->module~mod_set_integer module~mod_settings mod_settings module~mod_feature_calculator->module~mod_settings module~mod_timer mod_timer module~mod_feature_calculator->module~mod_timer module~mod_feature_map_neighborlist->module~mod_io module~mod_feature_map_neighborlist->module~mod_mpi module~mod_feature_map_neighborlist->module~mod_precision module~mod_feature_map_neighborlist->module~mod_structures module~mod_feature_map_neighborlist->module~mod_throw module~mod_feature_map_neighborlist->module~mod_abstract_cutoff_function module~mod_constants mod_constants module~mod_feature_map_neighborlist->module~mod_constants module~mod_neighborlist mod_neighborlist module~mod_feature_map_neighborlist->module~mod_neighborlist module~mod_feature_map_neighborlist->module~mod_timer module~mod_io->module~mod_optional_args module~mod_io->module~mod_precision iso_fortran_env iso_fortran_env module~mod_io->iso_fortran_env module~mod_memory_estimation->module~mod_feature_calculator module~mod_memory_estimation->module~mod_feature_map_neighborlist module~mod_memory_estimation->module~mod_io module~mod_memory_estimation->module~mod_mpi module~mod_memory_estimation->module~mod_periodic_table module~mod_memory_estimation->module~mod_precision module~mod_memory_estimation->module~mod_structures module~mod_memory_estimation->module~mod_throw module~mod_memory_estimation->module~mod_settings module~mod_table mod_table module~mod_memory_estimation->module~mod_table module~mod_mpi->module~mod_precision mpi_f08 mpi_f08 module~mod_mpi->mpi_f08 module~mod_periodic_table->module~mod_io module~mod_periodic_table->module~mod_precision module~mod_periodic_table->module~mod_throw module~mod_periodic_table->module~mod_array_helpers module~mod_periodic_table->module~mod_constants iso_c_binding iso_c_binding module~mod_precision->iso_c_binding module~mod_precision->iso_fortran_env module~mod_split_and_assign->module~mod_mpi module~mod_split_and_assign->module~mod_precision module~mod_split_and_assign->module~mod_throw module~mod_structures->module~mod_mpi module~mod_structures->module~mod_periodic_table module~mod_structures->module~mod_precision module~mod_structures->module~mod_throw module~mod_structures->module~mod_array_helpers module~mod_structures->module~mod_constants module~mod_linalg mod_linalg module~mod_structures->module~mod_linalg module~mod_structures_helper mod_structures_helper module~mod_structures->module~mod_structures_helper module~mod_throw->module~mod_io module~mod_throw->module~mod_mpi module~mod_throw->module~mod_precision module~mod_throw->iso_c_binding module~mod_throw->iso_fortran_env module~mod_sort mod_sort module~mod_throw->module~mod_sort module~mod_abstract_cutoff_function->module~mod_precision module~mod_angular_symmetry_function_3->module~mod_cutoff_function module~mod_angular_symmetry_function_3->module~mod_feature_map_neighborlist module~mod_angular_symmetry_function_3->module~mod_periodic_table module~mod_angular_symmetry_function_3->module~mod_precision module~mod_angular_symmetry_function_3->module~mod_structures module~mod_angular_symmetry_function_3->module~mod_throw module~mod_angular_symmetry_function_3->module~mod_constants module~mod_angular_symmetry_function_3->module~mod_feature_map module~mod_abstract_angular_symmetry_function mod_abstract_angular_symmetry_function module~mod_angular_symmetry_function_3->module~mod_abstract_angular_symmetry_function module~mod_abstract_symmetry_function mod_abstract_symmetry_function module~mod_angular_symmetry_function_3->module~mod_abstract_symmetry_function module~mod_fastmath mod_fastmath module~mod_angular_symmetry_function_3->module~mod_fastmath module~mod_hash mod_hash module~mod_angular_symmetry_function_3->module~mod_hash module~mod_angular_symmetry_function_8->module~mod_cutoff_function module~mod_angular_symmetry_function_8->module~mod_feature_map_neighborlist module~mod_angular_symmetry_function_8->module~mod_periodic_table module~mod_angular_symmetry_function_8->module~mod_precision module~mod_angular_symmetry_function_8->module~mod_structures module~mod_angular_symmetry_function_8->module~mod_throw module~mod_angular_symmetry_function_8->module~mod_constants module~mod_angular_symmetry_function_8->module~mod_feature_map module~mod_angular_symmetry_function_8->module~mod_abstract_angular_symmetry_function module~mod_angular_symmetry_function_8->module~mod_abstract_symmetry_function module~mod_angular_symmetry_function_8->module~mod_hash module~mod_angular_symmetry_function_9->module~mod_cutoff_function module~mod_angular_symmetry_function_9->module~mod_feature_map_neighborlist module~mod_angular_symmetry_function_9->module~mod_periodic_table module~mod_angular_symmetry_function_9->module~mod_precision module~mod_angular_symmetry_function_9->module~mod_structures module~mod_angular_symmetry_function_9->module~mod_throw module~mod_angular_symmetry_function_9->module~mod_constants module~mod_angular_symmetry_function_9->module~mod_feature_map module~mod_angular_symmetry_function_9->module~mod_abstract_angular_symmetry_function module~mod_angular_symmetry_function_9->module~mod_abstract_symmetry_function module~mod_angular_symmetry_function_9->module~mod_fastmath module~mod_angular_symmetry_function_9->module~mod_hash module~mod_array_helpers->module~mod_precision module~mod_array_helpers->module~mod_throw module~mod_atomic_descriptor->module~mod_precision module~mod_atomic_descriptor->module~mod_structures module~mod_atomic_descriptor->module~mod_element module~mod_constants->module~mod_precision module~mod_constants->iso_c_binding module~mod_cutoff_function_cosine->module~mod_precision module~mod_cutoff_function_cosine->module~mod_abstract_cutoff_function module~mod_cutoff_function_cosine->module~mod_constants module~mod_cutoff_function_hard->module~mod_precision module~mod_cutoff_function_hard->module~mod_abstract_cutoff_function module~mod_cutoff_function_hypertangent->module~mod_precision module~mod_cutoff_function_hypertangent->module~mod_abstract_cutoff_function module~mod_cutoff_function_hypertangent_approx->module~mod_precision module~mod_cutoff_function_hypertangent_approx->module~mod_abstract_cutoff_function module~mod_cutoff_function_null->module~mod_precision module~mod_cutoff_function_null->module~mod_throw module~mod_cutoff_function_null->module~mod_abstract_cutoff_function module~mod_cutoff_function_polynomial->module~mod_precision module~mod_cutoff_function_polynomial->module~mod_abstract_cutoff_function module~mod_cutoff_function_polynomial->module~mod_fastmath module~mod_cutoff_function_vdw->module~mod_precision module~mod_cutoff_function_vdw->module~mod_abstract_cutoff_function module~mod_data_container->module~mod_io module~mod_data_container->module~mod_precision module~mod_data_container->module~mod_throw module~mod_dataset_storage->module~mod_mpi module~mod_dataset_storage->module~mod_sort mod_property_list mod_property_list module~mod_dataset_storage->mod_property_list module~mod_storage_container mod_storage_container module~mod_dataset_storage->module~mod_storage_container module~mod_element->module~mod_cutoff_function module~mod_element->module~mod_io module~mod_element->module~mod_mpi module~mod_element->module~mod_optional_args module~mod_element->module~mod_periodic_table module~mod_element->module~mod_precision module~mod_element->module~mod_throw module~mod_element->module~mod_angular_symmetry_function_3 module~mod_element->module~mod_angular_symmetry_function_9 module~mod_element->module~mod_constants module~mod_element->module~mod_feature_map module~mod_element->module~mod_radial_symmetry_function_2 module~mod_element->module~mod_set_integer module~mod_arrays_init mod_arrays_init module~mod_element->module~mod_arrays_init module~mod_decorations mod_decorations module~mod_element->module~mod_decorations module~mod_dubious_feature_map_handler mod_dubious_feature_map_handler module~mod_element->module~mod_dubious_feature_map_handler module~mod_element->module~mod_hash module~mod_set_character mod_set_character module~mod_element->module~mod_set_character module~mod_extrapolation_warning->module~mod_periodic_table module~mod_extrapolation_warning->module~mod_precision module~mod_extrapolation_warning->module~mod_throw module~mod_extrapolation_warning->iso_c_binding module~mod_abstract_warning mod_abstract_warning module~mod_extrapolation_warning->module~mod_abstract_warning module~mod_feature_map->module~mod_cutoff_function module~mod_feature_map->module~mod_feature_map_neighborlist module~mod_feature_map->module~mod_io module~mod_feature_map->module~mod_precision module~mod_feature_map->module~mod_structures module~mod_feature_map->module~mod_throw module~mod_feature_map->module~mod_hash module~mod_sanitize mod_sanitize module~mod_feature_map->module~mod_sanitize module~mod_genericfile->module~mod_io module~mod_genericfile->module~mod_mpi module~mod_genericfile->module~mod_throw module~mod_linalg->module~mod_precision module~mod_linalg->module~mod_throw module~mod_linalg->module~mod_settings module~cublas_wrapper cublas_wrapper module~mod_linalg->module~cublas_wrapper module~mod_neighborlist->module~mod_io module~mod_neighborlist->module~mod_mpi module~mod_neighborlist->module~mod_precision module~mod_neighborlist->module~mod_structures module~mod_neighborlist->module~mod_throw module~mod_neighborlist->module~mod_settings module~mod_dynamic_double_array mod_dynamic_double_array module~mod_neighborlist->module~mod_dynamic_double_array module~mod_dynamic_int_array mod_dynamic_int_array module~mod_neighborlist->module~mod_dynamic_int_array module~mod_neighbor_grid mod_neighbor_grid module~mod_neighborlist->module~mod_neighbor_grid module~mod_neighborlist_helper mod_neighborlist_helper module~mod_neighborlist->module~mod_neighborlist_helper module~mod_overlap_matrix->module~mod_cutoff_function module~mod_overlap_matrix->module~mod_feature_map_neighborlist module~mod_overlap_matrix->module~mod_precision module~mod_overlap_matrix->module~mod_structures module~mod_overlap_matrix->module~mod_feature_map module~mod_path->module~mod_io module~mod_radial_symmetry_function_1->module~mod_cutoff_function module~mod_radial_symmetry_function_1->module~mod_feature_map_neighborlist module~mod_radial_symmetry_function_1->module~mod_io module~mod_radial_symmetry_function_1->module~mod_periodic_table module~mod_radial_symmetry_function_1->module~mod_precision module~mod_radial_symmetry_function_1->module~mod_structures module~mod_radial_symmetry_function_1->module~mod_throw module~mod_radial_symmetry_function_1->module~mod_feature_map module~mod_abstract_radial_symmetry_function mod_abstract_radial_symmetry_function module~mod_radial_symmetry_function_1->module~mod_abstract_radial_symmetry_function module~mod_radial_symmetry_function_1->module~mod_hash module~mod_radial_symmetry_function_2->module~mod_cutoff_function module~mod_radial_symmetry_function_2->module~mod_feature_map_neighborlist module~mod_radial_symmetry_function_2->module~mod_periodic_table module~mod_radial_symmetry_function_2->module~mod_precision module~mod_radial_symmetry_function_2->module~mod_structures module~mod_radial_symmetry_function_2->module~mod_throw module~mod_radial_symmetry_function_2->module~mod_feature_map module~mod_radial_symmetry_function_2->module~mod_abstract_radial_symmetry_function module~mod_radial_symmetry_function_2->module~mod_abstract_symmetry_function module~mod_radial_symmetry_function_2->module~mod_hash module~mod_random->module~mod_precision module~mod_random->module~mod_constants module~mod_set_integer->module~mod_throw module~mod_settings->module~mod_io module~mod_settings->module~mod_periodic_table module~mod_settings->module~mod_precision module~mod_settings->module~mod_throw module~mod_settings->module~mod_constants module~mod_cpu_vendor mod_cpu_vendor module~mod_settings->module~mod_cpu_vendor module~mod_sort->module~mod_precision module~mod_structures_helper->module~mod_precision module~mod_structures_helper->module~mod_linalg module~mod_table->module~mod_io module~mod_table->module~mod_precision module~mod_table->module~mod_throw module~mod_table->module~mod_decorations module~mod_timer->module~mod_io module~mod_timer->module~mod_optional_args module~mod_timer->module~mod_precision module~mod_timer->module~mod_throw ieee_arithmetic ieee_arithmetic module~mod_timer->ieee_arithmetic module~cublas_wrapper->iso_c_binding module~cublas_wrapper->module~mod_settings module~mod_abstract_angular_symmetry_function->module~mod_feature_map_neighborlist module~mod_abstract_angular_symmetry_function->module~mod_periodic_table module~mod_abstract_angular_symmetry_function->module~mod_precision module~mod_abstract_angular_symmetry_function->module~mod_structures module~mod_abstract_angular_symmetry_function->module~mod_abstract_symmetry_function module~mod_abstract_radial_symmetry_function->module~mod_feature_map_neighborlist module~mod_abstract_radial_symmetry_function->module~mod_precision module~mod_abstract_radial_symmetry_function->module~mod_structures module~mod_abstract_radial_symmetry_function->module~mod_abstract_symmetry_function module~mod_abstract_symmetry_function->module~mod_feature_map_neighborlist module~mod_abstract_symmetry_function->module~mod_io module~mod_abstract_symmetry_function->module~mod_precision module~mod_abstract_symmetry_function->module~mod_structures module~mod_abstract_symmetry_function->module~mod_throw module~mod_abstract_symmetry_function->module~mod_feature_map module~mod_electric_field_prefactor mod_electric_field_prefactor module~mod_abstract_symmetry_function->module~mod_electric_field_prefactor module~mod_spin_prefactor mod_spin_prefactor module~mod_abstract_symmetry_function->module~mod_spin_prefactor module~mod_arrays_init->module~mod_precision module~mod_cpu_vendor->iso_c_binding module~mod_decorations->module~mod_io module~mod_decorations->module~mod_precision module~mod_decorations->module~mod_throw module~mod_dubious_feature_map_handler->module~mod_io module~mod_dubious_feature_map_handler->module~mod_precision module~mod_dubious_feature_map_handler->module~mod_throw module~mod_dubious_feature_map_handler->module~mod_decorations module~mod_dubious_feature_map_handler->module~mod_dynamic_double_array module~mod_dubious_feature_map_handler->module~mod_dynamic_int_array module~mod_dynamic_line_array mod_dynamic_line_array module~mod_dubious_feature_map_handler->module~mod_dynamic_line_array module~mod_dynamic_double_array->module~mod_precision module~mod_dynamic_double_array->module~mod_throw module~mod_fastmath->module~mod_precision module~mod_hash->module~mod_io module~mod_hash->module~mod_periodic_table module~mod_hash->module~mod_precision module~mod_neighbor_grid->module~mod_mpi module~mod_neighbor_grid->module~mod_precision module~mod_neighbor_grid->module~mod_structures_helper module~mod_neighbor_grid->module~mod_dynamic_int_array module~mod_neighbor_grid->module~mod_neighborlist_helper module~mod_neighborlist_helper->module~mod_mpi module~mod_neighborlist_helper->module~mod_precision module~mod_neighborlist_helper->module~mod_linalg module~mod_sanitize->module~mod_throw module~mod_set_character->module~mod_throw module~mod_storage_container->module~mod_io module~mod_storage_container->module~mod_mpi module~mod_storage_container->module~mod_sort module~mod_storage_container->mod_property_list module~mod_storage_container->module~mod_dynamic_int_array module~mod_abstract_property_list mod_abstract_property_list module~mod_storage_container->module~mod_abstract_property_list module~mod_string mod_string module~mod_storage_container->module~mod_string module~mod_abstract_property_list->module~mod_mpi module~mod_abstract_property_list->module~mod_optional_args module~mod_abstract_property_list->module~mod_throw module~mod_abstract_property_list->module~mod_dynamic_int_array module~mod_abstract_property_list->module~mod_string module~mod_dynamic_int_array_2d mod_dynamic_int_array_2d module~mod_abstract_property_list->module~mod_dynamic_int_array_2d module~mod_dynamic_string_array mod_dynamic_string_array module~mod_abstract_property_list->module~mod_dynamic_string_array module~mod_electric_field_prefactor->module~mod_feature_map_neighborlist module~mod_electric_field_prefactor->module~mod_precision module~mod_electric_field_prefactor->module~mod_settings module~mod_spin_prefactor->module~mod_feature_map_neighborlist module~mod_spin_prefactor->module~mod_precision module~mod_string->module~mod_precision module~mod_dynamic_int_array_2d->module~mod_mpi module~mod_dynamic_string_array->module~mod_string

Used By

module~~mod_dataloader_file~~UsedByGraph module~mod_dataloader_file mod_dataloader_file module~mod_calculate_coulomb_matrix mod_calculate_coulomb_matrix module~mod_calculate_coulomb_matrix->module~mod_dataloader_file module~mod_cost_calculator mod_cost_calculator module~mod_cost_calculator->module~mod_dataloader_file module~mod_write_outfiles_training mod_write_outfiles_training module~mod_cost_calculator->module~mod_write_outfiles_training module~mod_dataset_statistics mod_dataset_statistics module~mod_dataset_statistics->module~mod_dataloader_file module~mod_dataset_statistics_chi_all_chunks mod_dataset_statistics_chi_all_chunks module~mod_dataset_statistics_chi_all_chunks->module~mod_dataloader_file module~mod_dataset_statistics_chi_all_chunks->module~mod_dataset_statistics module~mod_feature_map_scaling_all_chunks mod_feature_map_scaling_all_chunks module~mod_feature_map_scaling_all_chunks->module~mod_dataloader_file module~mod_precalc_nl_deriv_features_single_chunk mod_precalc_nl_deriv_features_single_chunk module~mod_feature_map_scaling_all_chunks->module~mod_precalc_nl_deriv_features_single_chunk module~mod_output_files_and_cost mod_output_files_and_cost module~mod_output_files_and_cost->module~mod_dataloader_file module~mod_output_files_and_cost->module~mod_cost_calculator module~mod_predict_multiple_structures mod_predict_multiple_structures module~mod_output_files_and_cost->module~mod_predict_multiple_structures module~mod_output_files_and_cost_4g_charges mod_output_files_and_cost_4g_charges module~mod_output_files_and_cost_4g_charges->module~mod_dataloader_file module~mod_output_files_and_cost_4g_charges->module~mod_cost_calculator module~mod_output_files_and_cost_4g_short_range mod_output_files_and_cost_4g_short_range module~mod_output_files_and_cost_4g_short_range->module~mod_dataloader_file module~mod_output_files_and_cost_4g_short_range->module~mod_cost_calculator module~mod_predict_qeq_feature_calculator_multiple_structures mod_predict_qeq_feature_calculator_multiple_structures module~mod_output_files_and_cost_4g_short_range->module~mod_predict_qeq_feature_calculator_multiple_structures module~mod_precalc_coulomb_matrix_all_chunks mod_precalc_coulomb_matrix_all_chunks module~mod_precalc_coulomb_matrix_all_chunks->module~mod_dataloader_file module~mod_precalc_nl_deriv_features_single_chunk->module~mod_dataloader_file module~mod_precompute_features_all_chunks mod_precompute_features_all_chunks module~mod_precompute_features_all_chunks->module~mod_dataloader_file module~mod_precompute_global_features_all_chunks mod_precompute_global_features_all_chunks module~mod_precompute_global_features_all_chunks->module~mod_dataloader_file module~mod_predict_ee4g_feature_calculator_multiple_structures mod_predict_ee4g_feature_calculator_multiple_structures module~mod_precompute_global_features_all_chunks->module~mod_predict_ee4g_feature_calculator_multiple_structures module~mod_precompute_global_features_all_chunks->module~mod_predict_qeq_feature_calculator_multiple_structures module~mod_predict_all_chunks mod_predict_all_chunks module~mod_predict_all_chunks->module~mod_dataloader_file module~mod_two_body mod_two_body module~mod_predict_all_chunks->module~mod_two_body module~mod_predict_all_chunks->module~mod_predict_multiple_structures module~mod_predict_ee4g_feature_calculator_multiple_structures->module~mod_dataloader_file module~mod_predict_mode mod_predict_mode module~mod_predict_mode->module~mod_dataloader_file module~mod_predict_mode->module~mod_two_body module~mod_initialize mod_initialize module~mod_predict_mode->module~mod_initialize module~mod_predict_mode->module~mod_predict_multiple_structures module~mod_write_outfiles mod_write_outfiles module~mod_predict_mode->module~mod_write_outfiles module~mod_predict_qeq_feature_calculator_multiple_structures->module~mod_dataloader_file module~mod_train_4g_atomic_charges_structure_batch mod_train_4g_atomic_charges_structure_batch module~mod_train_4g_atomic_charges_structure_batch->module~mod_dataloader_file module~mod_train_loop_4g_charges mod_train_loop_4g_charges module~mod_train_loop_4g_charges->module~mod_dataloader_file module~mod_train_loop_4g_charges->module~mod_cost_calculator module~mod_train_loop_4g_charges->module~mod_output_files_and_cost_4g_charges module~mod_train_loop_4g_charges->module~mod_train_4g_atomic_charges_structure_batch module~mod_train_loop_4g_charges->module~mod_initialize module~mod_train_loop_atomic_property mod_train_loop_atomic_property module~mod_train_loop_atomic_property->module~mod_dataloader_file module~mod_train_loop_atomic_property->module~mod_cost_calculator module~mod_train_loop_atomic_property->module~mod_output_files_and_cost module~mod_train_loop_atomic_property->module~mod_initialize module~mod_train_loop_short_range mod_train_loop_short_range module~mod_train_loop_short_range->module~mod_dataloader_file module~mod_train_loop_short_range->module~mod_cost_calculator module~mod_train_loop_short_range->module~mod_output_files_and_cost module~mod_train_loop_short_range->module~mod_output_files_and_cost_4g_short_range module~mod_train_loop_short_range->module~mod_predict_ee4g_feature_calculator_multiple_structures module~mod_train_loop_short_range->module~mod_predict_qeq_feature_calculator_multiple_structures module~mod_train_loop_short_range->module~mod_initialize module~mod_train_energy_multiple_structures mod_train_energy_multiple_structures module~mod_train_loop_short_range->module~mod_train_energy_multiple_structures module~mod_train_forces_multiple_structures mod_train_forces_multiple_structures module~mod_train_loop_short_range->module~mod_train_forces_multiple_structures module~mod_train_mode mod_train_mode module~mod_train_mode->module~mod_dataloader_file module~mod_train_mode->module~mod_dataset_statistics module~mod_train_mode->module~mod_dataset_statistics_chi_all_chunks module~mod_train_mode->module~mod_feature_map_scaling_all_chunks module~mod_train_mode->module~mod_precalc_coulomb_matrix_all_chunks module~mod_train_mode->module~mod_precalc_nl_deriv_features_single_chunk module~mod_train_mode->module~mod_predict_all_chunks module~mod_train_mode->module~mod_predict_ee4g_feature_calculator_multiple_structures module~mod_train_mode->module~mod_predict_qeq_feature_calculator_multiple_structures module~mod_train_mode->module~mod_train_loop_4g_charges module~mod_train_mode->module~mod_train_loop_atomic_property module~mod_train_mode->module~mod_train_loop_short_range module~mod_train_mode->module~mod_two_body module~mod_train_mode->module~mod_initialize module~mod_train_mode->module~mod_write_outfiles_training module~mod_two_body->module~mod_dataloader_file program~runner RuNNer program~runner->module~mod_dataloader_file program~runner->module~mod_calculate_coulomb_matrix program~runner->module~mod_precompute_features_all_chunks program~runner->module~mod_precompute_global_features_all_chunks program~runner->module~mod_predict_mode program~runner->module~mod_train_mode program~runner->module~mod_two_body program~runner->module~mod_initialize module~mod_initialize->module~mod_two_body module~mod_post_processing_wrapper_single_structure mod_post_processing_wrapper_single_structure module~mod_post_processing_wrapper_single_structure->module~mod_two_body module~mod_predict_multiple_structures->module~mod_two_body module~mod_predict_multiple_structures->module~mod_post_processing_wrapper_single_structure module~mod_predict_single_structure mod_predict_single_structure module~mod_predict_single_structure->module~mod_two_body module~mod_predict_single_structure->module~mod_post_processing_wrapper_single_structure module~mod_runner_interface mod_runner_interface module~mod_runner_interface->module~mod_two_body module~mod_runner_interface_bindings mod_runner_interface_bindings module~mod_runner_interface_bindings->module~mod_two_body module~mod_runner_interface_bindings->module~mod_initialize module~mod_runner_interface_bindings->module~mod_runner_interface module~mod_feature_calc_interface mod_feature_calc_interface module~mod_feature_calc_interface->module~mod_runner_interface module~mod_generic_interface mod_generic_interface module~mod_feature_calc_interface->module~mod_generic_interface module~mod_generic_interface->module~mod_predict_single_structure module~mod_generic_interface->module~mod_runner_interface module~mod_lammps_interface mod_lammps_interface module~mod_lammps_interface->module~mod_runner_interface module~mod_lammps_interface->module~mod_runner_interface_bindings module~mod_train_energy_multiple_structures->module~mod_initialize module~mod_train_forces_multiple_structures->module~mod_initialize module~mod_train_forces_multiple_structures->module~mod_train_energy_multiple_structures module~mod_write_outfiles->module~mod_initialize module~mod_write_outfiles_training->module~mod_write_outfiles

Derived Types Summary

Name Description
FileDataLoader A data loader which reads structures from a file. This class extends the DataBatcher...

Procedures Summary

Name Type Description
get_num_structures_in_chunk Function Return the number of structures in chunk ichunk.
get_id_strucs_in_chunk Function Return the global ids of the structures in chunk ichunk.
get_num_chunks Function Return the number of chunks currently configured.
get_target_chunksize Function Return the target chunksize that was determined or set by the user.
get_chunksize Function Return the size of the currently loaded chunk.
initialize_file_dataloader Subroutine Initialize all variables required by the FileDataLoader.
set_num_structures_in_chunk_per_process Subroutine Run a full pass through the dataset without reading or distributing structures to determine...
load_next_chunk Subroutine Load the next consecutive chunk from self%data_src. This routine will load a block of...
create_chunks Subroutine Separate a given list of indices into blocks. This routine requires self%target_chunksize and...
calculate_num_chunks Subroutine Make a conservative estimate of how many chunks are necessary. The goal is to ensure that the...
rewind_file_dataloader Subroutine Restore the dataloader to the state at the beginning of the first chunk. This will rewind...
estimate_memory_requirements Subroutine Determine, based on RAM size, which input information and intermediate results can be kept in...
set_num_chunks_and_target_chunksize Subroutine Manually set the number of chunks and size of each chunk. Use this to override automatically...

Derived Types Details

type FileDataLoader

A data loader which reads structures from a file.

This class extends the DataBatcher functionality to handle file-based inputs. Its primary purpose is to solve memory bottlenecks when processing very large datasets by loading "chunks" of structures sequentially.

Key features: 1. Chunking: It determines the optimal number of structures to load at once based on available RAM and the size of structures/features. 2. MPI Distribution: When a chunk is loaded, structures are read sequentially (usually by the root process) and distributed to other MPI images in a round-robin fashion. 3. Memory Estimation: It calculates whether features, derivatives, and neighbor lists can be precalculated and stored, or if they must be computed on-the-fly.

Components

  • batcher : type(DataBatcher)
  • num_chunks : integer– How many chunks there are in total.
  • target_chunksize : integer– How large chunks are supposed to be at most.
  • chunksize : integer– The actual number of structures in the currently loaded chunk.
  • data_src : class(AbstractStructureDataFile)– File from which the structures are read.
  • num_structures_in_chunk : integer (dim: (:))– The number of structures in each chunk. Set by the dataloader in `create_chunks`.
  • num_structures_in_chunk_per_process : integer (dim: (:))– The number of structures in each chunk per process. Filled during the first iteration of `load_next_chunk`. Do not use before the first call to rewind!
  • id_strucs_in_chunk : integer (dim: (:,:))– The `id` (global index across all processes) of the structures in each chunk.
  • atom_offset : integer– Global counter across all processes for the atoms. Used to set `atom%id` during parsing the chunk of structures.
  • atom_offset_local_across_chunks : integer– Local counter across all chunks for the atoms on this process.
  • struc_offset_local_across_chunks : integer– Local counter across all chunks for the structures on this process.
  • current_chunk_id : integer– The current chunk ID (1-based index).
  • max_num_structures_per_chunk_per_process : integer– An upper bound estimation for the number of structures that is loaded per chunk and per process.
  • max_num_atoms_per_chunk_per_process : integer– An upper bound estimation for the number of atoms that is loaded per chunk and per process.
  • max_num_structures_across_chunks_per_process : integer– The number of structures that are loaded on one MPI task across all chunks.
  • max_num_atoms_across_chunks_per_process : integer– The number of atoms that are loaded on one MPI task across all chunks.
  • size_structures : real(DP)
  • size_neighbors : real(DP)
  • size_features : real(DP)
  • size_feature_derivatives : real(DP)
  • size_global_features : real(DP)
  • size_global_feature_derivatives : real(DP)
  • size_coulomb_matrix : real(DP)
  • structures : type(Structure) (dim: (:))– The currently loaded structures on this process.
  • neighbor_lists : type(FeatureMapNeighborList) (dim: (:))– A list of neighbor lists, each of them for all `structures`.
  • enforce_features_precalc : logical– Flag to enforce precalculation of features.
  • enforce_feature_deriv_precalc : logical– Flag to enforce precalculation of feature derivatives.
  • enforce_global_features_precalc : logical– Flag to enforce precalculation of global features.
  • enforce_global_feature_deriv_precalc : logical– Flag to enforce precalculation of global feature derivatives.
  • enforce_coulomb_matrix_precalc : logical– Flag to enforce precalculation of the Coulomb matrix.

Type-Bound Procedures

Name Attributes Binding
initialize public ⇒ initialize_file_dataloader
load_next_chunk public ⇒ load_next_chunk
rewind public ⇒ rewind_file_dataloader
calculate_num_chunks public ⇒ calculate_num_chunks
create_chunks public ⇒ create_chunks
get_num_structures_in_chunk public ⇒ get_num_structures_in_chunk
set_num_structures_in_chunk_per_process private ⇒ set_num_structures_in_chunk_per_process
get_id_strucs_in_chunk public ⇒ get_id_strucs_in_chunk
get_num_chunks public ⇒ get_num_chunks
get_chunksize public ⇒ get_chunksize
get_target_chunksize public ⇒ get_target_chunksize
set_num_chunks_and_target_chunksize public ⇒ set_num_chunks_and_target_chunksize
estimate_memory_requirements public ⇒ estimate_memory_requirements

Inherits

type~~filedataloader~~InheritsGraph type~filedataloader FileDataLoader type~abstractstructuredatafile AbstractStructureDataFile type~filedataloader->type~abstractstructuredatafile data_src type~databatcher DataBatcher type~filedataloader->type~databatcher batcher type~featuremapneighborlist FeatureMapNeighborList type~filedataloader->type~featuremapneighborlist neighbor_lists type~structure Structure type~filedataloader->type~structure structures type~datasetstorage DatasetStorage type~abstractstructuredatafile->type~datasetstorage reference_properties type~genericfile GenericFile type~abstractstructuredatafile->type~genericfile type~randomnumbergenerator RandomNumberGenerator type~databatcher->type~randomnumbergenerator rng_shuffle, rng_split type~cutofffunctionptr CutoffFunctionPtr type~featuremapneighborlist->type~cutofffunctionptr cutoff_function type~neighborlist NeighborList type~featuremapneighborlist->type~neighborlist type~abstractcutofffunction AbstractCutoffFunction type~cutofffunctionptr->type~abstractcutofffunction ptr type~storagecontainer StorageContainer type~datasetstorage->type~storagecontainer structure, atom PropertyListInt PropertyListInt type~storagecontainer->PropertyListInt int PropertyListReal PropertyListReal type~storagecontainer->PropertyListReal real

Procedure Details

function get_num_structures_in_chunk

function get_num_structures_in_chunk( &
    self, &
    ichunk &
)

Return the number of structures in chunk ichunk.

Arguments

  • self : class(FileDataLoader) [in]
  • ichunk : integer [in]– The index of the chunk.

Called By

proc~~get_num_structures_in_chunk~~CalledByGraph proc~get_num_structures_in_chunk mod_dataloader_file::FileDataLoader%get_num_structures_in_chunk proc~dataset_statistics_chi_all_chunks mod_dataset_statistics_chi_all_chunks::dataset_statistics_chi_all_chunks proc~dataset_statistics_chi_all_chunks->proc~get_num_structures_in_chunk proc~determine_scaling_factor_and_centering_subtrahend_all_chunks~3 mod_feature_map_scaling_all_chunks::determine_scaling_factor_and_centering_subtrahend_all_chunks proc~determine_scaling_factor_and_centering_subtrahend_all_chunks~3->proc~get_num_structures_in_chunk proc~get_id_strucs_in_chunk mod_dataloader_file::FileDataLoader%get_id_strucs_in_chunk proc~get_id_strucs_in_chunk->proc~get_num_structures_in_chunk proc~output_files_and_cost mod_output_files_and_cost::output_files_and_cost proc~output_files_and_cost->proc~get_num_structures_in_chunk proc~output_files_and_cost_4g_charges mod_output_files_and_cost_4g_charges::output_files_and_cost_4g_charges proc~output_files_and_cost_4g_charges->proc~get_num_structures_in_chunk proc~output_files_and_cost_4g_short_range mod_output_files_and_cost_4g_short_range::output_files_and_cost_4g_short_range proc~output_files_and_cost_4g_short_range->proc~get_num_structures_in_chunk proc~precalc_coulomb_matrix_all_chunks mod_precalc_coulomb_matrix_all_chunks::precalc_coulomb_matrix_all_chunks proc~precalc_coulomb_matrix_all_chunks->proc~get_num_structures_in_chunk proc~precalc_nl_deriv_features_single_chunk mod_precalc_nl_deriv_features_single_chunk::precalc_nl_deriv_features_single_chunk proc~precalc_nl_deriv_features_single_chunk->proc~get_num_structures_in_chunk proc~precompute_coulomb_matrix_all_chunks mod_calculate_coulomb_matrix::precompute_coulomb_matrix_all_chunks proc~precompute_coulomb_matrix_all_chunks->proc~get_num_structures_in_chunk proc~create_chunks mod_dataloader_file::FileDataLoader%create_chunks proc~precompute_coulomb_matrix_all_chunks->proc~create_chunks proc~precompute_features_all_chunks mod_precompute_features_all_chunks::precompute_features_all_chunks proc~precompute_features_all_chunks->proc~get_num_structures_in_chunk proc~precompute_features_all_chunks->proc~create_chunks proc~precompute_global_features_all_chunks mod_precompute_global_features_all_chunks::precompute_global_features_all_chunks proc~precompute_global_features_all_chunks->proc~get_num_structures_in_chunk proc~precompute_global_features_all_chunks->proc~create_chunks proc~predict_all_chunks mod_predict_all_chunks::predict_all_chunks proc~predict_all_chunks->proc~get_num_structures_in_chunk proc~predict_ee4g_feature_calculator_multiple_structures_all_chunks mod_predict_ee4g_feature_calculator_multiple_structures::predict_ee4g_feature_calculator_multiple_structures_all_chunks proc~predict_ee4g_feature_calculator_multiple_structures_all_chunks->proc~get_num_structures_in_chunk proc~predict_mode mod_predict_mode::predict_mode proc~predict_mode->proc~get_num_structures_in_chunk proc~predict_mode->proc~create_chunks proc~predict_two_body_all_chunks mod_two_body::TwoBodyCalculator%predict_two_body_all_chunks proc~predict_two_body_all_chunks->proc~get_num_structures_in_chunk proc~prediction_statistics_all_chunks mod_feature_map_scaling_all_chunks::prediction_statistics_all_chunks proc~prediction_statistics_all_chunks->proc~get_num_structures_in_chunk proc~read_global_features_all_chunks mod_train_mode::read_global_features_all_chunks proc~read_global_features_all_chunks->proc~get_num_structures_in_chunk proc~set_num_structures_in_chunk_per_process mod_dataloader_file::FileDataLoader%set_num_structures_in_chunk_per_process proc~set_num_structures_in_chunk_per_process->proc~get_num_structures_in_chunk proc~train_loop_4g_charges mod_train_loop_4g_charges::train_loop_4g_charges proc~train_loop_4g_charges->proc~get_num_structures_in_chunk proc~train_loop_4g_charges->proc~output_files_and_cost_4g_charges proc~train_loop_atomic_property mod_train_loop_atomic_property::train_loop_atomic_property proc~train_loop_atomic_property->proc~get_num_structures_in_chunk proc~train_loop_atomic_property->proc~output_files_and_cost proc~train_loop_short_range mod_train_loop_short_range::train_loop_short_range proc~train_loop_short_range->proc~get_num_structures_in_chunk proc~train_loop_short_range->proc~output_files_and_cost proc~train_loop_short_range->proc~output_files_and_cost_4g_short_range proc~create_chunks->proc~set_num_structures_in_chunk_per_process proc~feature_scaling_and_prediction_statistics_all_chunks mod_feature_map_scaling_all_chunks::feature_scaling_and_prediction_statistics_all_chunks proc~feature_scaling_and_prediction_statistics_all_chunks->proc~determine_scaling_factor_and_centering_subtrahend_all_chunks~3 proc~feature_scaling_and_prediction_statistics_all_chunks->proc~prediction_statistics_all_chunks proc~train_mode mod_train_mode::train_mode proc~train_mode->proc~dataset_statistics_chi_all_chunks proc~train_mode->proc~precalc_coulomb_matrix_all_chunks proc~train_mode->proc~precalc_nl_deriv_features_single_chunk proc~train_mode->proc~predict_all_chunks proc~train_mode->proc~predict_ee4g_feature_calculator_multiple_structures_all_chunks proc~train_mode->proc~predict_two_body_all_chunks proc~train_mode->proc~read_global_features_all_chunks proc~train_mode->proc~train_loop_4g_charges proc~train_mode->proc~train_loop_atomic_property proc~train_mode->proc~train_loop_short_range proc~train_mode->proc~create_chunks proc~train_mode->proc~feature_scaling_and_prediction_statistics_all_chunks program~runner RuNNer program~runner->proc~precompute_coulomb_matrix_all_chunks program~runner->proc~precompute_features_all_chunks program~runner->proc~precompute_global_features_all_chunks program~runner->proc~predict_mode program~runner->proc~train_mode

function get_id_strucs_in_chunk

function get_id_strucs_in_chunk( &
    self, &
    ichunk &
)

Return the global ids of the structures in chunk ichunk.

Arguments

  • self : class(FileDataLoader) [in]
  • ichunk : integer [in]– The index of the chunk.

function get_num_chunks

function get_num_chunks( &
    self &
)

Return the number of chunks currently configured.

Arguments

Called By

proc~~get_num_chunks~~CalledByGraph proc~get_num_chunks mod_dataloader_file::FileDataLoader%get_num_chunks proc~dataset_statistics_chi_all_chunks mod_dataset_statistics_chi_all_chunks::dataset_statistics_chi_all_chunks proc~dataset_statistics_chi_all_chunks->proc~get_num_chunks proc~load_next_chunk mod_dataloader_file::FileDataLoader%load_next_chunk proc~dataset_statistics_chi_all_chunks->proc~load_next_chunk proc~determine_scaling_factor_and_centering_subtrahend_all_chunks~3 mod_feature_map_scaling_all_chunks::determine_scaling_factor_and_centering_subtrahend_all_chunks proc~determine_scaling_factor_and_centering_subtrahend_all_chunks~3->proc~get_num_chunks proc~determine_scaling_factor_and_centering_subtrahend_all_chunks~3->proc~load_next_chunk proc~load_next_chunk->proc~get_num_chunks proc~output_files_and_cost mod_output_files_and_cost::output_files_and_cost proc~output_files_and_cost->proc~get_num_chunks proc~output_files_and_cost->proc~load_next_chunk proc~output_files_and_cost_4g_charges mod_output_files_and_cost_4g_charges::output_files_and_cost_4g_charges proc~output_files_and_cost_4g_charges->proc~get_num_chunks proc~output_files_and_cost_4g_charges->proc~load_next_chunk proc~output_files_and_cost_4g_short_range mod_output_files_and_cost_4g_short_range::output_files_and_cost_4g_short_range proc~output_files_and_cost_4g_short_range->proc~get_num_chunks proc~output_files_and_cost_4g_short_range->proc~load_next_chunk proc~precalc_nl_deriv_features_single_chunk mod_precalc_nl_deriv_features_single_chunk::precalc_nl_deriv_features_single_chunk proc~precalc_nl_deriv_features_single_chunk->proc~get_num_chunks proc~precalc_nl_deriv_features_single_chunk->proc~load_next_chunk proc~precompute_coulomb_matrix_all_chunks mod_calculate_coulomb_matrix::precompute_coulomb_matrix_all_chunks proc~precompute_coulomb_matrix_all_chunks->proc~get_num_chunks proc~precompute_coulomb_matrix_all_chunks->proc~load_next_chunk proc~create_chunks mod_dataloader_file::FileDataLoader%create_chunks proc~precompute_coulomb_matrix_all_chunks->proc~create_chunks proc~precompute_features_all_chunks mod_precompute_features_all_chunks::precompute_features_all_chunks proc~precompute_features_all_chunks->proc~get_num_chunks proc~precompute_features_all_chunks->proc~load_next_chunk proc~precompute_features_all_chunks->proc~create_chunks proc~precompute_global_features_all_chunks mod_precompute_global_features_all_chunks::precompute_global_features_all_chunks proc~precompute_global_features_all_chunks->proc~get_num_chunks proc~precompute_global_features_all_chunks->proc~load_next_chunk proc~precompute_global_features_all_chunks->proc~create_chunks proc~predict_all_chunks mod_predict_all_chunks::predict_all_chunks proc~predict_all_chunks->proc~get_num_chunks proc~predict_all_chunks->proc~load_next_chunk proc~predict_ee4g_feature_calculator_multiple_structures_all_chunks mod_predict_ee4g_feature_calculator_multiple_structures::predict_ee4g_feature_calculator_multiple_structures_all_chunks proc~predict_ee4g_feature_calculator_multiple_structures_all_chunks->proc~get_num_chunks proc~predict_ee4g_feature_calculator_multiple_structures_all_chunks->proc~load_next_chunk proc~predict_mode mod_predict_mode::predict_mode proc~predict_mode->proc~get_num_chunks proc~predict_mode->proc~load_next_chunk proc~predict_mode->proc~create_chunks proc~predict_two_body_all_chunks mod_two_body::TwoBodyCalculator%predict_two_body_all_chunks proc~predict_two_body_all_chunks->proc~get_num_chunks proc~predict_two_body_all_chunks->proc~load_next_chunk proc~prediction_statistics_all_chunks mod_feature_map_scaling_all_chunks::prediction_statistics_all_chunks proc~prediction_statistics_all_chunks->proc~get_num_chunks proc~prediction_statistics_all_chunks->proc~load_next_chunk proc~read_global_features_all_chunks mod_train_mode::read_global_features_all_chunks proc~read_global_features_all_chunks->proc~get_num_chunks proc~read_global_features_all_chunks->proc~load_next_chunk proc~set_num_structures_in_chunk_per_process mod_dataloader_file::FileDataLoader%set_num_structures_in_chunk_per_process proc~set_num_structures_in_chunk_per_process->proc~get_num_chunks proc~set_num_structures_in_chunk_per_process->proc~load_next_chunk proc~train_loop_4g_charges mod_train_loop_4g_charges::train_loop_4g_charges proc~train_loop_4g_charges->proc~get_num_chunks proc~train_loop_4g_charges->proc~load_next_chunk proc~train_loop_4g_charges->proc~output_files_and_cost_4g_charges proc~train_loop_atomic_property mod_train_loop_atomic_property::train_loop_atomic_property proc~train_loop_atomic_property->proc~get_num_chunks proc~train_loop_atomic_property->proc~load_next_chunk proc~train_loop_atomic_property->proc~output_files_and_cost proc~train_loop_short_range mod_train_loop_short_range::train_loop_short_range proc~train_loop_short_range->proc~get_num_chunks proc~train_loop_short_range->proc~load_next_chunk proc~train_loop_short_range->proc~output_files_and_cost proc~train_loop_short_range->proc~output_files_and_cost_4g_short_range proc~train_mode mod_train_mode::train_mode proc~train_mode->proc~get_num_chunks proc~train_mode->proc~dataset_statistics_chi_all_chunks proc~train_mode->proc~precalc_nl_deriv_features_single_chunk proc~train_mode->proc~predict_all_chunks proc~train_mode->proc~predict_ee4g_feature_calculator_multiple_structures_all_chunks proc~train_mode->proc~predict_two_body_all_chunks proc~train_mode->proc~read_global_features_all_chunks proc~train_mode->proc~train_loop_4g_charges proc~train_mode->proc~train_loop_atomic_property proc~train_mode->proc~train_loop_short_range proc~train_mode->proc~create_chunks proc~feature_scaling_and_prediction_statistics_all_chunks mod_feature_map_scaling_all_chunks::feature_scaling_and_prediction_statistics_all_chunks proc~train_mode->proc~feature_scaling_and_prediction_statistics_all_chunks proc~precalc_coulomb_matrix_all_chunks mod_precalc_coulomb_matrix_all_chunks::precalc_coulomb_matrix_all_chunks proc~train_mode->proc~precalc_coulomb_matrix_all_chunks proc~create_chunks->proc~set_num_structures_in_chunk_per_process proc~feature_scaling_and_prediction_statistics_all_chunks->proc~determine_scaling_factor_and_centering_subtrahend_all_chunks~3 proc~feature_scaling_and_prediction_statistics_all_chunks->proc~prediction_statistics_all_chunks proc~precalc_coulomb_matrix_all_chunks->proc~load_next_chunk program~runner RuNNer program~runner->proc~precompute_coulomb_matrix_all_chunks program~runner->proc~precompute_features_all_chunks program~runner->proc~precompute_global_features_all_chunks program~runner->proc~predict_mode program~runner->proc~train_mode

function get_target_chunksize

function get_target_chunksize( &
    self &
)

Return the target chunksize that was determined or set by the user.

Arguments

function get_chunksize

function get_chunksize( &
    self &
)

Return the size of the currently loaded chunk.

Arguments

subroutine initialize_file_dataloader

subroutine initialize_file_dataloader( &
    self, &
    data_src, &
    seed_shuffle, &
    seed_split, &
    enforce_features_precalc, &
    enforce_feature_deriv_precalc, &
    enforce_global_features_precalc, &
    enforce_global_feature_deriv_precalc, &
    enforce_coulomb_matrix_precalc &
)

Initialize all variables required by the FileDataLoader.

Arguments

  • self : class(FileDataLoader) [inout]
  • data_src : class(AbstractStructureDataFile) [in]– The file from which data is read.
  • seed_shuffle : integer [in]– Seed for shuffling structures/atoms (e.g. for batches).
  • seed_split : integer [in]– Seed for splitting lists into smaller parts (e.g. train/test split).
  • enforce_features_precalc : logical [in]– Flag to enforce precalculation of features.
  • enforce_feature_deriv_precalc : logical [in]– Flag to enforce precalculation of feature derivatives.
  • enforce_global_features_precalc : logical [in]– Flag to enforce precalculation of global features.
  • enforce_global_feature_deriv_precalc : logical [in]– Flag to enforce precalculation of global feature derivatives.
  • enforce_coulomb_matrix_precalc : logical [in]– Flag to enforce precalculation of the Coulomb matrix.

Called By

proc~~initialize_file_dataloader~~CalledByGraph proc~initialize_file_dataloader mod_dataloader_file::FileDataLoader%initialize_file_dataloader program~runner RuNNer program~runner->proc~initialize_file_dataloader

subroutine set_num_structures_in_chunk_per_process

subroutine set_num_structures_in_chunk_per_process( &
    self, &
    split, &
    num_samples_per_part &
)

Run a full pass through the dataset without reading or distributing structures to determine self%num_structures_in_chunk_per_process.

Arguments

  • self : class(FileDataLoader) [inout]
  • split : integer (dim: (:,:)) [in]– A mapping array where `split(id, :)` defines the part assignment. If given, the distribution tracks which structures belong to which dataset partition (e.g. train/test).
  • num_samples_per_part : integer (dim: (:)) [in]– The total number of samples in each part of the split. Required if `split` is present.

Called By

proc~~set_num_structures_in_chunk_per_process~~CalledByGraph proc~set_num_structures_in_chunk_per_process mod_dataloader_file::FileDataLoader%set_num_structures_in_chunk_per_process proc~create_chunks mod_dataloader_file::FileDataLoader%create_chunks proc~create_chunks->proc~set_num_structures_in_chunk_per_process proc~precompute_coulomb_matrix_all_chunks mod_calculate_coulomb_matrix::precompute_coulomb_matrix_all_chunks proc~precompute_coulomb_matrix_all_chunks->proc~create_chunks proc~precompute_features_all_chunks mod_precompute_features_all_chunks::precompute_features_all_chunks proc~precompute_features_all_chunks->proc~create_chunks proc~precompute_global_features_all_chunks mod_precompute_global_features_all_chunks::precompute_global_features_all_chunks proc~precompute_global_features_all_chunks->proc~create_chunks proc~predict_mode mod_predict_mode::predict_mode proc~predict_mode->proc~create_chunks proc~train_mode mod_train_mode::train_mode proc~train_mode->proc~create_chunks program~runner RuNNer program~runner->proc~precompute_coulomb_matrix_all_chunks program~runner->proc~precompute_features_all_chunks program~runner->proc~precompute_global_features_all_chunks program~runner->proc~predict_mode program~runner->proc~train_mode

subroutine load_next_chunk

subroutine load_next_chunk( &
    self, &
    num_structures, &
    cutoff_functions, &
    bond_threshold, &
    split, &
    num_samples_per_part, &
    force_reload, &
    calculate_neighbor_lists, &
    split_chunk, &
    num_samples_per_part_chunk, &
    mean_num_neighbors, &
    skip_distribution &
)

Load the next consecutive chunk from self%data_src.

This routine will load a block of num_structures structures from the data source, starting at the position where the data source last stopped parsing.

Structures are read strictly sequentially (usually on the root process) and distributed round-robin to all MPI processes. This minimizes file seek times.

If split information is provided (e.g., for train/test splitting), this routine also determines which split part each loaded structure belongs to.

Use self%rewind to reset the file pointer to the beginning.

Arguments

  • self : class(FileDataLoader) [inout]
  • num_structures : integer [in]– The number of structures that will be read from the file in this chunk.
  • cutoff_functions : type(CutoffFunctionPtr) (dim: (:)) [in]– The cutoff functions for the neighbor lists. Only required when `calculate_neighbor_lists` is true.
  • bond_threshold : real(DP) [in]– The bond threshold used for neighbor list generation.
  • split : integer (dim: (:,:)) [in]– A mapping array where `split(id, :)` defines the part assignment. If given, the distribution tracks which structures belong to which dataset partition (e.g. train/test).
  • num_samples_per_part : integer (dim: (:)) [in]– The total number of samples in each part of the split. Required if `split` is present.
  • force_reload : logical [in]– Whether to reload the structures from the file, even if they are already loaded and there is only one chunk.
  • calculate_neighbor_lists : logical [in]– Whether to generate neighbor lists immediately after loading the structures.
  • split_chunk : integer (dim: (:,:)) [out]– If `split` is given, this returns the local indices (on this process) of the samples in this chunk for each part. Dimensions: (max_samples_local, num_parts).
  • num_samples_per_part_chunk : integer (dim: (:)) [out]– If `split` is given, this returns the count of samples in each part found within this specific chunk.
  • mean_num_neighbors : integer [in]– An estimate for the mean number of neighbors, used for memory pre-allocation of neighbor lists.
  • skip_distribution : logical [in]– Do not read structures on root or distribute them across MPI processes. This is sometimes needed to count the number of structures in the chunks before actually parsing the dataset.

Called By

proc~~load_next_chunk~~CalledByGraph proc~load_next_chunk mod_dataloader_file::FileDataLoader%load_next_chunk proc~dataset_statistics_chi_all_chunks mod_dataset_statistics_chi_all_chunks::dataset_statistics_chi_all_chunks proc~dataset_statistics_chi_all_chunks->proc~load_next_chunk proc~determine_scaling_factor_and_centering_subtrahend_all_chunks~3 mod_feature_map_scaling_all_chunks::determine_scaling_factor_and_centering_subtrahend_all_chunks proc~determine_scaling_factor_and_centering_subtrahend_all_chunks~3->proc~load_next_chunk proc~output_files_and_cost mod_output_files_and_cost::output_files_and_cost proc~output_files_and_cost->proc~load_next_chunk proc~output_files_and_cost_4g_charges mod_output_files_and_cost_4g_charges::output_files_and_cost_4g_charges proc~output_files_and_cost_4g_charges->proc~load_next_chunk proc~output_files_and_cost_4g_short_range mod_output_files_and_cost_4g_short_range::output_files_and_cost_4g_short_range proc~output_files_and_cost_4g_short_range->proc~load_next_chunk proc~precalc_coulomb_matrix_all_chunks mod_precalc_coulomb_matrix_all_chunks::precalc_coulomb_matrix_all_chunks proc~precalc_coulomb_matrix_all_chunks->proc~load_next_chunk proc~precalc_nl_deriv_features_single_chunk mod_precalc_nl_deriv_features_single_chunk::precalc_nl_deriv_features_single_chunk proc~precalc_nl_deriv_features_single_chunk->proc~load_next_chunk proc~precompute_coulomb_matrix_all_chunks mod_calculate_coulomb_matrix::precompute_coulomb_matrix_all_chunks proc~precompute_coulomb_matrix_all_chunks->proc~load_next_chunk proc~create_chunks mod_dataloader_file::FileDataLoader%create_chunks proc~precompute_coulomb_matrix_all_chunks->proc~create_chunks proc~precompute_features_all_chunks mod_precompute_features_all_chunks::precompute_features_all_chunks proc~precompute_features_all_chunks->proc~load_next_chunk proc~precompute_features_all_chunks->proc~create_chunks proc~precompute_global_features_all_chunks mod_precompute_global_features_all_chunks::precompute_global_features_all_chunks proc~precompute_global_features_all_chunks->proc~load_next_chunk proc~precompute_global_features_all_chunks->proc~create_chunks proc~predict_all_chunks mod_predict_all_chunks::predict_all_chunks proc~predict_all_chunks->proc~load_next_chunk proc~predict_ee4g_feature_calculator_multiple_structures_all_chunks mod_predict_ee4g_feature_calculator_multiple_structures::predict_ee4g_feature_calculator_multiple_structures_all_chunks proc~predict_ee4g_feature_calculator_multiple_structures_all_chunks->proc~load_next_chunk proc~predict_mode mod_predict_mode::predict_mode proc~predict_mode->proc~load_next_chunk proc~predict_mode->proc~create_chunks proc~predict_two_body_all_chunks mod_two_body::TwoBodyCalculator%predict_two_body_all_chunks proc~predict_two_body_all_chunks->proc~load_next_chunk proc~prediction_statistics_all_chunks mod_feature_map_scaling_all_chunks::prediction_statistics_all_chunks proc~prediction_statistics_all_chunks->proc~load_next_chunk proc~read_global_features_all_chunks mod_train_mode::read_global_features_all_chunks proc~read_global_features_all_chunks->proc~load_next_chunk proc~set_num_structures_in_chunk_per_process mod_dataloader_file::FileDataLoader%set_num_structures_in_chunk_per_process proc~set_num_structures_in_chunk_per_process->proc~load_next_chunk proc~train_loop_4g_charges mod_train_loop_4g_charges::train_loop_4g_charges proc~train_loop_4g_charges->proc~load_next_chunk proc~train_loop_4g_charges->proc~output_files_and_cost_4g_charges proc~train_loop_atomic_property mod_train_loop_atomic_property::train_loop_atomic_property proc~train_loop_atomic_property->proc~load_next_chunk proc~train_loop_atomic_property->proc~output_files_and_cost proc~train_loop_short_range mod_train_loop_short_range::train_loop_short_range proc~train_loop_short_range->proc~load_next_chunk proc~train_loop_short_range->proc~output_files_and_cost proc~train_loop_short_range->proc~output_files_and_cost_4g_short_range proc~create_chunks->proc~set_num_structures_in_chunk_per_process proc~feature_scaling_and_prediction_statistics_all_chunks mod_feature_map_scaling_all_chunks::feature_scaling_and_prediction_statistics_all_chunks proc~feature_scaling_and_prediction_statistics_all_chunks->proc~determine_scaling_factor_and_centering_subtrahend_all_chunks~3 proc~feature_scaling_and_prediction_statistics_all_chunks->proc~prediction_statistics_all_chunks proc~train_mode mod_train_mode::train_mode proc~train_mode->proc~dataset_statistics_chi_all_chunks proc~train_mode->proc~precalc_coulomb_matrix_all_chunks proc~train_mode->proc~precalc_nl_deriv_features_single_chunk proc~train_mode->proc~predict_all_chunks proc~train_mode->proc~predict_ee4g_feature_calculator_multiple_structures_all_chunks proc~train_mode->proc~predict_two_body_all_chunks proc~train_mode->proc~read_global_features_all_chunks proc~train_mode->proc~train_loop_4g_charges proc~train_mode->proc~train_loop_atomic_property proc~train_mode->proc~train_loop_short_range proc~train_mode->proc~create_chunks proc~train_mode->proc~feature_scaling_and_prediction_statistics_all_chunks program~runner RuNNer program~runner->proc~precompute_coulomb_matrix_all_chunks program~runner->proc~precompute_features_all_chunks program~runner->proc~precompute_global_features_all_chunks program~runner->proc~predict_mode program~runner->proc~train_mode

subroutine create_chunks

subroutine create_chunks( &
    self, &
    ids, &
    shuffle, &
    split, &
    num_samples_per_part &
)

Separate a given list of indices into blocks.

This routine requires self%target_chunksize and self%num_chunks to be set (via calculate_num_chunks or set_num_chunks...) before it is called.

It allocates and fills the chunk tracking arrays (id_strucs_in_chunk, num_structures_in_chunk, etc.) which drive the load_next_chunk routine.

Arguments

  • self : class(FileDataLoader) [inout]
  • ids : integer (dim: (:)) [in]– The full list of (structure) indices that will be chunked.
  • shuffle : logical [in]– Whether to shuffle the indices before creating the chunks. If `shuffle == True`, the order in the file will not match the processing order.
  • split : integer (dim: (:,:)) [in]– A mapping array where `split(id, :)` defines the part assignment. If given, the distribution tracks which structures belong to which dataset partition (e.g. train/test).
  • num_samples_per_part : integer (dim: (:)) [in]– The total number of samples in each part of the split. Required if `split` is present.

Called By

proc~~create_chunks~~CalledByGraph proc~create_chunks mod_dataloader_file::FileDataLoader%create_chunks proc~precompute_coulomb_matrix_all_chunks mod_calculate_coulomb_matrix::precompute_coulomb_matrix_all_chunks proc~precompute_coulomb_matrix_all_chunks->proc~create_chunks proc~precompute_features_all_chunks mod_precompute_features_all_chunks::precompute_features_all_chunks proc~precompute_features_all_chunks->proc~create_chunks proc~precompute_global_features_all_chunks mod_precompute_global_features_all_chunks::precompute_global_features_all_chunks proc~precompute_global_features_all_chunks->proc~create_chunks proc~predict_mode mod_predict_mode::predict_mode proc~predict_mode->proc~create_chunks proc~train_mode mod_train_mode::train_mode proc~train_mode->proc~create_chunks program~runner RuNNer program~runner->proc~precompute_coulomb_matrix_all_chunks program~runner->proc~precompute_features_all_chunks program~runner->proc~precompute_global_features_all_chunks program~runner->proc~predict_mode program~runner->proc~train_mode

subroutine calculate_num_chunks

subroutine calculate_num_chunks( &
    self, &
    available_ram, &
    luse_features, &
    luse_global_features, &
    luse_global_feature_derivatives, &
    luse_feature_derivatives, &
    num_neighborlists, &
    feature_calculators, &
    num_global_features, &
    max_num_neighbors_global_features, &
    luse_coulomb_matrix &
)

Make a conservative estimate of how many chunks are necessary.

The goal is to ensure that the structures, neighbor lists, and associated feature data of a single chunk do not exceed available_ram on any process.

This takes into account the number of MPI images. Since structures within a chunk are distributed across all images, using more MPI images allows for larger chunks (or fewer chunks total).

Arguments

  • self : class(FileDataLoader) [inout]
  • available_ram : real(DP) [in]– How much RAM is available per MPI process for the storage of structures and neighbor lists.
  • luse_features : logical [in]– Whether atomic features are being used.
  • luse_global_features : logical [in]– Whether global features are being used.
  • luse_global_feature_derivatives : logical [in]– Whether global feature derivatives are being used.
  • luse_feature_derivatives : logical [in]– Whether atomic feature derivatives are being used.
  • num_neighborlists : integer [in]– How many neighbor lists will be created for the given structures.
  • feature_calculators : type(FeatureCalculator) (dim: (:)) [in]– Calculator objects used to estimate feature size.
  • num_global_features : integer [in]– Number of global features per structure.
  • max_num_neighbors_global_features : integer (dim: (:)) [in]– Upper bound on neighbors for global features.
  • luse_coulomb_matrix : logical [in]– Wether coulomb matrix is stored.

Called By

proc~~calculate_num_chunks~~CalledByGraph proc~calculate_num_chunks mod_dataloader_file::FileDataLoader%calculate_num_chunks proc~precompute_coulomb_matrix_all_chunks mod_calculate_coulomb_matrix::precompute_coulomb_matrix_all_chunks proc~precompute_coulomb_matrix_all_chunks->proc~calculate_num_chunks proc~precompute_features_all_chunks mod_precompute_features_all_chunks::precompute_features_all_chunks proc~precompute_features_all_chunks->proc~calculate_num_chunks proc~precompute_global_features_all_chunks mod_precompute_global_features_all_chunks::precompute_global_features_all_chunks proc~precompute_global_features_all_chunks->proc~calculate_num_chunks proc~predict_mode mod_predict_mode::predict_mode proc~predict_mode->proc~calculate_num_chunks proc~train_mode mod_train_mode::train_mode proc~train_mode->proc~calculate_num_chunks program~runner RuNNer program~runner->proc~precompute_coulomb_matrix_all_chunks program~runner->proc~precompute_features_all_chunks program~runner->proc~precompute_global_features_all_chunks program~runner->proc~predict_mode program~runner->proc~train_mode

subroutine rewind_file_dataloader

subroutine rewind_file_dataloader( &
    self, &
    reset_rng_split &
)

Restore the dataloader to the state at the beginning of the first chunk.

This will rewind self%data_src (the file pointer) and crucially also reinitialize the random number generator for splitting, such that the same splits (e.g. train/test) are obtained reliably every epoch.

Arguments

  • self : class(FileDataLoader) [inout]
  • reset_rng_split : logical [in]– Whether to reset the RNG used for splitting operations.

subroutine estimate_memory_requirements

subroutine estimate_memory_requirements( &
    self, &
    available_ram, &
    num_structures, &
    num_atoms, &
    num_atoms_per_element, &
    num_neighborlists, &
    mean_num_neighbors, &
    feature_calculators, &
    num_global_features, &
    max_num_neighbors_global_features, &
    precompute_features, &
    precompute_feature_derivatives, &
    precompute_global_features, &
    precompute_global_feature_derivatives, &
    precompute_coulomb_matrix &
)

Determine, based on RAM size, which input information and intermediate results can be kept in memory and which have to calculated on the fly.

This routine estimates the size of structures, neighbor lists, features, feature derivatives, and Coulomb matrices. It then sets the corresponding output boolean flags (e.g., precompute_features) to true if the data fits in available_ram, or false otherwise.

Arguments

  • self : class(FileDataLoader) [inout]
  • available_ram : real(DP) [in]– The amount of RAM available per process.
  • num_structures : integer [in]– Total number of structures.
  • num_atoms : integer [in]– Total number of atoms.
  • num_atoms_per_element : integer (dim: (MAX_ELEM_NUM)) [in]– Atom count per chemical element.
  • num_neighborlists : integer [in]– Number of neighbor lists needed.
  • mean_num_neighbors : real(DP) [in]– Average neighbors per atom.
  • feature_calculators : type(FeatureCalculator) (dim: (:)) [in]– Feature definitions.
  • num_global_features : integer [in]– Number of global features.
  • max_num_neighbors_global_features : integer (dim: (:)) [in]– Neighbors for global features.
  • precompute_features : logical [out]– Result: True if features should be stored in RAM.
  • precompute_feature_derivatives : logical [out]– Result: True if feature derivatives should be stored in RAM.
  • precompute_global_features : logical [out]– Result: True if global features should be stored in RAM.
  • precompute_global_feature_derivatives : logical [out]– Result: True if global feature derivatives should be stored in RAM.
  • precompute_coulomb_matrix : logical [out]

Called By

proc~~estimate_memory_requirements~~CalledByGraph proc~estimate_memory_requirements mod_dataloader_file::FileDataLoader%estimate_memory_requirements proc~precalc_nl_deriv_features_single_chunk mod_precalc_nl_deriv_features_single_chunk::precalc_nl_deriv_features_single_chunk proc~precalc_nl_deriv_features_single_chunk->proc~estimate_memory_requirements proc~predict_mode mod_predict_mode::predict_mode proc~predict_mode->proc~estimate_memory_requirements proc~train_mode mod_train_mode::train_mode proc~train_mode->proc~precalc_nl_deriv_features_single_chunk program~runner RuNNer program~runner->proc~predict_mode program~runner->proc~train_mode

subroutine set_num_chunks_and_target_chunksize

subroutine set_num_chunks_and_target_chunksize( &
    self, &
    num_chunks, &
    chunksize &
)

Manually set the number of chunks and size of each chunk.

Use this to override automatically determined chunking parameters.

Arguments

  • self : class(FileDataLoader) [inout]
  • num_chunks : integer [in]– The total number of chunks to use.
  • chunksize : integer [in]– The maximum number of structures per chunk.