Gromacs
5.1
|
#include <stdio.h>
#include "gromacs/legacyheaders/vsite.h"
#include "gromacs/legacyheaders/types/commrec_fwd.h"
#include "gromacs/legacyheaders/types/constr.h"
#include "gromacs/legacyheaders/types/forcerec.h"
#include "gromacs/legacyheaders/types/hw_info.h"
#include "gromacs/legacyheaders/types/inputrec.h"
#include "gromacs/legacyheaders/types/mdatom.h"
#include "gromacs/legacyheaders/types/nrnb.h"
#include "gromacs/legacyheaders/types/shellfc.h"
#include "gromacs/legacyheaders/types/state.h"
#include "gromacs/math/vectypes.h"
#include "gromacs/timing/wallcycle.h"
#include "gromacs/topology/block.h"
#include "gromacs/topology/idef.h"
#include "gromacs/topology/topology.h"
#include "gromacs/utility/basedefinitions.h"
#include "gromacs/utility/real.h"
This file declares functions for mdrun to call to manage the details of its domain decomposition.
Enumerations | |
enum | { ddCyclStep, ddCyclPPduringPME, ddCyclF, ddCyclWaitGPU, ddCyclPME, ddCyclNr } |
Cycle counter indices used internally in the domain decomposition. | |
Functions | |
int | ddglatnr (gmx_domdec_t *dd, int i) |
Returns the global topology atom number belonging to local atom index i. More... | |
t_block * | dd_charge_groups_global (gmx_domdec_t *dd) |
Return a block struct for the charge groups of the whole system. | |
void | dd_store_state (gmx_domdec_t *dd, t_state *state) |
Store the global cg indices of the home cgs in state,. More... | |
gmx_domdec_zones_t * | domdec_zones (gmx_domdec_t *dd) |
Returns a pointer to the gmx_domdec_zones_t struct. | |
void | dd_get_ns_ranges (gmx_domdec_t *dd, int icg, int *jcg0, int *jcg1, ivec shift0, ivec shift1) |
Sets the j-charge-group range for i-charge-group icg . | |
int | dd_natoms_vsite (gmx_domdec_t *dd) |
Returns the atom range in the local state for atoms involved in virtual sites. | |
void | dd_get_constraint_range (gmx_domdec_t *dd, int *at_start, int *at_end) |
Sets the atom range for atom in the local state for atoms received in constraints communication. | |
void | get_pme_nnodes (const gmx_domdec_t *dd, int *npmenodes_x, int *npmenodes_y) |
Get the number of PME nodes along x and y, can be called with dd=NULL. | |
void | get_pme_ddnodes (t_commrec *cr, int pmenodeid, int *nmy_ddnodes, int **my_ddnodes, int *node_peer) |
Returns the set of DD nodes that communicate with pme node cr->nodeid. | |
int | dd_pme_maxshift_x (gmx_domdec_t *dd) |
Returns the maximum shift for coordinate communication in PME, dim x. | |
int | dd_pme_maxshift_y (gmx_domdec_t *dd) |
Returns the maximum shift for coordinate communication in PME, dim y. | |
void | make_dd_communicators (FILE *fplog, t_commrec *cr, int dd_node_order) |
Generates the MPI communicators for domain decomposition. | |
gmx_domdec_t * | init_domain_decomposition (FILE *fplog, t_commrec *cr, unsigned long Flags, ivec nc, real comm_distance_min, real rconstr, const char *dlb_opt, real dlb_scale, const char *sizex, const char *sizey, const char *sizez, gmx_mtop_t *mtop, t_inputrec *ir, matrix box, rvec *x, gmx_ddbox_t *ddbox, int *npme_x, int *npme_y) |
Initialized the domain decomposition, chooses the DD grid and PME ranks. | |
void | dd_init_bondeds (FILE *fplog, gmx_domdec_t *dd, gmx_mtop_t *mtop, gmx_vsite_t *vsite, t_inputrec *ir, gmx_bool bBCheck, cginfo_mb_t *cginfo_mb) |
Initialize data structures for bonded interactions. | |
gmx_bool | dd_bonded_molpbc (gmx_domdec_t *dd, int ePBC) |
Returns if we need to do pbc for calculating bonded interactions. | |
void | set_dd_parameters (FILE *fplog, gmx_domdec_t *dd, real dlb_scale, t_inputrec *ir, gmx_ddbox_t *ddbox) |
Set DD grid dimensions and limits. More... | |
gmx_bool | change_dd_cutoff (t_commrec *cr, t_state *state, t_inputrec *ir, real cutoff_req) |
Change the DD non-bonded communication cut-off. More... | |
void | set_dd_dlb_max_cutoff (t_commrec *cr, real cutoff) |
Limit DLB to preserve the option of returning to the current cut-off. More... | |
gmx_bool | dd_dlb_is_on (const gmx_domdec_t *dd) |
Return if we are currently using dynamic load balancing. | |
gmx_bool | dd_dlb_is_locked (const gmx_domdec_t *dd) |
Return if the DLB lock is set. | |
void | dd_dlb_lock (gmx_domdec_t *dd) |
Set a lock such that with DLB=auto DLB cannot get turned on. | |
void | dd_dlb_unlock (gmx_domdec_t *dd) |
Clear a lock such that with DLB=auto DLB may get turned on later. | |
void | dd_setup_dlb_resource_sharing (t_commrec *cr, const gmx_hw_info_t *hwinfo, const gmx_hw_opt_t *hw_opt) |
Set up communication for averaging GPU wait times over ranks. More... | |
void | setup_dd_grid (FILE *fplog, gmx_domdec_t *dd) |
Sets up the DD communication setup. | |
void | dd_collect_vec (gmx_domdec_t *dd, t_state *state_local, rvec *lv, rvec *v) |
Collects local rvec arrays lv to v on the master rank. | |
void | dd_collect_state (gmx_domdec_t *dd, t_state *state_local, t_state *state) |
Collects the local state state_local to state on the master rank. | |
void | dd_cycles_add (gmx_domdec_t *dd, float cycles, int ddCycl) |
Add the wallcycle count to the DD counter. | |
void | dd_force_flop_start (gmx_domdec_t *dd, t_nrnb *nrnb) |
Start the force flop count. | |
void | dd_force_flop_stop (gmx_domdec_t *dd, t_nrnb *nrnb) |
Stop the force flop count. | |
float | dd_pme_f_ratio (gmx_domdec_t *dd) |
Return the PME/PP force load ratio, or -1 if nothing was measured. More... | |
void | dd_move_x (gmx_domdec_t *dd, matrix box, rvec x[]) |
Communicate the coordinates to the neighboring cells and do pbc. | |
void | dd_move_f (gmx_domdec_t *dd, rvec f[], rvec *fshift) |
Sum the forces over the neighboring cells. More... | |
void | dd_atom_spread_real (gmx_domdec_t *dd, real v[]) |
Communicate a real for each atom to the neighboring cells. | |
void | dd_atom_sum_real (gmx_domdec_t *dd, real v[]) |
Sum the contributions to a real for each atom over the neighboring cells. | |
void | dd_partition_system (FILE *fplog, gmx_int64_t step, t_commrec *cr, gmx_bool bMasterState, int nstglobalcomm, t_state *state_global, gmx_mtop_t *top_global, t_inputrec *ir, t_state *state_local, rvec **f, t_mdatoms *mdatoms, gmx_localtop_t *top_local, t_forcerec *fr, gmx_vsite_t *vsite, gmx_shellfc_t shellfc, gmx_constr_t constr, t_nrnb *nrnb, gmx_wallcycle_t wcycle, gmx_bool bVerbose) |
Partition the system over the nodes. More... | |
void | reset_dd_statistics_counters (gmx_domdec_t *dd) |
Reset all the statistics and counters for total run counting. | |
void | print_dd_statistics (t_commrec *cr, t_inputrec *ir, FILE *fplog) |
Print statistics for domain decomposition communication. | |
void | dd_move_f_vsites (gmx_domdec_t *dd, rvec *f, rvec *fshift) |
Communicates the virtual site forces, reduces the shift forces when fshift != NULL. | |
void | dd_clear_f_vsites (gmx_domdec_t *dd, rvec *f) |
Clears the forces for virtual sites. | |
void | dd_move_x_constraints (gmx_domdec_t *dd, matrix box, rvec *x0, rvec *x1, gmx_bool bX1IsCoord) |
Move x0 and also x1 if x1!=NULL. bX1IsCoord tells if to do PBC on x1. | |
void | dd_move_x_vsites (gmx_domdec_t *dd, matrix box, rvec *x) |
Communicates the coordinates involved in virtual sites. | |
int * | dd_constraints_nlocalatoms (gmx_domdec_t *dd) |
Returns the local atom count array for all constraints. More... | |
void | dd_print_missing_interactions (FILE *fplog, t_commrec *cr, int local_count, gmx_mtop_t *top_global, t_state *state_local) |
Print error output when interactions are missing. | |
void | dd_make_reverse_top (FILE *fplog, gmx_domdec_t *dd, gmx_mtop_t *mtop, gmx_vsite_t *vsite, t_inputrec *ir, gmx_bool bBCheck) |
Generate and store the reverse topology. | |
void | dd_make_local_cgs (gmx_domdec_t *dd, t_block *lcgs) |
Store the local charge group index in lcgs . | |
void | dd_make_local_top (gmx_domdec_t *dd, gmx_domdec_zones_t *zones, int npbcdim, matrix box, rvec cellsize_min, ivec npulse, t_forcerec *fr, rvec *cgcm_or_x, gmx_vsite_t *vsite, gmx_mtop_t *top, gmx_localtop_t *ltop) |
Generate the local topology and virtual site data. | |
void | dd_sort_local_top (gmx_domdec_t *dd, t_mdatoms *mdatoms, gmx_localtop_t *ltop) |
Sort ltop->ilist when we are doing free energy. | |
gmx_localtop_t * | dd_init_local_top (gmx_mtop_t *top_global) |
Construct local topology. | |
void | dd_init_local_state (gmx_domdec_t *dd, t_state *state_global, t_state *local_state) |
Construct local state. | |
t_blocka * | make_charge_group_links (gmx_mtop_t *mtop, gmx_domdec_t *dd, cginfo_mb_t *cginfo_mb) |
Generate a list of links between charge groups that are linked by bonded interactions. | |
void | dd_bonded_cg_distance (FILE *fplog, gmx_mtop_t *mtop, t_inputrec *ir, rvec *x, matrix box, gmx_bool bBCheck, real *r_2b, real *r_mb) |
Calculate the maximum distance involved in 2-body and multi-body bonded interactions. | |
void | write_dd_pdb (const char *fn, gmx_int64_t step, const char *title, gmx_mtop_t *mtop, t_commrec *cr, int natoms, rvec x[], matrix box) |
Dump a pdb file with the current DD home + communicated atoms. More... | |
real | comm_box_frac (ivec dd_nc, real cutoff, gmx_ddbox_t *ddbox) |
Returns the volume fraction of the system that is communicated. | |
real | dd_choose_grid (FILE *fplog, t_commrec *cr, gmx_domdec_t *dd, t_inputrec *ir, gmx_mtop_t *mtop, matrix box, gmx_ddbox_t *ddbox, gmx_bool bDynLoadBal, real dlb_scale, real cellsize_limit, real cutoff_dd, gmx_bool bInterCGBondeds) |
Determines the optimal DD cell setup dd->nc and possibly npmenodes for the system. More... | |
void | set_ddbox (gmx_domdec_t *dd, gmx_bool bMasterState, t_commrec *cr_sum, t_inputrec *ir, matrix box, gmx_bool bCalcUnboundedSize, t_block *cgs, rvec *x, gmx_ddbox_t *ddbox) |
Set the box and PBC data in ddbox . | |
void | set_ddbox_cr (t_commrec *cr, ivec *dd_nc, t_inputrec *ir, matrix box, t_block *cgs, rvec *x, gmx_ddbox_t *ddbox) |
Set the box and PBC data in ddbox . | |
Change the DD non-bonded communication cut-off.
This could fail when trying to increase the cut-off, then FALSE will be returned and the cut-off is not modified.
real dd_choose_grid | ( | FILE * | fplog, |
t_commrec * | cr, | ||
gmx_domdec_t * | dd, | ||
t_inputrec * | ir, | ||
gmx_mtop_t * | mtop, | ||
matrix | box, | ||
gmx_ddbox_t * | ddbox, | ||
gmx_bool | bDynLoadBal, | ||
real | dlb_scale, | ||
real | cellsize_limit, | ||
real | cutoff_dd, | ||
gmx_bool | bInterCGBondeds | ||
) |
Determines the optimal DD cell setup dd->nc and possibly npmenodes for the system.
On the master node returns the actual cellsize limit used.
int* dd_constraints_nlocalatoms | ( | gmx_domdec_t * | dd | ) |
Returns the local atom count array for all constraints.
The local atom count for a constraint, possible values 2/1/0, is needed to avoid not/double-counting contributions linked to the Lagrange multiplier, such as the virial and free-energy derivatives.
void dd_move_f | ( | gmx_domdec_t * | dd, |
rvec | f[], | ||
rvec * | fshift | ||
) |
Sum the forces over the neighboring cells.
When fshift!=NULL the shift forces are updated to obtain the correct virial from the single sum including f.
void dd_partition_system | ( | FILE * | fplog, |
gmx_int64_t | step, | ||
t_commrec * | cr, | ||
gmx_bool | bMasterState, | ||
int | nstglobalcomm, | ||
t_state * | state_global, | ||
gmx_mtop_t * | top_global, | ||
t_inputrec * | ir, | ||
t_state * | state_local, | ||
rvec ** | f, | ||
t_mdatoms * | mdatoms, | ||
gmx_localtop_t * | top_local, | ||
t_forcerec * | fr, | ||
gmx_vsite_t * | vsite, | ||
gmx_shellfc_t | shellfc, | ||
gmx_constr_t | constr, | ||
t_nrnb * | nrnb, | ||
gmx_wallcycle_t | wcycle, | ||
gmx_bool | bVerbose | ||
) |
Partition the system over the nodes.
step is only used for printing error messages. If bMasterState==TRUE then state_global from the master node is used, else state_local is redistributed between the nodes. When f!=NULL, *f will be reallocated to the size of state_local.
float dd_pme_f_ratio | ( | gmx_domdec_t * | dd | ) |
Return the PME/PP force load ratio, or -1 if nothing was measured.
Should only be called on the DD master node.
void dd_setup_dlb_resource_sharing | ( | t_commrec * | cr, |
const gmx_hw_info_t * | hwinfo, | ||
const gmx_hw_opt_t * | hw_opt | ||
) |
Set up communication for averaging GPU wait times over ranks.
When domains (PP MPI ranks) share a GPU, the individual GPU wait times are meaningless, as it depends on the order in which tasks on the same GPU finish. Therefore there wait times need to be averaged over the ranks sharing the same GPU. This function sets up the communication for that.
void dd_store_state | ( | gmx_domdec_t * | dd, |
t_state * | state | ||
) |
Store the global cg indices of the home cgs in state,.
This means it can be reset, even after a new DD partitioning.
int ddglatnr | ( | gmx_domdec_t * | dd, |
int | i | ||
) |
Returns the global topology atom number belonging to local atom index i.
This function is intended for writing ASCII output and returns atom numbers starting at 1. When dd=NULL returns i+1.
void set_dd_dlb_max_cutoff | ( | t_commrec * | cr, |
real | cutoff | ||
) |
Limit DLB to preserve the option of returning to the current cut-off.
Domain boundary changes due to the DD dynamic load balancing can limit the cut-off distance that can be set in change_dd_cutoff. This function sets/changes the DLB limit such that using the passed (pair-list) cut-off should still be possible after subsequently setting a shorter cut-off with change_dd_cutoff.
void set_dd_parameters | ( | FILE * | fplog, |
gmx_domdec_t * | dd, | ||
real | dlb_scale, | ||
t_inputrec * | ir, | ||
gmx_ddbox_t * | ddbox | ||
) |
Set DD grid dimensions and limits.
Should be called after calling dd_init_bondeds.
void write_dd_pdb | ( | const char * | fn, |
gmx_int64_t | step, | ||
const char * | title, | ||
gmx_mtop_t * | mtop, | ||
t_commrec * | cr, | ||
int | natoms, | ||
rvec | x[], | ||
matrix | box | ||
) |
Dump a pdb file with the current DD home + communicated atoms.
When natoms=-1, dump all known atoms.