1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
|
#include "mpi.h"
#include <parmetisbin.h>
#include "fem.h"
static MPI_Request *mpi_out_requests = NULL;
static MPI_Request *mpi_in_requests = NULL;
static int Nmess = 0;
void MaxwellsMPISend3d(Mesh *mesh){
int p;
int procid = mesh->procid;
int nprocs = mesh->nprocs;
MPI_Status status;
if(mpi_out_requests==NULL){
mpi_out_requests = (MPI_Request*) calloc(nprocs, sizeof(MPI_Request));
mpi_in_requests = (MPI_Request*) calloc(nprocs, sizeof(MPI_Request));
}
#ifdef CUDA
get_partial_gpu_data3d(mesh->parNtotalout, mesh->c_parmapOUT, mesh->f_outQ);
#endif
/* non-blocked send/recv partition surface data */
Nmess = 0;
/* now send piece to each proc */
int sk = 0;
for(p=0;p<nprocs;++p){
if(p!=procid){
int Nout = mesh->Npar[p]*p_Nfields*p_Nfp;
if(Nout){
/* symmetric communications (different ordering) */
MPI_Isend(mesh->f_outQ+sk, Nout, MPI_FLOAT, p, 6666+p, MPI_COMM_WORLD, mpi_out_requests +Nmess);
MPI_Irecv(mesh->f_inQ+sk, Nout, MPI_FLOAT, p, 6666+procid, MPI_COMM_WORLD, mpi_in_requests +Nmess);
sk+=Nout;
++Nmess;
}
}
}
}
void MaxwellsMPIRecv3d(Mesh *mesh, float *c_partQ){
int p, n;
int nprocs = mesh->nprocs;
MPI_Status *instatus = (MPI_Status*) calloc(nprocs, sizeof(MPI_Status));
MPI_Status *outstatus = (MPI_Status*) calloc(nprocs, sizeof(MPI_Status));
MPI_Waitall(Nmess, mpi_in_requests, instatus);
#ifdef CUDA
cudaMemcpy(c_partQ, mesh->f_inQ, mesh->parNtotalout*sizeof(float), cudaMemcpyHostToDevice);
#endif
MPI_Waitall(Nmess, mpi_out_requests, outstatus);
free(outstatus);
free(instatus);
}
|
