creating branch for adding support for CUDA 3.x and Fermi

[git-p4: depot-paths = "//depot/gpgpu_sim_research/fermi/distribution/": change = 6829]

59 files changed, 17162 insertions, 0 deletions

diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/Makefile b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/Makefile
new file mode 100644
index 0000000..3281a07
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/Makefile
@@ -0,0 +1,48 @@
+include ../Makefile.in
+
+
+CFLAGS = $(COPTIONS) $(OPTFLAGS) -I. $(INCDIR)
+
+
+OBJS = comm.o util.o debug.o setup.o grsetup.o timer.o \
+       node_refine.o initmsection.o order.o \
+       xyzpart.o pspases.o frename.o \
+       iintsort.o iidxsort.o ikeysort.o ikeyvalsort.o \
+       kmetis.o gkmetis.o ometis.o \
+       initpart.o match.o coarsen.o \
+       kwayfm.o kwayrefine.o kwaybalance.o \
+       remap.o stat.o fpqueue.o \
+       ametis.o rmetis.o lmatch.o initbalance.o \
+       mdiffusion.o diffutil.o wave.o \
+       csrmatch.o redomylink.o balancemylink.o \
+       selectq.o akwayfm.o serial.o move.o \
+       mmetis.o mesh.o memory.o weird.o backcompat.o
+
+.c.o:
+	$(CC) $(CFLAGS) -c $*.c
+
+
+../libparmetis.a: $(OBJS)
+	$(AR) $@ $(OBJS)
+	$(RANLIB) $@
+
+clean:
+	rm -f *.o 
+
+realclean:
+	rm -f *.o ; rm -f ../libparmetis.a
+
+
+checkin:
+	@for file in *.[c,h]; \
+	do \
+	ci -u -m'Maintance' $$file;\
+	done 
+
+checkin2:
+	@for file in *.[c,h]; \
+	do \
+	ci $$file;\
+	rcs -U $$file;\
+	co $$file;\
+	done
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/adrivers.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/adrivers.c
new file mode 100644
index 0000000..e9c5fe0
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/adrivers.c
@@ -0,0 +1,117 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * adrivers.c
+ *
+ * This file contains the driving routines for the various parallel
+ * multilevel partitioning and repartitioning algorithms
+ *
+ * Started 11/19/96
+ * George
+ *
+ * $Id: adrivers.c,v 1.5 2003/07/30 18:37:58 karypis Exp $
+ *
+ */
+
+#include <parmetislib.h>
+
+
+
+/*************************************************************************
+* This function is the driver for the adaptive refinement mode of ParMETIS
+**************************************************************************/
+void Adaptive_Partition(CtrlType *ctrl, GraphType *graph, WorkSpaceType *wspace)
+{
+  int i;
+  int tewgt, tvsize;
+  float gtewgt, gtvsize;
+  float ubavg, lbavg, lbvec[MAXNCON];
+
+  /************************************/
+  /* Set up important data structures */
+  /************************************/
+  SetUp(ctrl, graph, wspace);
+
+  ubavg   = savg(graph->ncon, ctrl->ubvec);
+  tewgt   = idxsum(graph->nedges, graph->adjwgt);
+  tvsize  = idxsum(graph->nvtxs, graph->vsize);
+  gtewgt  = (float) GlobalSESum(ctrl, tewgt) + 1.0;  /* The +1 were added to remove any FPE */
+  gtvsize = (float) GlobalSESum(ctrl, tvsize) + 1.0;
+  ctrl->redist_factor = ctrl->redist_base * ((gtewgt/gtvsize)/ ctrl->edge_size_ratio);
+
+  IFSET(ctrl->dbglvl, DBG_PROGRESS, rprintf(ctrl, "[%6d %8d %5d %5d][%d]\n", 
+        graph->gnvtxs, GlobalSESum(ctrl, graph->nedges), GlobalSEMin(ctrl, graph->nvtxs), GlobalSEMax(ctrl, graph->nvtxs), ctrl->CoarsenTo));
+
+  if (graph->gnvtxs < 1.3*ctrl->CoarsenTo ||
+     (graph->finer != NULL && graph->gnvtxs > graph->finer->gnvtxs*COARSEN_FRACTION)) {
+
+    /***********************************************/
+    /* Balance the partition on the coarsest graph */
+    /***********************************************/
+    graph->where = idxsmalloc(graph->nvtxs+graph->nrecv, -1, "graph->where");
+    idxcopy(graph->nvtxs, graph->home, graph->where);
+
+    Moc_ComputeParallelBalance(ctrl, graph, graph->where, lbvec);
+    lbavg = savg(graph->ncon, lbvec);
+
+    if (lbavg > ubavg + 0.035 && ctrl->partType != REFINE_PARTITION)
+      Balance_Partition(ctrl, graph, wspace);
+
+    if (ctrl->dbglvl&DBG_PROGRESS) {
+      Moc_ComputeParallelBalance(ctrl, graph, graph->where, lbvec);
+      rprintf(ctrl, "nvtxs: %10d, balance: ", graph->gnvtxs);
+      for (i=0; i<graph->ncon; i++) 
+        rprintf(ctrl, "%.3f ", lbvec[i]);
+      rprintf(ctrl, "\n");
+    }
+
+    /* check if no coarsening took place */
+    if (graph->finer == NULL) {
+      Moc_ComputePartitionParams(ctrl, graph, wspace);
+      Moc_KWayBalance(ctrl, graph, wspace, graph->ncon);
+      Moc_KWayAdaptiveRefine(ctrl, graph, wspace, NGR_PASSES);
+    }
+  }
+  else {
+    /*******************************/
+    /* Coarsen it and partition it */
+    /*******************************/
+    switch (ctrl->ps_relation) {
+      case COUPLED:
+        Mc_LocalMatch_HEM(ctrl, graph, wspace);
+        break;
+      case DISCOUPLED:
+      default:
+        Moc_GlobalMatch_Balance(ctrl, graph, wspace);
+        break;
+    }
+
+    Adaptive_Partition(ctrl, graph->coarser, wspace);
+
+    /********************************/
+    /* project partition and refine */
+    /********************************/
+    Moc_ProjectPartition(ctrl, graph, wspace);
+    Moc_ComputePartitionParams(ctrl, graph, wspace);
+
+    if (graph->ncon > 1 && graph->level < 4) {
+      Moc_ComputeParallelBalance(ctrl, graph, graph->where, lbvec);
+      lbavg = savg(graph->ncon, lbvec);
+
+      if (lbavg > ubavg + 0.025) {
+        Moc_KWayBalance(ctrl, graph, wspace, graph->ncon);
+      }
+    }
+
+    Moc_KWayAdaptiveRefine(ctrl, graph, wspace, NGR_PASSES);
+
+    if (ctrl->dbglvl&DBG_PROGRESS) {
+      Moc_ComputeParallelBalance(ctrl, graph, graph->where, lbvec);
+      rprintf(ctrl, "nvtxs: %10d, cut: %8d, balance: ", graph->gnvtxs, graph->mincut);
+      for (i=0; i<graph->ncon; i++) 
+        rprintf(ctrl, "%.3f ", lbvec[i]);
+      rprintf(ctrl, "\n");
+    }
+  }
+}
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/akwayfm.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/akwayfm.c
new file mode 100644
index 0000000..7579d8b
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/akwayfm.c
@@ -0,0 +1,629 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * makwayfm.c
+ *
+ * This file contains code that performs the k-way refinement
+ *
+ * Started 3/1/96
+ * George
+ *
+ * $Id: akwayfm.c,v 1.3 2003/07/22 22:58:18 karypis Exp $
+ */
+
+#include <parmetislib.h>
+
+#define ProperSide(c, from, other) \
+              (((c) == 0 && (from)-(other) < 0) || ((c) == 1 && (from)-(other) > 0))
+
+
+/*************************************************************************
+* This function performs k-way refinement
+**************************************************************************/
+void Moc_KWayAdaptiveRefine(CtrlType *ctrl, GraphType *graph, WorkSpaceType *wspace, int npasses)
+{
+  int h, i, ii, iii, j, k, c;
+  int pass, nvtxs, nedges, ncon;
+  int nmoves, nmoved;
+  int me, firstvtx, lastvtx, yourlastvtx;
+  int from, to = -1, oldto, oldcut, mydomain, yourdomain, imbalanced, overweight;
+  int npes = ctrl->npes, mype = ctrl->mype, nparts = ctrl->nparts;
+  int nlupd, nsupd, nnbrs, nchanged;
+  idxtype *xadj, *ladjncy, *adjwgt, *vtxdist;
+  idxtype *where, *tmp_where, *moved;
+  float *lnpwgts, *gnpwgts, *ognpwgts, *pgnpwgts, *movewgts, *overfill;
+  idxtype *update, *supdate, *rupdate, *pe_updates;
+  idxtype *changed, *perm, *pperm, *htable;
+  idxtype *peind, *recvptr, *sendptr;
+  KeyValueType *swchanges, *rwchanges;
+  RInfoType *rinfo, *myrinfo, *tmp_myrinfo, *tmp_rinfo;
+  EdgeType *tmp_edegrees, *my_edegrees, *your_edegrees;
+  float lbvec[MAXNCON], *nvwgt, *badmaxpwgt, *ubvec, *tpwgts, lbavg, ubavg;
+  float oldgain, gain;
+  float ipc_factor, redist_factor, vsize;
+  int *nupds_pe, ndirty, nclean, dptr;
+  int better, worse;
+
+  IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->KWayTmr));
+
+  /*************************/
+  /* set up common aliases */
+  /*************************/
+  nvtxs = graph->nvtxs;
+  nedges = graph->nedges;
+  ncon = graph->ncon;
+
+  vtxdist = graph->vtxdist;
+  xadj = graph->xadj;
+  ladjncy = graph->adjncy;
+  adjwgt = graph->adjwgt;
+
+  firstvtx = vtxdist[mype];
+  lastvtx = vtxdist[mype+1];
+
+  where = graph->where;
+  rinfo = graph->rinfo;
+  lnpwgts = graph->lnpwgts;
+  gnpwgts = graph->gnpwgts;
+  ubvec = ctrl->ubvec;
+  tpwgts = ctrl->tpwgts;
+  ipc_factor = ctrl->ipc_factor;
+  redist_factor = ctrl->redist_factor;
+
+  nnbrs = graph->nnbrs;
+  peind = graph->peind;
+  recvptr = graph->recvptr;
+  sendptr = graph->sendptr;
+
+  changed = idxmalloc(nvtxs, "AKWR: changed");
+  rwchanges = wspace->pairs;
+  swchanges = rwchanges + recvptr[nnbrs];
+
+  /************************************/
+  /* set up important data structures */
+  /************************************/
+  perm = idxmalloc(nvtxs, "AKWR: perm");
+  pperm = idxmalloc(nparts, "AKWR: pperm");
+
+  update = idxmalloc(nvtxs, "AKWR: update");
+  supdate = wspace->indices;
+  rupdate = supdate + recvptr[nnbrs];
+  nupds_pe = imalloc(npes, "AKWR: nupds_pe");
+  htable = idxsmalloc(nvtxs+graph->nrecv, 0, "AKWR: lhtable");
+  badmaxpwgt = fmalloc(nparts*ncon, "badmaxpwgt");
+
+  for (i=0; i<nparts; i++) {
+    for (h=0; h<ncon; h++) {
+      badmaxpwgt[i*ncon+h] = ubvec[h]*tpwgts[i*ncon+h];
+    }
+  }
+
+  movewgts = fmalloc(ncon*nparts, "AKWR: movewgts");
+  ognpwgts = fmalloc(nparts*ncon, "AKWR: ognpwgts");
+  pgnpwgts = fmalloc(nparts*ncon, "AKWR: pgnpwgts");
+  overfill = fmalloc(nparts*ncon, "AKWR: overfill");
+  moved = idxmalloc(nvtxs, "AKWR: moved");
+  tmp_where = idxmalloc(nvtxs+graph->nrecv, "AKWR: tmp_where");
+  tmp_rinfo = (RInfoType *)GKmalloc(sizeof(RInfoType)*nvtxs, "AKWR: tmp_rinfo");
+  tmp_edegrees = (EdgeType *)GKmalloc(sizeof(EdgeType)*nedges, "AKWR: tmp_edegrees");
+
+  idxcopy(nvtxs+graph->nrecv, where, tmp_where);
+  for (i=0; i<nvtxs; i++) {
+    tmp_rinfo[i].id = rinfo[i].id;
+    tmp_rinfo[i].ed = rinfo[i].ed;
+    tmp_rinfo[i].ndegrees = rinfo[i].ndegrees;
+    tmp_rinfo[i].degrees = tmp_edegrees+xadj[i];
+
+    for (j=0; j<rinfo[i].ndegrees; j++) {
+      tmp_rinfo[i].degrees[j].edge = rinfo[i].degrees[j].edge;
+      tmp_rinfo[i].degrees[j].ewgt = rinfo[i].degrees[j].ewgt;
+    }
+  }
+
+  /*********************************************************/
+  /* perform a small number of passes through the vertices */
+  /*********************************************************/
+  for (pass=0; pass<npasses; pass++) {
+    oldcut = graph->mincut;
+    if (mype == 0)
+      RandomPermute(nparts, pperm, 1);
+    MPI_Bcast((void *)pperm, nparts, IDX_DATATYPE, 0, ctrl->comm);
+/*    FastRandomPermute(nvtxs, perm, 1); */
+
+    /*****************************/
+    /* move dirty vertices first */
+    /*****************************/
+    ndirty = 0;
+    for (i=0; i<nvtxs; i++)
+      if (where[i] != mype)
+        ndirty++;
+
+    dptr = 0;
+    for (i=0; i<nvtxs; i++)
+      if (where[i] != mype)
+        perm[dptr++] = i;
+      else
+        perm[ndirty++] = i;
+
+    ASSERT(ctrl, ndirty == nvtxs);
+    ndirty = dptr;
+    nclean = nvtxs-dptr;
+    FastRandomPermute(ndirty, perm, 0);
+    FastRandomPermute(nclean, perm+ndirty, 0);
+
+    /* check to see if the partitioning is imbalanced */
+    Moc_ComputeParallelBalance(ctrl, graph, graph->where, lbvec);
+    ubavg = savg(ncon, ubvec);
+    lbavg = savg(ncon, lbvec);
+    imbalanced = (lbavg > ubavg) ? 1 : 0;
+
+    for (c=0; c<2; c++) {
+      scopy(ncon*nparts, gnpwgts, ognpwgts);
+      sset(ncon*nparts, 0.0, movewgts);
+      nmoved = 0;
+
+      /**********************************************/
+      /* PASS ONE -- record stats for desired moves */
+      /**********************************************/
+      for (iii=0; iii<nvtxs; iii++) {
+        i     = perm[iii];
+        from  = tmp_where[i];
+        nvwgt = graph->nvwgt+i*ncon;
+        vsize = (float)(graph->vsize[i]);
+
+        for (h=0; h<ncon; h++) {
+          if (fabs(nvwgt[h]-gnpwgts[from*ncon+h]) < SMALLFLOAT)
+            break;
+        }
+        if (h < ncon) 
+          continue;
+
+        /* only check border vertices */
+        if (tmp_rinfo[i].ed <= 0)
+	  continue;
+
+        my_edegrees = tmp_rinfo[i].degrees;
+
+        for (k=0; k<tmp_rinfo[i].ndegrees; k++) {
+          to = my_edegrees[k].edge;
+          if (ProperSide(c, pperm[from], pperm[to])) {
+            for (h=0; h<ncon; h++) {
+              if (gnpwgts[to*ncon+h]+nvwgt[h] > badmaxpwgt[to*ncon+h] && nvwgt[h] > 0.0)
+                break;
+            }
+            if (h == ncon)
+              break;
+          }
+        }
+        oldto = to;
+
+        /* check if a subdomain was found that fits */
+        if (k < tmp_rinfo[i].ndegrees) {
+          /**************************/
+          /**************************/
+          switch (ctrl->ps_relation) {
+            case COUPLED:
+              better = (oldto == mype) ? 1 : 0;
+              worse = (from == mype) ? 1 : 0;
+              break;
+            case DISCOUPLED:
+            default:
+              better = (oldto == graph->home[i]) ? 1 : 0;
+              worse = (from == graph->home[i]) ? 1 : 0;
+              break;
+          }
+          /**************************/
+          /**************************/
+
+          oldgain = ipc_factor * (float)(my_edegrees[k].ewgt-tmp_rinfo[i].id);
+          if (better) oldgain += redist_factor * vsize;
+          if (worse) oldgain -= redist_factor * vsize;
+
+          for (j=k+1; j<tmp_rinfo[i].ndegrees; j++) {
+            to = my_edegrees[j].edge;
+            if (ProperSide(c, pperm[from], pperm[to])) {
+              /**************************/
+              /**************************/
+              switch (ctrl->ps_relation) {
+                case COUPLED:
+                  better = (to == mype) ? 1 : 0;
+                  break;
+                case DISCOUPLED:
+                default:
+                  better = (to == graph->home[i]) ? 1 : 0;
+                  break;
+              }
+              /**************************/
+              /**************************/
+
+              gain = ipc_factor * (float)(my_edegrees[j].ewgt-tmp_rinfo[i].id);
+              if (better) gain += redist_factor * vsize;
+              if (worse) gain -= redist_factor * vsize;
+
+              for (h=0; h<ncon; h++)
+                if (gnpwgts[to*ncon+h]+nvwgt[h] > badmaxpwgt[to*ncon+h] && nvwgt[h] > 0.0)
+                  break;
+
+              if (h == ncon) {
+                if (gain > oldgain ||
+                 (fabs(gain-oldgain) < SMALLFLOAT &&
+                 IsHBalanceBetterTT(ncon,gnpwgts+oldto*ncon,gnpwgts+to*ncon,nvwgt,ubvec))){
+                  oldgain = gain;
+                  oldto = to;
+                  k = j;
+                }
+              }
+            }
+          }
+          to = oldto;
+          gain = oldgain;
+
+          if (gain > 0.0 ||  
+          (gain > -1.0*SMALLFLOAT &&
+          (imbalanced ||  graph->level > 3  || iii % 8 == 0) &&
+          IsHBalanceBetterFT(ncon,gnpwgts+from*ncon,gnpwgts+to*ncon,nvwgt,ubvec))){
+
+            /****************************************/
+            /* Update tmp arrays of the moved vertex */
+            /****************************************/
+            tmp_where[i] = to;
+            moved[nmoved++] = i;
+            for (h=0; h<ncon; h++) {
+	      INC_DEC(lnpwgts[to*ncon+h], lnpwgts[from*ncon+h], nvwgt[h]);
+	      INC_DEC(gnpwgts[to*ncon+h], gnpwgts[from*ncon+h], nvwgt[h]);
+	      INC_DEC(movewgts[to*ncon+h], movewgts[from*ncon+h], nvwgt[h]);
+            }
+
+            tmp_rinfo[i].ed += tmp_rinfo[i].id-my_edegrees[k].ewgt;
+            SWAP(tmp_rinfo[i].id, my_edegrees[k].ewgt, j);
+            if (my_edegrees[k].ewgt == 0) {
+              tmp_rinfo[i].ndegrees--;
+              my_edegrees[k].edge = my_edegrees[tmp_rinfo[i].ndegrees].edge;
+              my_edegrees[k].ewgt = my_edegrees[tmp_rinfo[i].ndegrees].ewgt;
+            }
+            else {
+              my_edegrees[k].edge = from;
+            }
+
+            /* Update the degrees of adjacent vertices */
+            for (j=xadj[i]; j<xadj[i+1]; j++) {
+              /* no need to bother about vertices on different pe's */
+              if (ladjncy[j] >= nvtxs)
+                continue;
+
+              me = ladjncy[j];
+              mydomain = tmp_where[me];
+
+              myrinfo = tmp_rinfo+me;
+              your_edegrees = myrinfo->degrees;
+
+              if (mydomain == from) {
+                INC_DEC(myrinfo->ed, myrinfo->id, adjwgt[j]);
+              }
+              else {
+                if (mydomain == to) {
+                  INC_DEC(myrinfo->id, myrinfo->ed, adjwgt[j]);
+                }
+              }
+
+              /* Remove contribution from the .ed of 'from' */
+              if (mydomain != from) {
+                for (k=0; k<myrinfo->ndegrees; k++) {
+                  if (your_edegrees[k].edge == from) {
+                    if (your_edegrees[k].ewgt == adjwgt[j]) {
+                      myrinfo->ndegrees--;
+                      your_edegrees[k].edge = your_edegrees[myrinfo->ndegrees].edge;
+                      your_edegrees[k].ewgt = your_edegrees[myrinfo->ndegrees].ewgt;
+                    }
+                    else {
+                      your_edegrees[k].ewgt -= adjwgt[j];
+                    }
+                    break;
+                  }
+                }
+              }
+
+              /* Add contribution to the .ed of 'to' */
+              if (mydomain != to) {
+                for (k=0; k<myrinfo->ndegrees; k++) {
+                  if (your_edegrees[k].edge == to) {
+                    your_edegrees[k].ewgt += adjwgt[j];
+                    break;
+                  }
+                }
+                if (k == myrinfo->ndegrees) {
+                  your_edegrees[myrinfo->ndegrees].edge = to;
+                  your_edegrees[myrinfo->ndegrees++].ewgt = adjwgt[j];
+                }
+              }
+            }
+          }
+        }
+      }
+
+      /******************************************/
+      /* Let processors know the subdomain wgts */
+      /* if all proposed moves commit.          */
+      /******************************************/
+      MPI_Allreduce((void *)lnpwgts, (void *)pgnpwgts, nparts*ncon, MPI_FLOAT, MPI_SUM, ctrl->comm);
+
+      /**************************/
+      /* compute overfill array */
+      /**************************/
+      overweight = 0;
+      for (j=0; j<nparts; j++) {
+        for (h=0; h<ncon; h++) {
+          if (pgnpwgts[j*ncon+h] > ognpwgts[j*ncon+h]) 
+            overfill[j*ncon+h] = (pgnpwgts[j*ncon+h]-badmaxpwgt[j*ncon+h]) / (pgnpwgts[j*ncon+h]-ognpwgts[j*ncon+h]);
+          else 
+            overfill[j*ncon+h] = 0.0;
+
+          overfill[j*ncon+h] = amax(overfill[j*ncon+h], 0.0);
+          overfill[j*ncon+h] *= movewgts[j*ncon+h];
+
+          if (overfill[j*ncon+h] > 0.0)
+            overweight = 1;
+
+          ASSERTP(ctrl, ognpwgts[j*ncon+h] <= badmaxpwgt[j*ncon+h] || pgnpwgts[j*ncon+h] <= ognpwgts[j*ncon+h],
+                  (ctrl, "%.4f %.4f %.4f\n", ognpwgts[j*ncon+h], badmaxpwgt[j*ncon+h], pgnpwgts[j*ncon+h]));
+        }
+      }
+
+      /****************************************************/
+      /* select moves to undo according to overfill array */
+      /****************************************************/
+      if (overweight == 1) {
+        for (iii=0; iii<nmoved; iii++) {
+          i = moved[iii];
+          oldto = tmp_where[i];
+          nvwgt = graph->nvwgt+i*ncon;
+          my_edegrees = tmp_rinfo[i].degrees;
+
+          for (k=0; k<tmp_rinfo[i].ndegrees; k++)
+            if (my_edegrees[k].edge == where[i])
+              break;
+
+          for (h=0; h<ncon; h++)
+            if (nvwgt[h] > 0.0 && overfill[oldto*ncon+h] > nvwgt[h]/4.0)
+              break;
+
+          /**********************************/
+          /* nullify this move if necessary */
+          /**********************************/
+          if (k != tmp_rinfo[i].ndegrees && h != ncon) {
+            moved[iii] = -1;
+            from = oldto;
+            to = where[i];
+
+            for (h=0; h<ncon; h++) 
+              overfill[oldto*ncon+h] = amax(overfill[oldto*ncon+h]-nvwgt[h], 0.0);
+
+            tmp_where[i] = to;
+            tmp_rinfo[i].ed += tmp_rinfo[i].id-my_edegrees[k].ewgt;
+            SWAP(tmp_rinfo[i].id, my_edegrees[k].ewgt, j);
+            if (my_edegrees[k].ewgt == 0) {
+              tmp_rinfo[i].ndegrees--;
+              my_edegrees[k].edge = my_edegrees[tmp_rinfo[i].ndegrees].edge;
+              my_edegrees[k].ewgt = my_edegrees[tmp_rinfo[i].ndegrees].ewgt;
+            }
+            else {
+              my_edegrees[k].edge = from;
+            }
+
+            for (h=0; h<ncon; h++) 
+	      INC_DEC(lnpwgts[to*ncon+h], lnpwgts[from*ncon+h], nvwgt[h]);
+
+            /* Update the degrees of adjacent vertices */
+            for (j=xadj[i]; j<xadj[i+1]; j++) {
+              /* no need to bother about vertices on different pe's */
+              if (ladjncy[j] >= nvtxs)
+                continue;
+
+              me = ladjncy[j];
+              mydomain = tmp_where[me];
+
+              myrinfo = tmp_rinfo+me;
+              your_edegrees = myrinfo->degrees;
+
+              if (mydomain == from) {
+                INC_DEC(myrinfo->ed, myrinfo->id, adjwgt[j]);
+              }
+              else {
+                if (mydomain == to) {
+                  INC_DEC(myrinfo->id, myrinfo->ed, adjwgt[j]);
+                }
+              }
+
+              /* Remove contribution from the .ed of 'from' */
+              if (mydomain != from) {
+                for (k=0; k<myrinfo->ndegrees; k++) {
+                  if (your_edegrees[k].edge == from) {
+                    if (your_edegrees[k].ewgt == adjwgt[j]) {
+                      myrinfo->ndegrees--;
+                      your_edegrees[k].edge = your_edegrees[myrinfo->ndegrees].edge;
+                      your_edegrees[k].ewgt = your_edegrees[myrinfo->ndegrees].ewgt;
+                    }
+                    else {
+                      your_edegrees[k].ewgt -= adjwgt[j];
+                    }
+                    break;
+                  }
+                }
+              }
+
+              /* Add contribution to the .ed of 'to' */
+              if (mydomain != to) {
+                for (k=0; k<myrinfo->ndegrees; k++) {
+                  if (your_edegrees[k].edge == to) {
+                    your_edegrees[k].ewgt += adjwgt[j];
+                    break;
+                  }
+                }
+                if (k == myrinfo->ndegrees) {
+                  your_edegrees[myrinfo->ndegrees].edge = to;
+                  your_edegrees[myrinfo->ndegrees++].ewgt = adjwgt[j];
+                }
+              }
+            }
+          }
+        }
+      }
+
+      /*************************************************/
+      /* PASS TWO -- commit the remainder of the moves */
+      /*************************************************/
+      nlupd = nsupd = nmoves = nchanged = 0;
+      for (iii=0; iii<nmoved; iii++) {
+        i = moved[iii];
+        if (i == -1)
+          continue;
+
+        where[i] = tmp_where[i]; 
+
+        /* Make sure to update the vertex information */
+        if (htable[i] == 0) {
+          /* make sure you do the update */
+          htable[i] = 1;
+          update[nlupd++] = i;
+        }
+
+        /* Put the vertices adjacent to i into the update array */
+        for (j=xadj[i]; j<xadj[i+1]; j++) {
+          k = ladjncy[j];
+          if (htable[k] == 0) {
+            htable[k] = 1;
+            if (k<nvtxs)
+              update[nlupd++] = k;
+            else
+              supdate[nsupd++] = k;
+          }
+        }
+        nmoves++;
+
+        if (graph->pexadj[i+1]-graph->pexadj[i] > 0)
+          changed[nchanged++] = i;
+      }
+
+      /* Tell interested pe's the new where[] info for the interface vertices */
+      CommChangedInterfaceData(ctrl, graph, nchanged, changed, where, swchanges, rwchanges, wspace->pv4); 
+
+
+      IFSET(ctrl->dbglvl, DBG_RMOVEINFO, rprintf(ctrl, "\t[%d %d], [%.4f],  [%d %d %d]\n",
+            pass, c, badmaxpwgt[0], GlobalSESum(ctrl, nmoves), GlobalSESum(ctrl, nsupd), GlobalSESum(ctrl, nlupd)));
+
+      /*-------------------------------------------------------------
+      / Time to communicate with processors to send the vertices
+      / whose degrees need to be update.
+      /-------------------------------------------------------------*/
+      /* Issue the receives first */
+      for (i=0; i<nnbrs; i++) 
+        MPI_Irecv((void *)(rupdate+sendptr[i]), sendptr[i+1]-sendptr[i], IDX_DATATYPE, peind[i], 1, ctrl->comm, ctrl->rreq+i);
+
+      /* Issue the sends next. This needs some preporcessing */
+      for (i=0; i<nsupd; i++) {
+        htable[supdate[i]] = 0;
+        supdate[i] = graph->imap[supdate[i]];
+      }
+      iidxsort(nsupd, supdate);
+
+      for (j=i=0; i<nnbrs; i++) {
+        yourlastvtx = vtxdist[peind[i]+1];
+        for (k=j; k<nsupd && supdate[k] < yourlastvtx; k++); 
+        MPI_Isend((void *)(supdate+j), k-j, IDX_DATATYPE, peind[i], 1, ctrl->comm, ctrl->sreq+i);
+        j = k;
+      }
+
+      /* OK, now get into the loop waiting for the send/recv operations to finish */
+      MPI_Waitall(nnbrs, ctrl->rreq, ctrl->statuses);
+      for (i=0; i<nnbrs; i++) 
+        MPI_Get_count(ctrl->statuses+i, IDX_DATATYPE, nupds_pe+i);
+      MPI_Waitall(nnbrs, ctrl->sreq, ctrl->statuses);
+
+
+      /*-------------------------------------------------------------
+      / Place the recieved to-be updated vertices into update[] 
+      /-------------------------------------------------------------*/
+      for (i=0; i<nnbrs; i++) {
+        pe_updates = rupdate+sendptr[i];
+        for (j=0; j<nupds_pe[i]; j++) {
+          k = pe_updates[j];
+          if (htable[k-firstvtx] == 0) {
+            htable[k-firstvtx] = 1;
+            update[nlupd++] = k-firstvtx;
+          }
+        }
+      }
+
+
+      /*-------------------------------------------------------------
+      / Update the rinfo of the vertices in the update[] array
+      /-------------------------------------------------------------*/
+      for (ii=0; ii<nlupd; ii++) {
+        i = update[ii];
+        ASSERT(ctrl, htable[i] == 1);
+
+        htable[i] = 0;
+
+        mydomain = where[i];
+        myrinfo = rinfo+i;
+        tmp_myrinfo = tmp_rinfo+i;
+        my_edegrees = myrinfo->degrees;
+        your_edegrees = tmp_myrinfo->degrees;
+
+        graph->lmincut -= myrinfo->ed;
+        myrinfo->ndegrees = 0;
+        myrinfo->id = 0;
+        myrinfo->ed = 0;
+
+        for (j=xadj[i]; j<xadj[i+1]; j++) {
+          yourdomain = where[ladjncy[j]];
+          if (mydomain != yourdomain) {
+            myrinfo->ed += adjwgt[j];
+
+            for (k=0; k<myrinfo->ndegrees; k++) {
+              if (my_edegrees[k].edge == yourdomain) {
+                my_edegrees[k].ewgt += adjwgt[j];
+                your_edegrees[k].ewgt += adjwgt[j];
+                break;
+              }
+            }
+            if (k == myrinfo->ndegrees) {
+              my_edegrees[k].edge = yourdomain;
+              my_edegrees[k].ewgt = adjwgt[j];
+              your_edegrees[k].edge = yourdomain;
+              your_edegrees[k].ewgt = adjwgt[j];
+              myrinfo->ndegrees++;
+            }
+            ASSERT(ctrl, myrinfo->ndegrees <= xadj[i+1]-xadj[i]);
+            ASSERT(ctrl, tmp_myrinfo->ndegrees <= xadj[i+1]-xadj[i]);
+
+          }
+          else {
+            myrinfo->id += adjwgt[j];
+          }
+        }
+        graph->lmincut += myrinfo->ed;
+
+        tmp_myrinfo->id = myrinfo->id;
+        tmp_myrinfo->ed = myrinfo->ed;
+        tmp_myrinfo->ndegrees = myrinfo->ndegrees;
+      }
+
+      /* finally, sum-up the partition weights */
+      MPI_Allreduce((void *)lnpwgts, (void *)gnpwgts, nparts*ncon, MPI_FLOAT, MPI_SUM, ctrl->comm);
+    }
+    graph->mincut = GlobalSESum(ctrl, graph->lmincut)/2;
+
+    if (graph->mincut == oldcut)
+      break;
+  }
+
+  GKfree((void **)&badmaxpwgt, (void **)&update, (void **)&nupds_pe, (void **)&htable, LTERM);
+  GKfree((void **)&changed, (void **)&pperm, (void **)&perm, (void **)&moved, LTERM);
+  GKfree((void **)&pgnpwgts, (void **)&ognpwgts, (void **)&overfill, (void **)&movewgts, LTERM);
+  GKfree((void **)&tmp_where, (void **)&tmp_rinfo, (void **)&tmp_edegrees, LTERM);
+
+  IFSET(ctrl->dbglvl, DBG_TIME, stoptimer(ctrl->KWayTmr));
+}
+
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/ametis.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/ametis.c
new file mode 100644
index 0000000..6fac271
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/ametis.c
@@ -0,0 +1,272 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * ametis.c
+ *
+ * This is the entry point of parallel difussive repartitioning routines
+ *
+ * Started 10/19/96
+ * George
+ *
+ * $Id: ametis.c,v 1.6 2003/07/25 04:01:03 karypis Exp $
+ *
+ */
+
+#include <parmetislib.h>
+
+
+
+/***********************************************************************************
+* This function is the entry point of the parallel multilevel local diffusion
+* algorithm. It uses parallel undirected diffusion followed by adaptive k-way 
+* refinement. This function utilizes local coarsening.
+************************************************************************************/
+void ParMETIS_V3_AdaptiveRepart(idxtype *vtxdist, idxtype *xadj, idxtype *adjncy,
+  idxtype *vwgt, idxtype *vsize, idxtype *adjwgt, int *wgtflag, int *numflag,
+  int *ncon, int *nparts, float *tpwgts, float *ubvec, float *ipc2redist,
+  int *options, int *edgecut, idxtype *part, MPI_Comm *comm)
+{
+  int h, i;
+  int npes, mype;
+  CtrlType ctrl;
+  WorkSpaceType wspace;
+  GraphType *graph;
+  int tewgt, tvsize, nmoved, maxin, maxout, vtx_factor;
+  float gtewgt, gtvsize, avg, maximb;
+  int ps_relation, seed, dbglvl = 0;
+  int iwgtflag, inumflag, incon, inparts, ioptions[10];
+  float iipc2redist, *itpwgts, iubvec[MAXNCON];
+
+  MPI_Comm_size(*comm, &npes);
+  MPI_Comm_rank(*comm, &mype);
+
+  /********************************/
+  /* Try and take care bad inputs */
+  /********************************/
+  if (options != NULL && options[0] == 1)
+    dbglvl = options[PMV3_OPTION_DBGLVL];
+  CheckInputs(ADAPTIVE_PARTITION, npes, dbglvl, wgtflag, &iwgtflag, numflag, &inumflag,
+              ncon, &incon, nparts, &inparts, tpwgts, &itpwgts, ubvec, iubvec, 
+	      ipc2redist, &iipc2redist, options, ioptions, part, comm);
+
+  /* ADD: take care of disconnected graph */
+  /* ADD: take care of highly unbalanced vtxdist */
+  /*********************************/
+  /* Take care the nparts = 1 case */
+  /*********************************/
+  if (inparts == 1) {
+    idxset(vtxdist[mype+1]-vtxdist[mype], 0, part); 
+    *edgecut = 0;
+    return;
+  }
+
+  /**************************/
+  /* Set up data structures */
+  /**************************/
+  if (inumflag == 1) 
+    ChangeNumbering(vtxdist, xadj, adjncy, part, npes, mype, 1);
+
+  /*****************************/
+  /* Set up control structures */
+  /*****************************/
+  if (ioptions[0] == 1) {
+    dbglvl      = ioptions[PMV3_OPTION_DBGLVL];
+    seed        = ioptions[PMV3_OPTION_SEED];
+    ps_relation = (npes == inparts ? ioptions[PMV3_OPTION_PSR] : DISCOUPLED);
+  }
+  else {
+    dbglvl      = GLOBAL_DBGLVL;
+    seed        = GLOBAL_SEED;
+    ps_relation = (npes == inparts ? COUPLED : DISCOUPLED);
+  }
+
+  SetUpCtrl(&ctrl, inparts, dbglvl, *comm);
+  vtx_factor         = (amax(npes, inparts) > 256) ? 20 : 50;
+  ctrl.CoarsenTo     = amin(vtxdist[npes]+1, vtx_factor*incon*amax(npes, inparts));
+  ctrl.ipc_factor    = iipc2redist;
+  ctrl.redist_factor = 1.0;
+  ctrl.redist_base   = 1.0;
+  ctrl.seed          = (seed == 0 ? mype : seed*mype);
+  ctrl.sync          = GlobalSEMax(&ctrl, seed);
+  ctrl.partType      = ADAPTIVE_PARTITION;
+  ctrl.ps_relation   = ps_relation;
+  ctrl.tpwgts        = itpwgts;
+
+  graph = Moc_SetUpGraph(&ctrl, incon, vtxdist, xadj, vwgt, adjncy, adjwgt, &iwgtflag);
+  graph->vsize = (vsize == NULL ? idxsmalloc(graph->nvtxs, 1, "vsize") : vsize);
+
+  graph->home = idxmalloc(graph->nvtxs, "home");
+  if (ctrl.ps_relation == COUPLED)
+    idxset(graph->nvtxs, mype, graph->home);
+  else {
+    /* Downgrade the partition numbers if part[] has more partitions that nparts */
+    for (i=0; i<graph->nvtxs; i++)
+      part[i] = (part[i] >= ctrl.nparts ? 0 : part[i]);
+
+    idxcopy(graph->nvtxs, part, graph->home);
+  }
+
+  tewgt   = idxsum(graph->nedges, graph->adjwgt);
+  tvsize  = idxsum(graph->nvtxs, graph->vsize);
+  gtewgt  = (float) GlobalSESum(&ctrl, tewgt) + 1.0/graph->gnvtxs;  /* The +1/graph->gnvtxs were added to remove any FPE */
+  gtvsize = (float) GlobalSESum(&ctrl, tvsize) + 1.0/graph->gnvtxs;
+  ctrl.edge_size_ratio = gtewgt/gtvsize;
+  scopy(incon, iubvec, ctrl.ubvec);
+
+  PreAllocateMemory(&ctrl, graph, &wspace);
+
+  /***********************/
+  /* Partition and Remap */
+  /***********************/
+  IFSET(ctrl.dbglvl, DBG_TIME, InitTimers(&ctrl));
+  IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
+  IFSET(ctrl.dbglvl, DBG_TIME, starttimer(ctrl.TotalTmr));
+
+  Adaptive_Partition(&ctrl, graph, &wspace);
+  ParallelReMapGraph(&ctrl, graph, &wspace);
+
+  IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
+  IFSET(ctrl.dbglvl, DBG_TIME, stoptimer(ctrl.TotalTmr));
+
+  idxcopy(graph->nvtxs, graph->where, part);
+  if (edgecut != NULL)
+    *edgecut = graph->mincut;
+
+  /***********************/
+  /* Take care of output */
+  /***********************/
+  IFSET(ctrl.dbglvl, DBG_TIME, PrintTimingInfo(&ctrl));
+  IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
+
+  if (ctrl.dbglvl&DBG_INFO) {
+    Mc_ComputeMoveStatistics(&ctrl, graph, &nmoved, &maxin, &maxout);
+    rprintf(&ctrl, "Final %3d-way Cut: %6d \tBalance: ", inparts, graph->mincut);
+    avg = 0.0;
+    for (h=0; h<incon; h++) {
+      maximb = 0.0;
+      for (i=0; i<inparts; i++)
+        maximb = amax(maximb, graph->gnpwgts[i*incon+h]/itpwgts[i*incon+h]);
+      avg += maximb;
+      rprintf(&ctrl, "%.3f ", maximb);
+    }
+    rprintf(&ctrl, "\nNMoved: %d %d %d %d\n", nmoved, maxin, maxout, maxin+maxout);
+  }
+
+  /*************************************/
+  /* Free memory, renumber, and return */
+  /*************************************/
+  GKfree((void **)&graph->lnpwgts, (void **)&graph->gnpwgts, (void **)&graph->nvwgt, (void **)(&graph->home), LTERM);
+  if (vsize == NULL)
+    GKfree((void **)(&graph->vsize), LTERM);
+  GKfree((void **)&itpwgts, LTERM);
+  FreeInitialGraphAndRemap(graph, iwgtflag);
+  FreeWSpace(&wspace);
+  FreeCtrl(&ctrl);
+
+  if (inumflag == 1)
+    ChangeNumbering(vtxdist, xadj, adjncy, part, npes, mype, 0);
+
+  return;
+}
+
+
+
+
+/*************************************************************************
+* This function is the driver for the adaptive refinement mode of ParMETIS
+**************************************************************************/
+void Adaptive_Partition(CtrlType *ctrl, GraphType *graph, WorkSpaceType *wspace)
+{
+  int i;
+  int tewgt, tvsize;
+  float gtewgt, gtvsize;
+  float ubavg, lbavg, lbvec[MAXNCON];
+
+  /************************************/
+  /* Set up important data structures */
+  /************************************/
+  SetUp(ctrl, graph, wspace);
+
+  ubavg   = savg(graph->ncon, ctrl->ubvec);
+  tewgt   = idxsum(graph->nedges, graph->adjwgt);
+  tvsize  = idxsum(graph->nvtxs, graph->vsize);
+  gtewgt  = (float) GlobalSESum(ctrl, tewgt) + 1.0/graph->gnvtxs;  /* The +1/graph->gnvtxs were added to remove any FPE */
+  gtvsize = (float) GlobalSESum(ctrl, tvsize) + 1.0/graph->gnvtxs;
+  ctrl->redist_factor = ctrl->redist_base * ((gtewgt/gtvsize)/ ctrl->edge_size_ratio);
+
+  IFSET(ctrl->dbglvl, DBG_PROGRESS, rprintf(ctrl, "[%6d %8d %5d %5d][%d]\n", 
+        graph->gnvtxs, GlobalSESum(ctrl, graph->nedges), GlobalSEMin(ctrl, graph->nvtxs), GlobalSEMax(ctrl, graph->nvtxs), ctrl->CoarsenTo));
+
+  if (graph->gnvtxs < 1.3*ctrl->CoarsenTo ||
+     (graph->finer != NULL && graph->gnvtxs > graph->finer->gnvtxs*COARSEN_FRACTION)) {
+
+    /***********************************************/
+    /* Balance the partition on the coarsest graph */
+    /***********************************************/
+    graph->where = idxsmalloc(graph->nvtxs+graph->nrecv, -1, "graph->where");
+    idxcopy(graph->nvtxs, graph->home, graph->where);
+
+    Moc_ComputeParallelBalance(ctrl, graph, graph->where, lbvec);
+    lbavg = savg(graph->ncon, lbvec);
+
+    if (lbavg > ubavg + 0.035 && ctrl->partType != REFINE_PARTITION)
+      Balance_Partition(ctrl, graph, wspace);
+
+    if (ctrl->dbglvl&DBG_PROGRESS) {
+      Moc_ComputeParallelBalance(ctrl, graph, graph->where, lbvec);
+      rprintf(ctrl, "nvtxs: %10d, balance: ", graph->gnvtxs);
+      for (i=0; i<graph->ncon; i++) 
+        rprintf(ctrl, "%.3f ", lbvec[i]);
+      rprintf(ctrl, "\n");
+    }
+
+    /* check if no coarsening took place */
+    if (graph->finer == NULL) {
+      Moc_ComputePartitionParams(ctrl, graph, wspace);
+      Moc_KWayBalance(ctrl, graph, wspace, graph->ncon);
+      Moc_KWayAdaptiveRefine(ctrl, graph, wspace, NGR_PASSES);
+    }
+  }
+  else {
+    /*******************************/
+    /* Coarsen it and partition it */
+    /*******************************/
+    switch (ctrl->ps_relation) {
+      case COUPLED:
+        Mc_LocalMatch_HEM(ctrl, graph, wspace);
+        break;
+      case DISCOUPLED:
+      default:
+        Moc_GlobalMatch_Balance(ctrl, graph, wspace);
+        break;
+    }
+
+    Adaptive_Partition(ctrl, graph->coarser, wspace);
+
+    /********************************/
+    /* project partition and refine */
+    /********************************/
+    Moc_ProjectPartition(ctrl, graph, wspace);
+    Moc_ComputePartitionParams(ctrl, graph, wspace);
+
+    if (graph->ncon > 1 && graph->level < 4) {
+      Moc_ComputeParallelBalance(ctrl, graph, graph->where, lbvec);
+      lbavg = savg(graph->ncon, lbvec);
+
+      if (lbavg > ubavg + 0.025) {
+        Moc_KWayBalance(ctrl, graph, wspace, graph->ncon);
+      }
+    }
+
+    Moc_KWayAdaptiveRefine(ctrl, graph, wspace, NGR_PASSES);
+
+    if (ctrl->dbglvl&DBG_PROGRESS) {
+      Moc_ComputeParallelBalance(ctrl, graph, graph->where, lbvec);
+      rprintf(ctrl, "nvtxs: %10d, cut: %8d, balance: ", graph->gnvtxs, graph->mincut);
+      for (i=0; i<graph->ncon; i++) 
+        rprintf(ctrl, "%.3f ", lbvec[i]);
+      rprintf(ctrl, "\n");
+    }
+  }
+}
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/backcompat.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/backcompat.c
new file mode 100644
index 0000000..b62d4bb
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/backcompat.c
@@ -0,0 +1,517 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * backcompat.c
+ *
+ * This file ensures backwards compatability with previous ParMETIS releases
+ *
+ * Started 10/19/96
+ * George
+ *
+ * $Id: backcompat.c,v 1.2 2003/07/21 17:18:48 karypis Exp $
+ *
+ */
+
+#include <parmetislib.h>
+
+/*****************************************************************************
+*  This function computes a partitioning.
+*****************************************************************************/
+void ParMETIS_PartKway(idxtype *vtxdist, idxtype *xadj, idxtype *adjncy, idxtype *vwgt,
+       idxtype *adjwgt, int *wgtflag, int *numflag, int *nparts, int *options, int *edgecut,
+       idxtype *part, MPI_Comm *comm)
+{
+  int i;
+  int ncon = 1;
+  float *tpwgts, ubvec[MAXNCON];
+  int myoptions[10];
+
+  tpwgts = fmalloc(*nparts*ncon, "tpwgts");
+  for (i=0; i<*nparts*ncon; i++)
+    tpwgts[i] = 1.0/(float)(*nparts);
+  for (i=0; i<ncon; i++)
+    ubvec[i] = UNBALANCE_FRACTION;
+
+  if (options[0] == 0) {
+    myoptions[0] = 0;
+  }
+  else {
+    myoptions[0] = 1;
+    myoptions[PMV3_OPTION_DBGLVL] = options[OPTION_DBGLVL];
+    myoptions[PMV3_OPTION_SEED] = GLOBAL_SEED;
+  }
+
+  ParMETIS_V3_PartKway(vtxdist, xadj, adjncy, vwgt, adjwgt, wgtflag, numflag,
+  &ncon, nparts, tpwgts, ubvec, myoptions, edgecut, part, comm);
+
+  free(tpwgts);
+}
+
+
+
+/***********************************************************************************
+ * * This function is the entry point of the parallel k-way multilevel partitionioner.
+ * * This function assumes nothing about the graph distribution.
+ * * It is the general case.
+ * ************************************************************************************/
+void PARKMETIS(idxtype *vtxdist, idxtype *xadj, idxtype *vwgt, idxtype *adjncy, idxtype *adjwgt,
+		               idxtype *part, int *options, MPI_Comm comm)
+{
+  int wgtflag, numflag, edgecut, newoptions[5];
+  int npes;
+
+  MPI_Comm_size(comm, &npes);
+
+  newoptions[0] = 1;
+  newoptions[OPTION_IPART] = options[2];
+  newoptions[OPTION_FOLDF] = options[1];
+  newoptions[OPTION_DBGLVL] = options[4];
+
+		        numflag = options[3];
+  wgtflag = (vwgt == NULL ? 0 : 2) + (adjwgt == NULL ? 0 : 1);
+
+  ParMETIS_PartKway(vtxdist, xadj, adjncy, vwgt, adjwgt, &wgtflag, &numflag, &npes,
+	newoptions, &edgecut, part, &comm);
+
+  options[0] = edgecut;
+
+}
+
+
+
+/*****************************************************************************
+*  This function computes a partitioning using coordinate data.
+*****************************************************************************/
+void ParMETIS_PartGeomKway(idxtype *vtxdist, idxtype *xadj, idxtype *adjncy, idxtype *vwgt,
+       idxtype *adjwgt, int *wgtflag, int *numflag, int *ndims, float *xyz, int *nparts,
+       int *options, int *edgecut, idxtype *part, MPI_Comm *comm)
+{
+  int i;
+  int ncon = 1;
+  float *tpwgts, ubvec[MAXNCON];
+  int myoptions[10];
+
+  tpwgts = fmalloc(*nparts*ncon, "tpwgts");
+  for (i=0; i<*nparts*ncon; i++)
+    tpwgts[i] = 1.0/(float)(*nparts);
+  for (i=0; i<ncon; i++)
+    ubvec[i] = UNBALANCE_FRACTION;
+
+  if (options[0] == 0) {
+    myoptions[0] = 0;
+  }
+  else {
+    myoptions[0] = 1;
+    myoptions[PMV3_OPTION_DBGLVL] = options[OPTION_DBGLVL];
+    myoptions[PMV3_OPTION_SEED] = GLOBAL_SEED;
+  }
+
+  ParMETIS_V3_PartGeomKway(vtxdist, xadj, adjncy, vwgt, adjwgt, wgtflag, numflag, ndims, xyz,
+  &ncon, nparts, tpwgts, ubvec, myoptions, edgecut, part, comm);
+
+  free(tpwgts);
+  return;
+}
+
+
+/***********************************************************************************
+* This function is the entry point of the parallel ordering algorithm.
+* This function assumes that the graph is already nice partitioned among the
+* processors and then proceeds to perform recursive bisection.
+************************************************************************************/
+void ParMETIS_PartGeom(idxtype *vtxdist, int *ndims, float *xyz, idxtype *part, MPI_Comm *comm)
+{
+  ParMETIS_V3_PartGeom(vtxdist, ndims, xyz, part, comm);
+}
+
+
+/*****************************************************************************
+*  This function computes a partitioning using coordinate data.
+*****************************************************************************/
+void ParMETIS_PartGeomRefine(idxtype *vtxdist, idxtype *xadj, idxtype *adjncy,
+  idxtype *vwgt, idxtype *adjwgt, int *wgtflag, int *numflag, int *ndims,
+  float *xyz, int *options, int *edgecut, idxtype *part, MPI_Comm *comm)
+{
+  int i;
+  int npes, nparts, ncon = 1;
+  float *tpwgts, ubvec[MAXNCON];
+  int myoptions[10];
+
+  MPI_Comm_size(*comm, &npes);
+  nparts = npes;
+
+  tpwgts = fmalloc(nparts*ncon, "tpwgts");
+  for (i=0; i<nparts*ncon; i++)
+    tpwgts[i] = 1.0/(float)(nparts);
+  for (i=0; i<ncon; i++)
+    ubvec[i] = UNBALANCE_FRACTION;
+
+  if (options[0] == 0) {
+    myoptions[0] = 0;
+  }
+  else {
+    myoptions[0] = 1;
+    myoptions[PMV3_OPTION_DBGLVL] = options[OPTION_DBGLVL];
+    myoptions[PMV3_OPTION_SEED] = GLOBAL_SEED;
+  }
+
+  ParMETIS_V3_PartGeomKway(vtxdist, xadj, adjncy, vwgt, adjwgt, wgtflag, numflag, ndims, xyz,
+  &ncon, &nparts, tpwgts, ubvec, myoptions, edgecut, part, comm);
+
+  free(tpwgts);
+  return;
+}
+
+
+/***********************************************************************************
+* This function is the entry point of the parallel kmetis algorithm that uses
+* coordinates to compute an initial graph distribution.
+************************************************************************************/
+void PARGKMETIS(idxtype *vtxdist, idxtype *xadj, idxtype *vwgt, idxtype *adjncy, idxtype *adjwgt,
+                int ndims, float *xyz, idxtype *part, int *options, MPI_Comm comm)
+{
+  int npes, wgtflag, numflag, edgecut, newoptions[5];
+
+  MPI_Comm_size(comm, &npes);
+
+  newoptions[0] = 1;
+  newoptions[OPTION_IPART] = options[2];
+  newoptions[OPTION_FOLDF] = options[1];
+  newoptions[OPTION_DBGLVL] = options[4];
+
+  numflag = options[3];
+  wgtflag = (vwgt == NULL ? 0 : 2) + (adjwgt == NULL ? 0 : 1);
+
+  ParMETIS_PartGeomKway(vtxdist, xadj, adjncy, vwgt, adjwgt, &wgtflag, &numflag,
+     &ndims, xyz, &npes, newoptions, &edgecut, part, &comm);
+
+  options[0] = edgecut;
+
+}
+
+
+/***********************************************************************************
+* This function is the entry point of the parallel rmetis algorithm that uses
+* coordinates to compute an initial graph distribution.
+************************************************************************************/
+void PARGRMETIS(idxtype *vtxdist, idxtype *xadj, idxtype *vwgt, idxtype *adjncy, idxtype *adjwgt,
+                int ndims, float *xyz, idxtype *part, int *options, MPI_Comm comm)
+{
+  int wgtflag, numflag, edgecut, newoptions[5];
+
+  newoptions[0] = 1;
+  newoptions[OPTION_IPART] = options[2];
+  newoptions[OPTION_FOLDF] = options[1];
+  newoptions[OPTION_DBGLVL] = options[4];
+
+  numflag = options[3];
+  wgtflag = (vwgt == NULL ? 0 : 2) + (adjwgt == NULL ? 0 : 1);
+
+  ParMETIS_PartGeomRefine(vtxdist, xadj, adjncy, vwgt, adjwgt, &wgtflag, &numflag,
+     &ndims, xyz, newoptions, &edgecut, part, &comm);
+
+  options[0] = edgecut;
+
+}
+
+/***********************************************************************************
+* This function is the entry point of the parallel ordering algorithm.
+* This function assumes that the graph is already nice partitioned among the
+* processors and then proceeds to perform recursive bisection.
+************************************************************************************/
+void PARGMETIS(idxtype *vtxdist, idxtype *xadj, idxtype *adjncy, int ndims, float *xyz,
+               idxtype *part, int *options, MPI_Comm comm)
+{
+
+  ParMETIS_PartGeom(vtxdist, &ndims, xyz, part, &comm);
+
+  options[0] = -1;
+
+}
+
+/*****************************************************************************
+*  This function performs refinement on a partitioning.
+*****************************************************************************/
+void ParMETIS_RefineKway(idxtype *vtxdist, idxtype *xadj, idxtype *adjncy,
+       idxtype *vwgt, idxtype *adjwgt, int *wgtflag, int *numflag, int *options,
+       int *edgecut, idxtype *part, MPI_Comm *comm)
+{
+  int i;
+  int nparts;
+  int ncon = 1;
+  float *tpwgts, ubvec[MAXNCON];
+  int myoptions[10];
+
+  MPI_Comm_size(*comm, &nparts);
+  tpwgts = fmalloc(nparts*ncon, "tpwgts");
+  for (i=0; i<nparts*ncon; i++)
+    tpwgts[i] = 1.0/(float)(nparts);
+  for (i=0; i<ncon; i++)
+    ubvec[i] = UNBALANCE_FRACTION;
+
+  if (options[0] == 0) {
+    myoptions[0] = 0;
+  }
+  else {
+    myoptions[0] = 1;
+    myoptions[PMV3_OPTION_DBGLVL] = options[OPTION_DBGLVL];
+    myoptions[PMV3_OPTION_SEED] = GLOBAL_SEED;
+    myoptions[PMV3_OPTION_PSR] = COUPLED;
+  }
+
+  ParMETIS_V3_RefineKway(vtxdist, xadj, adjncy, vwgt, adjwgt, wgtflag, numflag,
+  &ncon, &nparts, tpwgts, ubvec, myoptions, edgecut, part, comm);
+
+  free(tpwgts);
+}
+
+
+/***********************************************************************************
+* This function is the entry point of the parallel k-way multilevel partitionioner.
+* This function assumes nothing about the graph distribution.
+* It is the general case.
+************************************************************************************/
+void PARRMETIS(idxtype *vtxdist, idxtype *xadj, idxtype *vwgt, idxtype *adjncy, idxtype *adjwgt,
+               idxtype *part, int *options, MPI_Comm comm)
+{
+  int wgtflag, numflag, edgecut, newoptions[5];
+
+  newoptions[0] = 1;
+  newoptions[OPTION_IPART] = options[2];
+  newoptions[OPTION_FOLDF] = options[1];
+  newoptions[OPTION_DBGLVL] = options[4];
+
+  numflag = options[3];
+  wgtflag = (vwgt == NULL ? 0 : 2) + (adjwgt == NULL ? 0 : 1);
+
+  ParMETIS_RefineKway(vtxdist, xadj, adjncy, vwgt, adjwgt, &wgtflag, &numflag,
+     newoptions, &edgecut, part, &comm);
+
+  options[0] = edgecut;
+
+}
+
+
+/*****************************************************************************
+*  This function computes a repartitioning by local diffusion.
+*****************************************************************************/
+void ParMETIS_RepartLDiffusion(idxtype *vtxdist, idxtype *xadj, idxtype *adjncy, 
+       idxtype *vwgt, idxtype *adjwgt, int *wgtflag, int *numflag, int *options,
+       int *edgecut, idxtype *part, MPI_Comm *comm)
+{
+  int i;
+  int nparts;
+  int ncon = 1;
+  float *tpwgts, ubvec[MAXNCON];
+  float ipc_factor = 1.0;
+  int myoptions[10];
+
+  MPI_Comm_size(*comm, &nparts);
+  tpwgts = fmalloc(nparts*ncon, "tpwgts");
+  for (i=0; i<nparts*ncon; i++)
+    tpwgts[i] = 1.0/(float)(nparts);
+  for (i=0; i<ncon; i++)
+    ubvec[i] = UNBALANCE_FRACTION;
+
+  if (options[0] == 0) {
+    myoptions[0] = 0;
+  }
+  else {
+    myoptions[0] = 1;
+    myoptions[PMV3_OPTION_DBGLVL] = options[OPTION_DBGLVL];
+    myoptions[PMV3_OPTION_SEED] = GLOBAL_SEED;
+    myoptions[PMV3_OPTION_PSR] = COUPLED;
+  }
+
+  ParMETIS_V3_AdaptiveRepart(vtxdist, xadj, adjncy, vwgt, NULL, adjwgt, wgtflag, numflag,
+  &ncon, &nparts, tpwgts, ubvec, &ipc_factor, myoptions, edgecut, part, comm);
+
+  free(tpwgts);
+}
+
+
+/***********************************************************************************
+* This function is the entry point of the parallel multilevel undirected diffusion
+* algorithm. It uses parallel undirected diffusion followed by adaptive k-way
+* refinement. This function utilizes local coarsening.
+************************************************************************************/
+void PARUAMETIS(idxtype *vtxdist, idxtype *xadj, idxtype *vwgt, idxtype *adjncy, idxtype *adjwgt,
+               idxtype *part, int *options, MPI_Comm comm)
+{
+  int wgtflag, numflag, edgecut, newoptions[5];
+
+  newoptions[0] = 1;
+  newoptions[OPTION_IPART] = options[2];
+  newoptions[OPTION_FOLDF] = options[1];
+  newoptions[OPTION_DBGLVL] = options[4];
+
+  numflag = options[3];
+  wgtflag = (vwgt == NULL ? 0 : 2) + (adjwgt == NULL ? 0 : 1);
+
+  ParMETIS_RepartLDiffusion(vtxdist, xadj, adjncy, vwgt, adjwgt, &wgtflag, &numflag,
+     newoptions, &edgecut, part, &comm);
+
+  options[0] = edgecut;
+
+}
+
+/*****************************************************************************
+*  This function computes a repartitioning by global diffusion.
+*****************************************************************************/
+void ParMETIS_RepartGDiffusion(idxtype *vtxdist, idxtype *xadj, idxtype *adjncy, 
+       idxtype *vwgt, idxtype *adjwgt, int *wgtflag, int *numflag, int *options,
+       int *edgecut, idxtype *part, MPI_Comm *comm)
+{
+  int i;
+  int nparts;
+  int ncon = 1;
+  float *tpwgts, ubvec[MAXNCON];
+  float ipc_factor = 100.0;
+  int myoptions[10];
+
+  MPI_Comm_size(*comm, &nparts);
+  tpwgts = fmalloc(nparts*ncon, "tpwgts");
+  for (i=0; i<nparts*ncon; i++)
+    tpwgts[i] = 1.0/(float)(nparts);
+  for (i=0; i<ncon; i++)
+    ubvec[i] = UNBALANCE_FRACTION;
+
+  if (options[0] == 0) {
+    myoptions[0] = 0;
+  }
+  else {
+    myoptions[0] = 1;
+    myoptions[PMV3_OPTION_DBGLVL] = options[OPTION_DBGLVL];
+    myoptions[PMV3_OPTION_SEED] = GLOBAL_SEED;
+    myoptions[PMV3_OPTION_PSR] = COUPLED;
+  }
+
+  ParMETIS_V3_AdaptiveRepart(vtxdist, xadj, adjncy, vwgt, NULL, adjwgt, wgtflag, numflag,
+  &ncon, &nparts, tpwgts, ubvec, &ipc_factor, myoptions, edgecut, part, comm);
+
+  free(tpwgts);
+}
+
+/***********************************************************************************
+* This function is the entry point of the parallel multilevel directed diffusion
+* algorithm. It uses parallel undirected diffusion followed by adaptive k-way
+* refinement. This function utilizes local coarsening.
+************************************************************************************/
+void PARDAMETIS(idxtype *vtxdist, idxtype *xadj, idxtype *vwgt, idxtype *adjncy, idxtype *adjwgt,
+               idxtype *part, int *options, MPI_Comm comm)
+{
+  int wgtflag, numflag, edgecut, newoptions[5];
+
+  newoptions[0] = 1;
+  newoptions[OPTION_IPART] = options[2];
+  newoptions[OPTION_FOLDF] = options[1];
+  newoptions[OPTION_DBGLVL] = options[4];
+
+  numflag = options[3];
+  wgtflag = (vwgt == NULL ? 0 : 2) + (adjwgt == NULL ? 0 : 1);
+
+  ParMETIS_RepartGDiffusion(vtxdist, xadj, adjncy, vwgt, adjwgt, &wgtflag, &numflag,
+     newoptions, &edgecut, part, &comm);
+
+  options[0] = edgecut;
+
+}
+
+/*****************************************************************************
+*  This function computes a repartitioning by scratch-remap.
+*****************************************************************************/
+void ParMETIS_RepartRemap(idxtype *vtxdist, idxtype *xadj, idxtype *adjncy,
+       idxtype *vwgt, idxtype *adjwgt, int *wgtflag, int *numflag, int *options,
+       int *edgecut, idxtype *part, MPI_Comm *comm)
+{
+  int i;
+  int nparts;
+  int ncon = 1;
+  float *tpwgts, ubvec[MAXNCON];
+  float ipc_factor = 1000.0;
+  int myoptions[10];
+
+  MPI_Comm_size(*comm, &nparts);
+  tpwgts = fmalloc(nparts*ncon, "tpwgts");
+  for (i=0; i<nparts*ncon; i++)
+    tpwgts[i] = 1.0/(float)(nparts);
+  for (i=0; i<ncon; i++)
+    ubvec[i] = UNBALANCE_FRACTION;
+
+  if (options[0] == 0) {
+    myoptions[0] = 0;
+  }
+  else {
+    myoptions[0] = 1;
+    myoptions[PMV3_OPTION_DBGLVL] = options[OPTION_DBGLVL];
+    myoptions[PMV3_OPTION_SEED] = GLOBAL_SEED;
+    myoptions[PMV3_OPTION_PSR] = COUPLED;
+  }
+
+  ParMETIS_V3_AdaptiveRepart(vtxdist, xadj, adjncy, vwgt, NULL, adjwgt, wgtflag, numflag,
+  &ncon, &nparts, tpwgts, ubvec, &ipc_factor, myoptions, edgecut, part, comm);
+
+  free(tpwgts);
+}
+
+
+/*****************************************************************************
+*  This function computes a repartitioning by LMSR scratch-remap.
+*****************************************************************************/
+void ParMETIS_RepartMLRemap(idxtype *vtxdist, idxtype *xadj, idxtype *adjncy,
+       idxtype *vwgt, idxtype *adjwgt, int *wgtflag, int *numflag, int *options,
+       int *edgecut, idxtype *part, MPI_Comm *comm)
+{
+  int i;
+  int nparts;
+  int ncon = 1;
+  float *tpwgts, ubvec[MAXNCON];
+  float ipc_factor = 1000.0;
+  int myoptions[10];
+
+  MPI_Comm_size(*comm, &nparts);
+  tpwgts = fmalloc(nparts*ncon, "tpwgts");
+  for (i=0; i<nparts*ncon; i++)
+    tpwgts[i] = 1.0/(float)(nparts);
+  for (i=0; i<ncon; i++)
+    ubvec[i] = UNBALANCE_FRACTION;
+
+  if (options[0] == 0) {
+    myoptions[0] = 0;
+  }
+  else {
+    myoptions[0] = 1;
+    myoptions[PMV3_OPTION_DBGLVL] = options[OPTION_DBGLVL];
+    myoptions[PMV3_OPTION_SEED] = GLOBAL_SEED;
+    myoptions[PMV3_OPTION_PSR] = COUPLED;
+  }
+
+  ParMETIS_V3_AdaptiveRepart(vtxdist, xadj, adjncy, vwgt, NULL, adjwgt, wgtflag, numflag,
+  &ncon, &nparts, tpwgts, ubvec, &ipc_factor, myoptions, edgecut, part, comm);
+
+  free(tpwgts);
+}
+
+/***********************************************************************************
+* This function is the entry point of the parallel ordering algorithm.
+* This function assumes that the graph is already nice partitioned among the
+* processors and then proceeds to perform recursive bisection.
+************************************************************************************/
+void ParMETIS_NodeND(idxtype *vtxdist, idxtype *xadj, idxtype *adjncy, int *numflag,
+  int *options, idxtype *order, idxtype *sizes, MPI_Comm *comm)
+{
+  int myoptions[10];
+
+  if (options[0] == 0) {
+    myoptions[0] = 0;
+  } 
+  else {
+    myoptions[0] = 1;
+    myoptions[PMV3_OPTION_DBGLVL] = options[OPTION_DBGLVL];
+    myoptions[PMV3_OPTION_SEED] = GLOBAL_SEED;
+    myoptions[PMV3_OPTION_IPART] = options[OPTION_IPART];
+  }
+
+  ParMETIS_V3_NodeND(vtxdist, xadj, adjncy, numflag, myoptions, order, sizes, comm);
+}
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/balancemylink.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/balancemylink.c
new file mode 100644
index 0000000..dd944d1
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/balancemylink.c
@@ -0,0 +1,342 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * balancemylink.c
+ *
+ * This file contains code that implements the edge-based FM refinement
+ *
+ * Started 7/23/97
+ * George
+ *
+ * $Id: balancemylink.c,v 1.2 2003/07/21 17:18:48 karypis Exp $
+ */
+
+#include <parmetislib.h>
+#define	PE	0
+
+/*************************************************************************
+* This function performs an edge-based FM refinement
+**************************************************************************/
+int BalanceMyLink(CtrlType *ctrl, GraphType *graph, idxtype *home, int me,
+  int you, float *flows, float maxdiff, float *diff_cost, float *diff_lbavg,
+  float avgvwgt)
+{
+  int h, i, ii, j, k;
+  int nvtxs, ncon;
+  int nqueues, minval, maxval, higain, vtx, edge, totalv;
+  int from, to, qnum, index, nchanges, cut, tmp;
+  int pass, nswaps, nmoves, multiplier;
+  idxtype *xadj, *vsize, *adjncy, *adjwgt, *where, *ed, *id;
+  idxtype *hval, *nvpq, *inq, *map, *rmap, *ptr, *myqueue, *changes;
+  float *nvwgt, lbvec[MAXNCON], pwgts[MAXNCON*2], tpwgts[MAXNCON*2], my_wgt[MAXNCON];
+  float newgain, oldgain = 0.0;
+  float lbavg, bestflow, mycost;
+  float ipc_factor, redist_factor, ftmp;
+  FPQueueType *queues;
+int mype;
+MPI_Comm_rank(MPI_COMM_WORLD, &mype);
+
+  nvtxs = graph->nvtxs;
+  ncon = graph->ncon;
+  xadj = graph->xadj;
+  nvwgt = graph->nvwgt;
+  vsize = graph->vsize;
+  adjncy = graph->adjncy;
+  adjwgt = graph->adjwgt;
+  where = graph->where;
+  ipc_factor = ctrl->ipc_factor;
+  redist_factor = ctrl->redist_factor;
+
+  hval = idxmalloc(nvtxs*7, "hval");
+  id = hval + nvtxs;
+  ed = hval + nvtxs*2;
+  map = hval + nvtxs*3;
+  rmap = hval + nvtxs*4;
+  myqueue = hval + nvtxs*5;
+  changes = hval + nvtxs*6;
+
+  sset(ncon*2, 0.0, pwgts);
+  for (h=0; h<ncon; h++) {
+    tpwgts[h] = -1.0 * flows[h];
+    tpwgts[ncon+h] = flows[h];
+  }
+
+  for (i=0; i<nvtxs; i++) {
+    if (where[i] == me) {
+      for (h=0; h<ncon; h++) {
+        tpwgts[h] += nvwgt[i*ncon+h];
+        pwgts[h] += nvwgt[i*ncon+h];
+      }
+    }
+    else {
+      ASSERTS(where[i] == you);
+      for (h=0; h<ncon; h++) {
+        tpwgts[ncon+h] += nvwgt[i*ncon+h];
+        pwgts[ncon+h] += nvwgt[i*ncon+h];
+      }
+    }
+  }
+
+  /* we don't want any tpwgts to be less than zero */
+  for (h=0; h<ncon; h++) {
+    if (tpwgts[h] < 0.0) {
+      tpwgts[ncon+h] += tpwgts[h];
+      tpwgts[h] = 0.0;
+    }
+
+    if (tpwgts[ncon+h] < 0.0) {
+      tpwgts[h] += tpwgts[ncon+h];
+      tpwgts[ncon+h] = 0.0;
+    }
+  }
+
+  /*******************************/
+  /* insert vertices into queues */
+  /*******************************/
+  minval = maxval = 0;
+  multiplier = 1;
+  for (i=0; i<ncon; i++) {
+    multiplier *= (i+1);
+    maxval += i*multiplier;
+    minval += (ncon-1-i)*multiplier;
+  }
+
+  nqueues = maxval-minval+1;
+  nvpq = idxsmalloc(nqueues, 0, "nvpq");
+  ptr = idxmalloc(nqueues+1, "ptr");
+  inq = idxmalloc(nqueues*2, "inq");
+  queues = (FPQueueType *)(GKmalloc(sizeof(FPQueueType)*nqueues*2, "queues"));
+
+  for (i=0; i<nvtxs; i++)
+    hval[i] = Moc_HashVwgts(ncon, nvwgt+i*ncon) - minval;
+
+  for (i=0; i<nvtxs; i++)
+    nvpq[hval[i]]++;
+
+  ptr[0] = 0;
+  for (i=0; i<nqueues; i++)
+    ptr[i+1] = ptr[i] + nvpq[i];
+
+  for (i=0; i<nvtxs; i++) {
+    map[i] = ptr[hval[i]];
+    rmap[ptr[hval[i]]++] = i;
+  }
+
+  for (i=nqueues-1; i>0; i--)
+    ptr[i] = ptr[i-1];
+  ptr[0] = 0;
+
+  /* initialize queues */
+  for (i=0; i<nqueues; i++)
+    if (nvpq[i] > 0) {
+      FPQueueInit(queues+i, nvpq[i]);
+      FPQueueInit(queues+i+nqueues, nvpq[i]);
+    }
+
+  /* compute internal/external degrees */
+  idxset(nvtxs, 0, id);
+  idxset(nvtxs, 0, ed);
+  for (j=0; j<nvtxs; j++)
+    for (k=xadj[j]; k<xadj[j+1]; k++)
+      if (where[adjncy[k]] == where[j])
+        id[j] += adjwgt[k];
+      else 
+        ed[j] += adjwgt[k];
+
+  nswaps = 0;
+  for (pass=0; pass<N_MOC_BAL_PASSES; pass++) {
+    idxset(nvtxs, -1, myqueue); 
+    idxset(nqueues*2, 0, inq);
+
+    /* insert vertices into correct queues */
+    for (j=0; j<nvtxs; j++) {
+      index = (where[j] == me) ? 0 : nqueues;
+
+      newgain = ipc_factor*(float)(ed[j]-id[j]);
+      if (home[j] == me || home[j] == you) {
+        if (where[j] == home[j])
+          newgain -= redist_factor*(float)vsize[j];
+        else
+          newgain += redist_factor*(float)vsize[j];
+      }
+
+      FPQueueInsert(queues+hval[j]+index, map[j]-ptr[hval[j]], newgain);
+      myqueue[j] = (where[j] == me) ? 0 : 1;
+      inq[hval[j]+index]++;
+    }
+
+/*    bestflow = sfavg(ncon, flows); */
+    for (j=0, h=0; h<ncon; h++)
+      if (fabs(flows[h]) > fabs(flows[j])) j = h;
+        bestflow = fabs(flows[j]);
+
+    nchanges = nmoves = 0;
+    for (ii=0; ii<nvtxs/2; ii++) {
+      from = -1;
+      Moc_DynamicSelectQueue(nqueues, ncon, me, you, inq, flows, &from,
+      &qnum, minval, avgvwgt, maxdiff);
+
+      /* can't find a vertex in one subdomain, try the other */
+      if (from != -1 && qnum == -1) {
+        from = (from == me) ? you : me;
+
+        if (from == me) {
+          for (j=0; j<ncon; j++)
+            if (flows[j] > avgvwgt)
+              break;
+        }
+        else {
+          for (j=0; j<ncon; j++)
+            if (flows[j] < -1.0*avgvwgt)
+              break;
+        }
+
+        if (j != ncon)
+          Moc_DynamicSelectQueue(nqueues, ncon, me, you, inq, flows, &from,
+          &qnum, minval, avgvwgt, maxdiff);
+      }
+
+      if (qnum == -1)
+        break;
+
+      to = (from == me) ? you : me;
+      index = (from == me) ? 0 : nqueues;
+      higain = FPQueueGetMax(queues+qnum+index);
+      inq[qnum+index]--;
+      ASSERTS(higain != -1);
+
+      /*****************/
+      /* make the swap */
+      /*****************/
+      vtx = rmap[higain+ptr[qnum]];
+      myqueue[vtx] = -1;
+      where[vtx] = to;
+      nswaps++;
+      nmoves++;
+
+      /* update the flows */
+      for (j=0; j<ncon; j++)
+        flows[j] += (to == me) ? nvwgt[vtx*ncon+j] : -1.0*nvwgt[vtx*ncon+j];
+ 
+/*      ftmp = sfavg(ncon, flows); */
+      for (j=0, h=0; h<ncon; h++)
+        if (fabs(flows[h]) > fabs(flows[j])) j = h;
+          ftmp = fabs(flows[j]);
+
+      if (ftmp < bestflow) {
+        bestflow = ftmp;
+        nchanges = 0;
+      }
+      else {
+        changes[nchanges++] = vtx;
+      }
+
+      SWAP(id[vtx], ed[vtx], tmp);
+
+      for (j=xadj[vtx]; j<xadj[vtx+1]; j++) {
+        edge = adjncy[j];
+
+        /* must compute oldgain before changing id/ed */
+        if (myqueue[edge] != -1) {
+          oldgain = ipc_factor*(float)(ed[edge]-id[edge]);
+          if (home[edge] == me || home[edge] == you) {
+            if (where[edge] == home[edge])
+              oldgain -= redist_factor*(float)vsize[edge];
+            else
+              oldgain += redist_factor*(float)vsize[edge];
+          }
+        }
+
+        tmp = (to == where[edge] ? adjwgt[j] : -adjwgt[j]);
+        INC_DEC(id[edge], ed[edge], tmp);
+
+        if (myqueue[edge] != -1) {
+          newgain = ipc_factor*(float)(ed[edge]-id[edge]);
+          if (home[edge] == me || home[edge] == you) {
+            if (where[edge] == home[edge])
+              newgain -= redist_factor*(float)vsize[edge];
+            else
+              newgain += redist_factor*(float)vsize[edge];
+          }
+
+          FPQueueUpdate(queues+hval[edge]+(nqueues*myqueue[edge]),
+          map[edge]-ptr[hval[edge]], oldgain, newgain);
+        }
+      }
+    }
+
+    /****************************/
+    /* now go back to best flow */
+    /****************************/
+    nswaps -= nchanges;
+    nmoves -= nchanges;
+    for (i=0; i<nchanges; i++) {
+      vtx = changes[i];
+      from = where[vtx];
+      where[vtx] = to = (from == me) ? you : me;
+
+      SWAP(id[vtx], ed[vtx], tmp);
+      for (j=xadj[vtx]; j<xadj[vtx+1]; j++) {
+        edge = adjncy[j];
+        tmp = (to == where[edge] ? adjwgt[j] : -adjwgt[j]);
+        INC_DEC(id[edge], ed[edge], tmp);
+      }
+    }
+
+    for (i=0; i<nqueues; i++) {
+      if (nvpq[i] > 0) {
+        FPQueueReset(queues+i);
+        FPQueueReset(queues+i+nqueues);
+      }
+    }
+
+    if (nmoves == 0)
+      break;
+  }
+
+  /***************************/
+  /* compute 2-way imbalance */
+  /***************************/
+  sset(ncon, 0.0, my_wgt);
+  for (i=0; i<nvtxs; i++)
+    if (where[i] == me)
+      for (h=0; h<ncon; h++)
+        my_wgt[h] += nvwgt[i*ncon+h];
+
+  for (i=0; i<ncon; i++) {
+    ftmp =  (pwgts[i]+pwgts[ncon+i])/2.0;
+    if (ftmp != 0.0)
+      lbvec[i] = fabs(my_wgt[i]-tpwgts[i]) / ftmp;
+    else
+      lbvec[i] = 0.0;
+  }
+  lbavg = savg(ncon, lbvec);
+  *diff_lbavg = lbavg;
+
+  /****************/
+  /* compute cost */
+  /****************/
+  cut = totalv = 0;
+  for (i=0; i<nvtxs; i++) {
+    if (where[i] != home[i])
+      totalv += vsize[i];
+
+      for (j=xadj[i]; j<xadj[i+1]; j++) 
+        if (where[adjncy[j]] != where[i])
+          cut += adjwgt[j];
+  }
+  cut /= 2;
+  mycost = cut*ipc_factor + totalv*redist_factor;
+  *diff_cost = mycost;
+
+  /* free memory */
+  for (i=0; i<nqueues; i++)
+    if (nvpq[i] > 0) {
+      FPQueueFree(queues+i);
+      FPQueueFree(queues+i+nqueues);
+    }
+
+  GKfree((void **)&hval, (void **)&nvpq, (void **)&ptr, (void **)&inq, (void **)&queues, LTERM);
+  return nswaps;
+}
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/coarsen.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/coarsen.c
new file mode 100644
index 0000000..70f48c2
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/coarsen.c
@@ -0,0 +1,485 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * mcoarsen.c
+ *
+ * This file contains code that performs graph coarsening
+ *
+ * Started 2/22/96
+ * George
+ *
+ * $Id: coarsen.c,v 1.2 2003/07/21 17:18:48 karypis Exp $
+ *
+ */
+
+#include <parmetislib.h>
+
+
+/*************************************************************************
+* This function creates the coarser graph
+**************************************************************************/
+void Moc_Global_CreateCoarseGraph(CtrlType *ctrl, GraphType *graph,
+	WorkSpaceType *wspace, int cnvtxs)
+{
+  int h, i, j, k, l, ii, jj, ll, nnbrs, nvtxs, nedges, ncon;
+  int firstvtx, lastvtx, cfirstvtx, clastvtx, otherlastvtx;
+  int npes=ctrl->npes, mype=ctrl->mype;
+  int cnedges, nsend, nrecv, nkeepsize, nrecvsize, nsendsize, v, u;
+  idxtype *xadj, *ladjncy, *adjwgt, *vwgt, *vsize, *vtxdist, *home;
+  idxtype *match, *cmap, *rcmap, *scmap;
+  idxtype *cxadj, *cadjncy, *cadjwgt, *cvwgt, *cvsize = NULL, *chome = NULL, *cvtxdist;
+  idxtype *rsizes, *ssizes, *rlens, *slens, *rgraph, *sgraph, *perm;
+  idxtype *peind, *recvptr, *recvind;
+  float *nvwgt, *cnvwgt;
+  GraphType *cgraph;
+  KeyValueType *scand, *rcand;
+  int mask=(1<<13)-1, htable[8192], htableidx[8192];
+
+  nvtxs = graph->nvtxs;
+  ncon = graph->ncon;
+
+  vtxdist = graph->vtxdist;
+  xadj = graph->xadj;
+  vwgt = graph->vwgt;
+  vsize = graph->vsize;
+  nvwgt = graph->nvwgt;
+  home = graph->home;
+  ladjncy = graph->adjncy;
+  adjwgt = graph->adjwgt;
+
+  match = graph->match;
+
+  firstvtx = vtxdist[mype];
+  lastvtx = vtxdist[mype+1];
+
+  cmap = graph->cmap = idxmalloc(nvtxs+graph->nrecv, "CreateCoarseGraph: cmap");
+
+  nnbrs = graph->nnbrs;
+  peind = graph->peind;
+  recvind = graph->recvind;
+  recvptr = graph->recvptr;
+
+  /* Use wspace->indices as the tmp space for map of the boundary
+   * vertices that are sent and received */
+  scmap = wspace->indices;
+  rcmap = cmap + nvtxs;
+
+
+  /* Initialize the coarser graph */
+  cgraph = CreateGraph();
+  cgraph->nvtxs = cnvtxs;
+  cgraph->ncon = ncon;
+  cgraph->level = graph->level+1;
+  cgraph->finer = graph;
+  graph->coarser = cgraph;
+
+
+
+  /*************************************************************
+  * Obtain the vtxdist of the coarser graph 
+  **************************************************************/
+  cvtxdist = cgraph->vtxdist = idxmalloc(npes+1, "CreateCoarseGraph: cvtxdist");
+  cvtxdist[npes] = cnvtxs;  /* Use last position in the cvtxdist as a temp buffer */
+
+  MPI_Allgather((void *)(cvtxdist+npes), 1, IDX_DATATYPE, (void *)cvtxdist, 1, IDX_DATATYPE, ctrl->comm);
+
+  MAKECSR(i, npes, cvtxdist);
+
+  cgraph->gnvtxs = cvtxdist[npes];
+
+#ifdef DEBUG_CONTRACT
+  PrintVector(ctrl, npes+1, 0, cvtxdist, "cvtxdist");
+#endif
+
+
+  /*************************************************************
+  * Construct the cmap vector 
+  **************************************************************/
+  cfirstvtx = cvtxdist[mype];
+  clastvtx = cvtxdist[mype+1];
+
+  /* Create the cmap of what you know so far locally */
+  cnvtxs = 0;
+  for (i=0; i<nvtxs; i++) {
+    if (match[i] >= KEEP_BIT) {
+      k = match[i] - KEEP_BIT;
+      if (k>=firstvtx && k<firstvtx+i)
+        continue;  /* Both (i,k) are local and i has been matched via the (k,i) side */
+
+      cmap[i] = cfirstvtx + cnvtxs++;
+      if (k != firstvtx+i && (k>=firstvtx && k<lastvtx)) { /* I'm matched locally */
+        cmap[k-firstvtx] = cmap[i];
+        match[k-firstvtx] += KEEP_BIT;  /* Add the KEEP_BIT to simplify coding */
+      }
+    }
+  }
+  ASSERT(ctrl, cnvtxs == clastvtx-cfirstvtx);
+
+  CommInterfaceData(ctrl, graph, cmap, scmap, rcmap);
+
+  /* Update the cmap of the locally stored vertices that will go away. 
+   * The remote processor assigned cmap for them */
+  for (i=0; i<nvtxs; i++) {
+    if (match[i] < KEEP_BIT) { /* Only vertices that go away satisfy this*/
+      cmap[i] = rcmap[BSearch(graph->nrecv, recvind, match[i])];
+    }
+  }
+
+  CommInterfaceData(ctrl, graph, cmap, scmap, rcmap);
+
+
+#ifdef DEBUG_CONTRACT
+  PrintVector(ctrl, nvtxs, firstvtx, cmap, "Cmap");
+#endif
+
+
+  /*************************************************************
+  * Determine how many adjcency lists you need to send/receive.
+  **************************************************************/
+  /* Use wspace->pairs as the tmp space for the boundary vertices that are sent and received */
+  scand = wspace->pairs;
+  rcand = graph->rcand = (KeyValueType *)GKmalloc(recvptr[nnbrs]*sizeof(KeyValueType), "CreateCoarseGraph: rcand");
+
+  nkeepsize = nsend = nrecv = 0;
+  for (i=0; i<nvtxs; i++) {
+    if (match[i] < KEEP_BIT) { /* This is going away */
+      scand[nsend].key = match[i];
+      scand[nsend].val = i;
+      nsend++;
+    }
+    else {
+      nkeepsize += (xadj[i+1]-xadj[i]);
+
+      k = match[i]-KEEP_BIT;
+      if (k<firstvtx || k>=lastvtx) { /* This is comming from afar */
+        rcand[nrecv].key = k;
+        rcand[nrecv].val = cmap[i] - cfirstvtx;  /* Set it for use during the partition projection */
+        ASSERT(ctrl, rcand[nrecv].val>=0 && rcand[nrecv].val<cnvtxs);
+        nrecv++;
+      }
+    }
+  }
+
+
+#ifdef DEBUG_CONTRACT
+  PrintPairs(ctrl, nsend, scand, "scand");
+  PrintPairs(ctrl, nrecv, rcand, "rcand");
+#endif
+
+  /***************************************************************
+  * Determine how many lists and their sizes  you will send and 
+  * received for each of the neighboring PEs
+  ****************************************************************/
+  rsizes = wspace->pv1;
+  ssizes = wspace->pv2;
+  idxset(nnbrs, 0, ssizes);
+  idxset(nnbrs, 0, rsizes);
+  rlens = graph->rlens = idxmalloc(nnbrs+1, "CreateCoarseGraph: graph->rlens");
+  slens = graph->slens = idxmalloc(nnbrs+1, "CreateCoarseGraph: graph->slens");
+
+  /* Take care the sending data first */
+  ikeyvalsort(nsend, scand);
+  slens[0] = 0;
+  for (k=i=0; i<nnbrs; i++) {
+    otherlastvtx = vtxdist[peind[i]+1];
+    for (; k<nsend && scand[k].key < otherlastvtx; k++)
+      ssizes[i] += (xadj[scand[k].val+1]-xadj[scand[k].val]);
+    slens[i+1] = k;
+  }
+
+  /* Take care the receiving data next. You cannot yet determine the rsizes[] */
+  ikeyvalsort(nrecv, rcand);
+  rlens[0] = 0;
+  for (k=i=0; i<nnbrs; i++) {
+    otherlastvtx = vtxdist[peind[i]+1];
+    for (; k<nrecv && rcand[k].key < otherlastvtx; k++);
+    rlens[i+1] = k;
+  }
+
+#ifdef DEBUG_CONTRACT
+  PrintVector(ctrl, nnbrs+1, 0, slens, "slens");
+  PrintVector(ctrl, nnbrs+1, 0, rlens, "rlens");
+#endif
+
+  /***************************************************************
+  * Exchange size information
+  ****************************************************************/
+  /* Issue the receives first. */
+  for (i=0; i<nnbrs; i++) {
+    if (rlens[i+1]-rlens[i] > 0)  /* Issue a receive only if you are getting something */
+      MPI_Irecv((void *)(rsizes+i), 1, IDX_DATATYPE, peind[i], 1, ctrl->comm, ctrl->rreq+i);
+  }
+
+  /* Take care the sending data next */
+  for (i=0; i<nnbrs; i++) {
+    if (slens[i+1]-slens[i] > 0)  /* Issue a send only if you are sending something */
+      MPI_Isend((void *)(ssizes+i), 1, IDX_DATATYPE, peind[i], 1, ctrl->comm, ctrl->sreq+i);
+  }
+
+  /* OK, now get into the loop waiting for the operations to finish */
+  for (i=0; i<nnbrs; i++) {
+    if (rlens[i+1]-rlens[i] > 0)  
+      MPI_Wait(ctrl->rreq+i, &ctrl->status);
+  }
+  for (i=0; i<nnbrs; i++) {
+    if (slens[i+1]-slens[i] > 0)  
+      MPI_Wait(ctrl->sreq+i, &ctrl->status);
+  }
+
+
+#ifdef DEBUG_CONTRACT
+  PrintVector(ctrl, nnbrs, 0, rsizes, "rsizes");
+  PrintVector(ctrl, nnbrs, 0, ssizes, "ssizes");
+#endif
+
+  /*************************************************************
+  * Allocate memory for received/sent graphs and start sending 
+  * and receiving data.
+  * rgraph and sgraph is a different data structure than CSR
+  * to facilitate single message exchange.
+  **************************************************************/
+  nrecvsize = idxsum(nnbrs, rsizes);
+  nsendsize = idxsum(nnbrs, ssizes);
+  if ((4+ncon)*(nrecv+nsend) + 2*(nrecvsize+nsendsize) <= wspace->nlarge) {  
+    rgraph = (idxtype *)wspace->degrees;
+    sgraph = rgraph + (4+ncon)*nrecv+2*nrecvsize;
+  }
+  else {
+    rgraph = idxmalloc((4+ncon)*nrecv+2*nrecvsize, "CreateCoarseGraph: rgraph");
+    sgraph = idxmalloc((4+ncon)*nsend+2*nsendsize, "CreateCoarseGraph: sgraph");
+  }
+
+  /* Deal with the received portion first */
+  for (l=i=0; i<nnbrs; i++) {
+    /* Issue a receive only if you are getting something */
+    if (rlens[i+1]-rlens[i] > 0) {
+      MPI_Irecv((void *)(rgraph+l), (4+ncon)*(rlens[i+1]-rlens[i])+2*rsizes[i], IDX_DATATYPE, peind[i], 1, ctrl->comm, ctrl->rreq+i);
+      l += (4+ncon)*(rlens[i+1]-rlens[i])+2*rsizes[i];
+    }
+  }
+
+
+  /* Deal with the sent portion now */
+  for (ll=l=i=0; i<nnbrs; i++) {
+    if (slens[i+1]-slens[i] > 0) {  /* Issue a send only if you are sending something */
+      for (k=slens[i]; k<slens[i+1]; k++) {
+        ii = scand[k].val;
+        sgraph[ll++] = firstvtx+ii;
+        sgraph[ll++] = xadj[ii+1]-xadj[ii];
+        for (h=0; h<ncon; h++)
+          sgraph[ll++] = vwgt[ii*ncon+h];
+        sgraph[ll++] = (ctrl->partType == STATIC_PARTITION) ? -1 : vsize[ii];
+        sgraph[ll++] = (ctrl->partType == STATIC_PARTITION) ? -1 : home[ii];
+        for (jj=xadj[ii]; jj<xadj[ii+1]; jj++) {
+          sgraph[ll++] = cmap[ladjncy[jj]];
+          sgraph[ll++] = adjwgt[jj];
+        }
+      }
+
+      ASSERT(ctrl, ll-l == (4+ncon)*(slens[i+1]-slens[i])+2*ssizes[i]);
+
+      /* myprintf(ctrl, "Sending to pe:%d, %d lists of size %d\n", peind[i], slens[i+1]-slens[i], ssizes[i]); */
+      MPI_Isend((void *)(sgraph+l), ll-l, IDX_DATATYPE, peind[i], 1, ctrl->comm, ctrl->sreq+i);
+      l = ll;
+    }
+  }
+
+  /* OK, now get into the loop waiting for the operations to finish */
+  for (i=0; i<nnbrs; i++) {
+    if (rlens[i+1]-rlens[i] > 0)  
+      MPI_Wait(ctrl->rreq+i, &ctrl->status);
+  }
+  for (i=0; i<nnbrs; i++) {
+    if (slens[i+1]-slens[i] > 0)  
+      MPI_Wait(ctrl->sreq+i, &ctrl->status);
+  }
+
+
+#ifdef DEBUG_CONTRACT
+  rprintf(ctrl, "Graphs were sent!\n");
+  PrintTransferedGraphs(ctrl, nnbrs, peind, slens, rlens, sgraph, rgraph);
+#endif
+
+  /*************************************************************
+  * Setup the mapping from indices returned by BSearch to 
+  * those that are actually stored 
+  **************************************************************/
+  perm = idxsmalloc(recvptr[nnbrs], -1, "CreateCoarseGraph: perm");
+  for (j=i=0; i<nrecv; i++) {
+  /*   myprintf(ctrl, "For received vertex %d, set perm[%d]=%d\n", rgraph[j], BSearch(graph->nrecv, recvind, rgraph[j]), j+ncon); */
+    perm[BSearch(graph->nrecv, recvind, rgraph[j])] = j+1;
+    j += (4+ncon)+2*rgraph[j+1];
+  }
+
+  /*************************************************************
+  * Finally, create the coarser graph
+  **************************************************************/
+  /* Allocate memory for the coarser graph, and fire up coarsening */
+  cxadj = cgraph->xadj = idxmalloc(cnvtxs+1, "CreateCoarserGraph: cxadj");
+  cvwgt = cgraph->vwgt = idxmalloc(cnvtxs*ncon, "CreateCoarserGraph: cvwgt");
+  if (ctrl->partType == ADAPTIVE_PARTITION || ctrl->partType == REFINE_PARTITION) {
+    cvsize = cgraph->vsize = idxmalloc(cnvtxs, "CreateCoarserGraph: cvsize");
+    chome = cgraph->home = idxmalloc(cnvtxs, "CreateCoarserGraph: chome");
+  }
+  cnvwgt = cgraph->nvwgt = fmalloc(cnvtxs*ncon, "CreateCoarserGraph: cnvwgt");
+  cadjncy = idxmalloc(2*(nkeepsize+nrecvsize), "CreateCoarserGraph: cadjncy");
+  cadjwgt = cadjncy + nkeepsize+nrecvsize;
+
+  iset(8192, -1, htable);
+
+  cxadj[0] = cnvtxs = cnedges = 0;
+  for (i=0; i<nvtxs; i++) {
+    if (match[i] >= KEEP_BIT) {
+      v = firstvtx+i; 
+      u = match[i]-KEEP_BIT;
+
+      if (u>=firstvtx && u<lastvtx && v > u) 
+        continue;  /* I have already collapsed it as (u,v) */
+
+      /* Collapse the v vertex first, which you know is local */
+      for (h=0; h<ncon; h++)
+        cvwgt[cnvtxs*ncon+h] = vwgt[i*ncon+h];
+      if (ctrl->partType == ADAPTIVE_PARTITION || ctrl->partType == REFINE_PARTITION) {
+        cvsize[cnvtxs] = vsize[i];
+        chome[cnvtxs] = home[i];
+      }
+      nedges = 0;
+
+      for (j=xadj[i]; j<xadj[i+1]; j++) {
+        k = cmap[ladjncy[j]];
+        if (k != cfirstvtx+cnvtxs) {  /* If this is not an internal edge */
+          l = k&mask;
+          if (htable[l] == -1) { /* Seeing this for first time */
+            htable[l] = k;
+            htableidx[l] = cnedges+nedges;
+            cadjncy[cnedges+nedges] = k; 
+            cadjwgt[cnedges+nedges++] = adjwgt[j];
+          }
+          else if (htable[l] == k) {
+            cadjwgt[htableidx[l]] += adjwgt[j];
+          }
+          else { /* Now you have to go and do a search. Expensive case */
+            for (l=0; l<nedges; l++) {
+              if (cadjncy[cnedges+l] == k)
+                break;
+            }
+            if (l < nedges) {
+              cadjwgt[cnedges+l] += adjwgt[j];
+            }
+            else {
+              cadjncy[cnedges+nedges] = k; 
+              cadjwgt[cnedges+nedges++] = adjwgt[j];
+            }
+          }
+        }
+      }
+
+      /* Collapse the u vertex next */
+      if (v != u) { 
+        if (u>=firstvtx && u<lastvtx) { /* Local vertex */
+          u -= firstvtx;
+          for (h=0; h<ncon; h++)
+            cvwgt[cnvtxs*ncon+h] += vwgt[u*ncon+h];
+          if (ctrl->partType == ADAPTIVE_PARTITION || ctrl->partType == REFINE_PARTITION) {
+            cvsize[cnvtxs] += vsize[u];
+            /* chome[cnvtxs] = home[u]; */
+          }
+
+          for (j=xadj[u]; j<xadj[u+1]; j++) {
+            k = cmap[ladjncy[j]];
+            if (k != cfirstvtx+cnvtxs) {  /* If this is not an internal edge */
+              l = k&mask;
+              if (htable[l] == -1) { /* Seeing this for first time */
+                htable[l] = k;
+                htableidx[l] = cnedges+nedges;
+                cadjncy[cnedges+nedges] = k; 
+                cadjwgt[cnedges+nedges++] = adjwgt[j];
+              }
+              else if (htable[l] == k) {
+                cadjwgt[htableidx[l]] += adjwgt[j];
+              }
+              else { /* Now you have to go and do a search. Expensive case */
+                for (l=0; l<nedges; l++) {
+                  if (cadjncy[cnedges+l] == k)
+                    break;
+                }
+                if (l < nedges) {
+                  cadjwgt[cnedges+l] += adjwgt[j];
+                }
+                else {
+                  cadjncy[cnedges+nedges] = k; 
+                  cadjwgt[cnedges+nedges++] = adjwgt[j];
+                }
+              }
+            }
+          }
+        }
+        else { /* Remote vertex */
+          u = perm[BSearch(graph->nrecv, recvind, u)];
+          for (h=0; h<ncon; h++)
+            /* Remember that the +1 stores the vertex weight */
+            cvwgt[cnvtxs*ncon+h] += rgraph[(u+1)+h];
+            if (ctrl->partType == ADAPTIVE_PARTITION || ctrl->partType == REFINE_PARTITION) {
+              cvsize[cnvtxs] += rgraph[u+1+ncon];
+              chome[cnvtxs] = rgraph[u+2+ncon];
+            }
+          for (j=0; j<rgraph[u]; j++) {
+            k = rgraph[u+3+ncon+2*j];
+            if (k != cfirstvtx+cnvtxs) {  /* If this is not an internal edge */
+              l = k&mask;
+              if (htable[l] == -1) { /* Seeing this for first time */
+                htable[l] = k;
+                htableidx[l] = cnedges+nedges;
+                cadjncy[cnedges+nedges] = k; 
+                cadjwgt[cnedges+nedges++] = rgraph[u+3+ncon+2*j+1];
+              }
+              else if (htable[l] == k) {
+                cadjwgt[htableidx[l]] += rgraph[u+3+ncon+2*j+1];
+              }
+              else { /* Now you have to go and do a search. Expensive case */
+                for (l=0; l<nedges; l++) {
+                  if (cadjncy[cnedges+l] == k)
+                    break;
+                }
+                if (l < nedges) {
+                  cadjwgt[cnedges+l] += rgraph[u+3+ncon+2*j+1];
+                }
+                else {
+                  cadjncy[cnedges+nedges] = k; 
+                  cadjwgt[cnedges+nedges++] = rgraph[u+3+ncon+2*j+1];
+                }
+              }
+            }
+          }
+        }
+      }
+
+      cnedges += nedges;
+      for (j=cxadj[cnvtxs]; j<cnedges; j++)
+        htable[cadjncy[j]&mask] = -1;  /* reset the htable */
+      cxadj[++cnvtxs] = cnedges;
+    }
+  }
+
+  cgraph->nedges = cnedges;
+
+  /* ADD:  In order to keep from having to change this too much */
+  /* ADD:  I kept vwgt array and recomputed nvwgt for each coarser graph */
+  for (j=0; j<cnvtxs; j++)
+    for (h=0; h<ncon; h++)
+      cgraph->nvwgt[j*ncon+h] = (float)(cvwgt[j*ncon+h])/(float)(ctrl->tvwgts[h]);
+
+  cgraph->adjncy = idxmalloc(cnedges, "CreateCoarserGraph: cadjncy");
+  cgraph->adjwgt = idxmalloc(cnedges, "CreateCoarserGraph: cadjwgt");
+  idxcopy(cnedges, cadjncy, cgraph->adjncy);
+  idxcopy(cnedges, cadjwgt, cgraph->adjwgt);
+  free(cadjncy);
+
+  free(perm);
+
+  if (rgraph != (idxtype *)wspace->degrees) 
+    GKfree((void **)&rgraph, (void **)&sgraph, LTERM);
+
+}
+
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/comm.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/comm.c
new file mode 100644
index 0000000..fb18789
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/comm.c
@@ -0,0 +1,213 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * comm.c
+ *
+ * This function provides various high level communication functions 
+ *
+ * $Id: comm.c,v 1.2 2003/07/21 17:18:48 karypis Exp $
+ */
+
+#include <parmetislib.h>
+
+
+
+/*************************************************************************
+* This function performs the gather/scatter for the boundary vertices
+**************************************************************************/
+void CommInterfaceData(CtrlType *ctrl, GraphType *graph, idxtype *data, 
+                       idxtype *sendvector, idxtype *recvvector)
+{
+  int i, k, nnbrs, firstvtx;
+  idxtype *peind, *sendptr, *sendind, *recvptr, *recvind;
+
+  firstvtx = graph->vtxdist[ctrl->mype];
+  nnbrs = graph->nnbrs;
+  peind = graph->peind;
+  sendptr = graph->sendptr;
+  sendind = graph->sendind;
+  recvptr = graph->recvptr;
+  recvind = graph->recvind;
+
+  /* Issue the receives first */
+  for (i=0; i<nnbrs; i++) {
+    MPI_Irecv((void *)(recvvector+recvptr[i]), recvptr[i+1]-recvptr[i], IDX_DATATYPE, 
+              peind[i], 1, ctrl->comm, ctrl->rreq+i);
+  }
+
+  /* Issue the sends next */
+  k = sendptr[nnbrs];
+  for (i=0; i<k; i++) 
+    sendvector[i] = data[sendind[i]-firstvtx];
+
+  for (i=0; i<nnbrs; i++) {
+    MPI_Isend((void *)(sendvector+sendptr[i]), sendptr[i+1]-sendptr[i], IDX_DATATYPE, 
+              peind[i], 1, ctrl->comm, ctrl->sreq+i); 
+  }
+
+  /* OK, now get into the loop waiting for the operations to finish */
+  MPI_Waitall(nnbrs, ctrl->rreq, ctrl->statuses); 
+  MPI_Waitall(nnbrs, ctrl->sreq, ctrl->statuses); 
+
+}
+
+
+
+/*************************************************************************
+* This function performs the gather/scatter for the boundary vertices
+**************************************************************************/
+void CommChangedInterfaceData(CtrlType *ctrl, GraphType *graph, 
+        int nchanged, idxtype *changed, idxtype *data,
+        KeyValueType *sendpairs, KeyValueType *recvpairs, idxtype *psendptr)
+{
+  int i, j, k, n, penum, nnbrs, firstvtx, nrecv;
+  idxtype *peind, *sendptr, *recvptr, *recvind, *pexadj, *peadjncy, *peadjloc;
+  KeyValueType *pairs;
+
+  firstvtx = graph->vtxdist[ctrl->mype];
+  nnbrs = graph->nnbrs;
+  nrecv = graph->nrecv;
+  peind = graph->peind;
+  sendptr = graph->sendptr;
+  recvptr = graph->recvptr;
+  recvind = graph->recvind;
+  pexadj = graph->pexadj;
+  peadjncy = graph->peadjncy;
+  peadjloc = graph->peadjloc;
+
+  /* Issue the receives first */
+  for (i=0; i<nnbrs; i++) {
+    MPI_Irecv((void *)(recvpairs+recvptr[i]), 2*(recvptr[i+1]-recvptr[i]), IDX_DATATYPE, 
+              peind[i], 1, ctrl->comm, ctrl->rreq+i);
+  }
+
+  if (nchanged != 0) {
+    idxcopy(ctrl->npes, sendptr, psendptr);
+
+    /* Copy the changed values into the sendvector */
+    for (i=0; i<nchanged; i++) {
+      j = changed[i];
+      for (k=pexadj[j]; k<pexadj[j+1]; k++) {
+        penum = peadjncy[k];
+        sendpairs[psendptr[penum]].key = peadjloc[k];
+        sendpairs[psendptr[penum]].val = data[j];
+        psendptr[penum]++;
+      }
+    }
+
+    for (i=0; i<nnbrs; i++) {
+      MPI_Isend((void *)(sendpairs+sendptr[i]), 2*(psendptr[i]-sendptr[i]), IDX_DATATYPE, 
+                peind[i], 1, ctrl->comm, ctrl->sreq+i);
+    }
+  }
+  else {
+    for (i=0; i<nnbrs; i++) 
+      MPI_Isend((void *)(sendpairs), 0, IDX_DATATYPE, peind[i], 1, ctrl->comm, ctrl->sreq+i);
+  }
+
+  /* OK, now get into the loop waiting for the operations to finish */
+  for (i=0; i<nnbrs; i++) {
+    MPI_Wait(ctrl->rreq+i, &(ctrl->status));
+    MPI_Get_count(&ctrl->status, IDX_DATATYPE, &n);
+    if (n != 0) {
+      n = n/2;
+      pairs = recvpairs+graph->recvptr[i];
+      for (k=0; k<n; k++) 
+        data[pairs[k].key] = pairs[k].val;
+    }
+  }
+
+  MPI_Waitall(nnbrs, ctrl->sreq, ctrl->statuses);
+}
+
+
+
+/*************************************************************************
+* This function computes the max of a single element
+**************************************************************************/
+int GlobalSEMax(CtrlType *ctrl, int value)
+{
+  int max;
+
+  MPI_Allreduce((void *)&value, (void *)&max, 1, MPI_INT, MPI_MAX, ctrl->comm);
+
+  return max;
+}
+
+/*************************************************************************
+* This function computes the max of a single element
+**************************************************************************/
+double GlobalSEMaxDouble(CtrlType *ctrl, double value)
+{
+  double max;
+
+  MPI_Allreduce((void *)&value, (void *)&max, 1, MPI_DOUBLE, MPI_MAX, ctrl->comm);
+
+  return max;
+}
+
+
+
+/*************************************************************************
+* This function computes the max of a single element
+**************************************************************************/
+int GlobalSEMin(CtrlType *ctrl, int value)
+{
+  int min;
+
+  MPI_Allreduce((void *)&value, (void *)&min, 1, MPI_INT, MPI_MIN, ctrl->comm);
+
+  return min;
+}
+
+/*************************************************************************
+* This function computes the max of a single element
+**************************************************************************/
+int GlobalSESum(CtrlType *ctrl, int value)
+{
+  int sum;
+
+  MPI_Allreduce((void *)&value, (void *)&sum, 1, MPI_INT, MPI_SUM, ctrl->comm);
+
+  return sum;
+}
+
+
+/*************************************************************************
+* This function computes the max of a single element
+**************************************************************************/
+float GlobalSEMaxFloat(CtrlType *ctrl, float value)
+{
+  float max;
+
+  MPI_Allreduce((void *)&value, (void *)&max, 1, MPI_FLOAT, MPI_MAX, ctrl->comm);
+
+  return max;
+}
+
+
+
+/*************************************************************************
+* This function computes the max of a single element
+**************************************************************************/
+float GlobalSEMinFloat(CtrlType *ctrl, float value)
+{
+  float min;
+
+  MPI_Allreduce((void *)&value, (void *)&min, 1, MPI_FLOAT, MPI_MIN, ctrl->comm);
+
+  return min;
+}
+
+/*************************************************************************
+* This function computes the max of a single element
+**************************************************************************/
+float GlobalSESumFloat(CtrlType *ctrl, float value)
+{
+  float sum;
+
+  MPI_Allreduce((void *)&value, (void *)&sum, 1, MPI_FLOAT, MPI_SUM, ctrl->comm);
+
+  return sum;
+}
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/csrmatch.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/csrmatch.c
new file mode 100644
index 0000000..ace7998
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/csrmatch.c
@@ -0,0 +1,88 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * csrmatch.c
+ *
+ * This file contains the code that computes matchings
+ *
+ * Started 7/23/97
+ * George
+ *
+ * $Id: csrmatch.c,v 1.2 2003/07/21 17:18:48 karypis Exp $
+ *
+ */
+
+#include <parmetislib.h>
+
+
+
+
+/*************************************************************************
+* This function finds a matching using the HEM heuristic
+**************************************************************************/
+void CSR_Match_SHEM(MatrixType *matrix, idxtype *match, idxtype *mlist,
+     idxtype *skip, int ncon)
+{
+  int h, i, ii, j;
+  int nrows, edge, maxidx, count;
+  float maxwgt;
+  idxtype *rowptr, *colind;
+  float *transfer;
+  KVType *links;
+
+  nrows = matrix->nrows;
+  rowptr = matrix->rowptr;
+  colind = matrix->colind;
+  transfer = matrix->transfer;
+
+  idxset(nrows, UNMATCHED, match);
+
+  links = (KVType *)GKmalloc(sizeof(KVType)*nrows, "links");
+  for (i=0; i<nrows; i++) { 
+    links[i].key = i; 
+    links[i].val = 0.0;
+  }
+
+  for (i=0; i<nrows; i++)
+    for (j=rowptr[i]; j<rowptr[i+1]; j++) 
+      for (h=0; h<ncon; h++)
+        if (links[i].val < fabs(transfer[j*ncon+h]))
+          links[i].val = fabs(transfer[j*ncon+h]);
+
+  qsort(links, nrows, sizeof(KVType), myvalkeycompare);
+
+  count = 0;
+  for (ii=0; ii<nrows; ii++) {
+    i = links[ii].key;
+
+    if (match[i] == UNMATCHED) {
+      maxidx = i;
+      maxwgt = 0.0;
+
+      /* Find a heavy-edge matching */
+      for (j=rowptr[i]; j<rowptr[i+1]; j++) {
+        edge = colind[j];
+        if (match[edge] == UNMATCHED && edge != i && skip[j] == 0) {
+          for (h=0; h<ncon; h++)
+            if (maxwgt < fabs(transfer[j*ncon+h]))
+              break;
+
+          if (h != ncon) {
+            maxwgt = fabs(transfer[j*ncon+h]);
+            maxidx = edge;
+          }
+        }
+      }
+
+      if (maxidx != i) {
+        match[i] = maxidx;
+        match[maxidx] = i;
+        mlist[count++] = amax(i, maxidx);
+        mlist[count++] = amin(i, maxidx);
+      }
+    }
+  }
+
+  GKfree((void **)&links, LTERM);
+}
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/debug.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/debug.c
new file mode 100644
index 0000000..ebdf69c
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/debug.c
@@ -0,0 +1,247 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * debug.c
+ *
+ * This file contains various functions that are used to display debuging 
+ * information
+ *
+ * Started 10/20/96
+ * George
+ *
+ * $Id: debug.c,v 1.2 2003/07/21 17:18:48 karypis Exp $
+ *
+ */
+
+#include <parmetislib.h>
+
+
+/*************************************************************************
+* This function prints a vector stored in each processor 
+**************************************************************************/
+void PrintVector(CtrlType *ctrl, int n, int first, idxtype *vec, char *title)
+{
+  int i, penum;
+
+  for (penum=0; penum<ctrl->npes; penum++) {
+    if (ctrl->mype == penum) {
+      if (ctrl->mype == 0)
+        printf("%s\n", title);
+      printf("\t%3d. ", ctrl->mype);
+      for (i=0; i<n; i++)
+        printf("[%d %hd] ", first+i, vec[i]);
+      printf("\n");
+      fflush(stdout);
+    }
+    MPI_Barrier(ctrl->comm);
+  }
+}
+
+
+/*************************************************************************
+* This function prints a vector stored in each processor 
+**************************************************************************/
+void PrintVector2(CtrlType *ctrl, int n, int first, idxtype *vec, char *title)
+{
+  int i, penum;
+
+  for (penum=0; penum<ctrl->npes; penum++) {
+    if (ctrl->mype == penum) {
+      if (ctrl->mype == 0)
+        printf("%s\n", title);
+      printf("\t%3d. ", ctrl->mype);
+      for (i=0; i<n; i++)
+        printf("[%d %d.%hd] ", first+i, (vec[i]>=KEEP_BIT ? 1 : 0), (vec[i]>=KEEP_BIT ? vec[i]-KEEP_BIT : vec[i]));
+      printf("\n");
+      fflush(stdout);
+    }
+    MPI_Barrier(ctrl->comm);
+  }
+}
+
+
+/*************************************************************************
+* This function prints a vector stored in each processor 
+**************************************************************************/
+void PrintPairs(CtrlType *ctrl, int n, KeyValueType *pairs, char *title)
+{
+  int i, penum;
+
+  for (penum=0; penum<ctrl->npes; penum++) {
+    if (ctrl->mype == penum) {
+      if (ctrl->mype == 0)
+        printf("%s\n", title);
+      printf("\t%3d. ", ctrl->mype);
+      for (i=0; i<n; i++)
+        printf("[%d %hd,%hd] ", i, pairs[i].key, pairs[i].val);
+      printf("\n");
+      fflush(stdout);
+    }
+    MPI_Barrier(ctrl->comm);
+  }
+}
+
+
+
+/*************************************************************************
+* This function prints the local portion of the graph stored at each 
+* processor
+**************************************************************************/
+void PrintGraph(CtrlType *ctrl, GraphType *graph)
+{
+  int i, j, penum;
+  int firstvtx;
+
+  MPI_Barrier(ctrl->comm);
+
+  firstvtx = graph->vtxdist[ctrl->mype];
+
+  for (penum=0; penum<ctrl->npes; penum++) {
+    if (ctrl->mype == penum) {
+      printf("\t%d", penum);
+      for (i=0; i<graph->nvtxs; i++) {
+        if (i==0)
+          printf("\t%2d %2d\t", firstvtx+i, graph->vwgt[i]);
+        else
+          printf("\t\t%2d %2d\t", firstvtx+i, graph->vwgt[i]);
+        for (j=graph->xadj[i]; j<graph->xadj[i+1]; j++)
+          printf("[%d %d] ", graph->adjncy[j], graph->adjwgt[j]);
+        printf("\n");
+      }
+      fflush(stdout);
+    }
+    MPI_Barrier(ctrl->comm);
+  }
+}
+
+
+/*************************************************************************
+* This function prints the local portion of the graph stored at each 
+* processor along with degree information during refinement
+**************************************************************************/
+void PrintGraph2(CtrlType *ctrl, GraphType *graph)
+{
+  int i, j, penum;
+  int firstvtx;
+
+  MPI_Barrier(ctrl->comm);
+
+  firstvtx = graph->vtxdist[ctrl->mype];
+
+  for (penum=0; penum<ctrl->npes; penum++) {
+    if (ctrl->mype == penum) {
+      printf("\t%d", penum);
+      for (i=0; i<graph->nvtxs; i++) {
+        if (i==0)
+          printf("\t%2d %2d [%d %d %d]\t", firstvtx+i, graph->vwgt[i], graph->where[i], graph->rinfo[i].id, graph->rinfo[i].ed);
+        else
+          printf("\t\t%2d %2d [%d %d %d]\t", firstvtx+i, graph->vwgt[i], graph->where[i], graph->rinfo[i].id, graph->rinfo[i].ed);
+        for (j=graph->xadj[i]; j<graph->xadj[i+1]; j++)
+          printf("[%d %d] ", graph->adjncy[j], graph->adjwgt[j]);
+        printf("\n");
+      }
+      fflush(stdout);
+    }
+    MPI_Barrier(ctrl->comm);
+  }
+}
+
+
+/*************************************************************************
+* This function prints the information computed during setup
+**************************************************************************/
+void PrintSetUpInfo(CtrlType *ctrl, GraphType *graph)
+{
+  int i, j, penum;
+
+  MPI_Barrier(ctrl->comm);
+
+  for (penum=0; penum<ctrl->npes; penum++) {
+    if (ctrl->mype == penum) {
+      printf("PE: %d, nnbrs: %d\n", ctrl->mype, graph->nnbrs);
+      printf("\tSending...\n");
+      for (i=0; i<graph->nnbrs; i++) {
+        printf("\t\tTo: %d: ", graph->peind[i]);
+        for (j=graph->sendptr[i]; j<graph->sendptr[i+1]; j++)
+          printf("%d ", graph->sendind[j]);
+        printf("\n");
+      }
+      printf("\tReceiving...\n");
+      for (i=0; i<graph->nnbrs; i++) {
+        printf("\t\tFrom: %d: ", graph->peind[i]);
+        for (j=graph->recvptr[i]; j<graph->recvptr[i+1]; j++)
+          printf("%d ", graph->recvind[j]);
+        printf("\n");
+      }
+      printf("\n");
+    }
+    MPI_Barrier(ctrl->comm);
+  }
+
+}
+
+
+/*************************************************************************
+* This function prints information about the graphs that were sent/received
+**************************************************************************/
+void PrintTransferedGraphs(CtrlType *ctrl, int nnbrs, idxtype *peind, idxtype *slens, 
+      idxtype *rlens, idxtype *sgraph, idxtype *rgraph)
+{
+  int i, ii, jj, ll, penum;
+
+  MPI_Barrier(ctrl->comm);
+  for (penum=0; penum<ctrl->npes; penum++) {
+    if (ctrl->mype == penum) {
+      printf("PE: %d, nnbrs: %d", ctrl->mype, nnbrs);
+      for (ll=i=0; i<nnbrs; i++) {
+        if (slens[i+1]-slens[i] > 0) {
+          printf("\n\tTo %d\t", peind[i]);
+          for (ii=slens[i]; ii<slens[i+1]; ii++) {
+            printf("%d %d %d, ", sgraph[ll], sgraph[ll+1], sgraph[ll+2]);
+            for (jj=0; jj<sgraph[ll+1]; jj++)
+              printf("[%d %d] ", sgraph[ll+3+2*jj], sgraph[ll+3+2*jj+1]);
+            printf("\n\t\t");
+            ll += 3+2*sgraph[ll+1];
+          }
+        }
+      }
+
+      for (ll=i=0; i<nnbrs; i++) {
+        if (rlens[i+1]-rlens[i] > 0) {
+          printf("\n\tFrom %d\t", peind[i]);
+          for (ii=rlens[i]; ii<rlens[i+1]; ii++) {
+            printf("%d %d %d, ", rgraph[ll], rgraph[ll+1], rgraph[ll+2]);
+            for (jj=0; jj<rgraph[ll+1]; jj++)
+              printf("[%d %d] ", rgraph[ll+3+2*jj], rgraph[ll+3+2*jj+1]);
+            printf("\n\t\t");
+            ll += 3+2*rgraph[ll+1];
+          }
+        }
+      }
+      printf("\n");
+    }
+    MPI_Barrier(ctrl->comm);
+  }
+
+}
+
+
+/*************************************************************************
+* This function writes a graph in the format used by serial METIS
+**************************************************************************/
+void WriteMetisGraph(int nvtxs, idxtype *xadj, idxtype *adjncy, idxtype *vwgt, idxtype *adjwgt)
+{
+  int i, j;
+  FILE *fp;
+
+  fp = fopen("test.graph", "w");
+
+  fprintf(fp, "%d %d 11", nvtxs, xadj[nvtxs]/2);
+  for (i=0; i<nvtxs; i++) {
+    fprintf(fp, "\n%d ", vwgt[i]);
+    for (j=xadj[i]; j<xadj[i+1]; j++)
+      fprintf(fp, " %d %d", adjncy[j]+1, adjwgt[j]);
+  }
+  fclose(fp);
+}
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/defs.h b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/defs.h
new file mode 100644
index 0000000..3d47f1d
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/defs.h
@@ -0,0 +1,102 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * defs.h
+ *
+ * This file contains constant definitions
+ *
+ * Started 8/27/94
+ * George
+ *
+ * $Id: defs.h,v 1.4 2003/07/22 20:29:05 karypis Exp $
+ *
+ */
+
+
+#define GLOBAL_DBGLVL			0
+#define GLOBAL_SEED			15
+
+#define MC_FLOW_BALANCE_THRESHOLD       0.2
+#define MOC_GD_GRANULARITY_FACTOR       1.0
+#define RIP_SPLIT_FACTOR                8
+#define MAX_NPARTS_MULTIPLIER		20
+
+#define STATIC_PARTITION        1
+#define ADAPTIVE_PARTITION      2
+#define REFINE_PARTITION        3
+#define MESH_PARTITION		4
+
+#define REDIST_WGT              2.0
+#define MAXNVWGT_FACTOR         2.0
+
+#define MAXNCON                 12
+#define MAXNOBJ                 12
+#define N_MOC_REDO_PASSES       10
+#define N_MOC_GR_PASSES         8
+#define NREMAP_PASSES           8
+#define N_MOC_GD_PASSES         6
+#define N_MOC_BAL_PASSES        4
+#define NMATCH_PASSES           4
+
+#define COUPLED			1
+#define DISCOUPLED		2
+
+#define MAX_NCON_FOR_DIFFUSION  2
+#define SMALLGRAPH              10000
+
+#define LTERM                   (void **) 0     /* List terminator for GKfree() */
+
+#define NGD_PASSES		20
+
+#define OPTION_IPART		1
+#define OPTION_FOLDF		2
+#define OPTION_DBGLVL		3
+
+#define PMV3_OPTION_DBGLVL	1
+#define PMV3_OPTION_SEED	2
+#define PMV3_OPTION_IPART	3
+#define PMV3_OPTION_PSR		3
+
+#define XYZ_XCOORD		1
+#define XYZ_SPFILL		2
+
+/* Type of initial vertex separator algorithms */
+#define ISEP_EDGE		1
+#define ISEP_NODE		2
+
+#define UNMATCHED		-1
+#define MAYBE_MATCHED		-2
+#define TOO_HEAVY		-3
+
+
+#define HTABLE_EMPTY    	-1
+
+#define NGR_PASSES		4	/* Number of greedy refinement passes */
+#define NIPARTS			8	/* Number of random initial partitions */
+#define NLGR_PASSES		5	/* Number of GR refinement during IPartition */
+
+#define SMALLFLOAT		0.00001
+/* #define KEEP_BIT        (idxtype)536870912    */    /* 1<<29 */
+#define KEEP_BIT        ((idxtype)(1<<((sizeof(idxtype)*8)-2)))
+
+#define MAX_PES			8192
+#define MAX_NPARTS		67108864
+
+#define COARSEN_FRACTION	0.75	/* Node reduction between succesive coarsening levels */
+#define COARSEN_FRACTION2	0.55	/* Node reduction between succesive coarsening levels */
+#define UNBALANCE_FRACTION		1.05
+#define ORDER_UNBALANCE_FRACTION	1.05
+
+#define MAXVWGT_FACTOR		1.4
+
+#define MATCH_LOCAL		1
+#define MATCH_GLOBAL		2
+
+/* Debug Levels */
+#define DBG_TIME	1		/* Perform timing analysis */
+#define DBG_INFO	2		/* Perform timing analysis */
+#define DBG_PROGRESS   	4		/* Show the coarsening progress */
+#define DBG_REFINEINFO	8		/* Show info on communication during folding */
+#define DBG_MATCHINFO	16		/* Show info on matching */
+#define DBG_RMOVEINFO	32		/* Show info on communication during folding */
+#define DBG_REMAP	64		/* Determines if remapping will take place */
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/diffutil.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/diffutil.c
new file mode 100644
index 0000000..f31da64
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/diffutil.c
@@ -0,0 +1,298 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * wavefrontK.c 
+ *
+ * This file contains code for the initial directed diffusion at the coarsest
+ * graph
+ *
+ * Started 5/19/97, Kirk, George
+ *
+ * $Id: diffutil.c,v 1.2 2003/07/21 17:18:48 karypis Exp $
+ *
+ */
+
+#include <parmetislib.h>
+
+
+/*************************************************************************
+*  This function computes the load for each subdomain
+**************************************************************************/
+void SetUpConnectGraph(GraphType *graph, MatrixType *matrix, idxtype *workspace)
+{
+  int i, ii, j, jj, k, l;
+  int nvtxs, nrows;
+  idxtype *xadj, *adjncy, *where;
+  idxtype *rowptr, *colind;
+  idxtype *pcounts, *perm, *marker;
+  float *values;
+
+  nvtxs = graph->nvtxs;
+  xadj = graph->xadj;
+  adjncy = graph->adjncy;
+  where = graph->where;
+
+  nrows = matrix->nrows;
+  rowptr = matrix->rowptr;
+  colind = matrix->colind;
+  values = matrix->values;
+
+  perm = workspace;
+  marker = idxset(nrows, -1, workspace+nvtxs);
+  pcounts = idxset(nrows+1, 0, workspace+nvtxs+nrows);
+
+  for (i=0; i<nvtxs; i++)
+    pcounts[where[i]]++;
+  MAKECSR(i, nrows, pcounts);
+
+  for (i=0; i<nvtxs; i++)
+    perm[pcounts[where[i]]++] = i;
+
+  for (i=nrows; i>0; i--)
+    pcounts[i] = pcounts[i-1];
+  pcounts[0] = 0;
+
+  /************************/
+  /* Construct the matrix */
+  /************************/
+  rowptr[0] = k = 0;
+  for (ii=0; ii<nrows; ii++) {
+    colind[k++] = ii;
+    marker[ii] = ii;
+
+    for (jj=pcounts[ii]; jj<pcounts[ii+1]; jj++) {
+      i = perm[jj];
+      for (j=xadj[i]; j<xadj[i+1]; j++) {
+        l = where[adjncy[j]];
+        if (marker[l] != ii) {
+          colind[k] = l;
+          values[k++] = -1.0;
+          marker[l] = ii;
+        }
+      }
+    }
+    values[rowptr[ii]] = (float)(k-rowptr[ii]-1);
+    rowptr[ii+1] = k;
+  }
+  matrix->nnzs = rowptr[nrows];
+
+  return;
+}
+
+
+/*************************************************************************
+* This function computes movement statistics for adaptive refinement
+* schemes
+**************************************************************************/
+void Mc_ComputeMoveStatistics(CtrlType *ctrl, GraphType *graph, int *nmoved, int *maxin, int *maxout)
+{
+  int i, nvtxs, nparts, myhome;
+  idxtype *vwgt, *where;
+  idxtype *lend, *gend, *lleft, *gleft, *lstart, *gstart;
+
+  nvtxs = graph->nvtxs;
+  vwgt = graph->vwgt;
+  where = graph->where;
+  nparts = ctrl->nparts;
+
+  lstart = idxsmalloc(nparts, 0, "ComputeMoveStatistics: lstart");
+  gstart = idxsmalloc(nparts, 0, "ComputeMoveStatistics: gstart");
+  lleft = idxsmalloc(nparts, 0, "ComputeMoveStatistics: lleft");
+  gleft = idxsmalloc(nparts, 0, "ComputeMoveStatistics: gleft");
+  lend = idxsmalloc(nparts, 0, "ComputeMoveStatistics: lend");
+  gend = idxsmalloc(nparts, 0, "ComputeMoveStatistics: gend");
+
+  for (i=0; i<nvtxs; i++) {
+    myhome = (ctrl->ps_relation == COUPLED) ? ctrl->mype : graph->home[i];
+    lstart[myhome] += (graph->vsize == NULL) ? 1 : graph->vsize[i];
+    lend[where[i]] += (graph->vsize == NULL) ? 1 : graph->vsize[i];
+    if (where[i] != myhome)
+      lleft[myhome] += (graph->vsize == NULL) ? 1 : graph->vsize[i];
+  }
+
+  /* PrintVector(ctrl, ctrl->npes, 0, lend, "Lend: "); */
+
+  MPI_Allreduce((void *)lstart, (void *)gstart, nparts, IDX_DATATYPE, MPI_SUM, ctrl->comm);
+  MPI_Allreduce((void *)lleft, (void *)gleft, nparts, IDX_DATATYPE, MPI_SUM, ctrl->comm);
+  MPI_Allreduce((void *)lend, (void *)gend, nparts, IDX_DATATYPE, MPI_SUM, ctrl->comm);
+
+  *nmoved = idxsum(nparts, gleft);
+  *maxout = gleft[idxamax(nparts, gleft)];
+  for (i=0; i<nparts; i++)
+    lstart[i] = gend[i]+gleft[i]-gstart[i];
+  *maxin = lstart[idxamax(nparts, lstart)];
+
+  GKfree((void **)&lstart, (void **)&gstart, (void **)&lleft, (void **)&gleft, (void **)&lend, (void **)&gend, LTERM);
+}
+
+/*************************************************************************
+*  This function computes the TotalV of a serial graph.
+**************************************************************************/
+int Mc_ComputeSerialTotalV(GraphType *graph, idxtype *home)
+{
+  int i;
+  int totalv = 0;
+
+  for (i=0; i<graph->nvtxs; i++) {
+    if (graph->where[i] != home[i])
+      totalv += (graph->vsize == NULL) ? graph->vwgt[i*graph->ncon] : graph->vsize[i];
+  }
+
+  return totalv;
+}
+
+
+
+/*************************************************************************
+*  This function computes the load for each subdomain
+**************************************************************************/
+void ComputeLoad(GraphType *graph, int nparts, float *load, float *tpwgts, int index)
+{
+  int i;
+  int nvtxs, ncon;
+  idxtype *where;
+  float *nvwgt;
+
+  nvtxs = graph->nvtxs;
+  ncon = graph->ncon;
+  where = graph->where;
+  nvwgt = graph->nvwgt;
+
+  sset(nparts, 0.0, load);
+
+  for (i=0; i<nvtxs; i++)
+    load[where[i]] += nvwgt[i*ncon+index];
+
+  ASSERTS(fabs(ssum(nparts, load)-1.0) < 0.001);
+
+  for (i=0; i<nparts; i++) {
+    load[i] -= tpwgts[i*ncon+index];
+  }
+
+  return;
+}
+
+
+/*************************************************************************
+* This function implements the CG solver used during the directed diffusion
+**************************************************************************/
+void ConjGrad2(MatrixType *A, float *b, float *x, float tol, float *workspace)
+{
+  int i, k, n;
+  float *p, *r, *q, *z, *M;
+  float alpha, beta, rho, rho_1 = -1.0, error, bnrm2, tmp;
+  idxtype *rowptr, *colind;
+  float *values;
+
+  n = A->nrows;
+  rowptr = A->rowptr;
+  colind = A->colind;
+  values = A->values;
+
+  /* Initial Setup */
+  p = workspace;
+  r = workspace + n;
+  q = workspace + 2*n;
+  z = workspace + 3*n;
+  M = workspace + 4*n;
+
+  for (i=0; i<n; i++) {
+    x[i] = 0.0;
+    if (values[rowptr[i]] != 0.0)
+      M[i] = 1.0/values[rowptr[i]];
+    else
+      M[i] = 0.0;
+  }
+
+  /* r = b - Ax */
+  mvMult2(A, x, r);
+  for (i=0; i<n; i++)
+    r[i] = b[i]-r[i];
+
+  bnrm2 = snorm2(n, b);
+  if (bnrm2 > 0.0) {
+    error = snorm2(n, r) / bnrm2;
+
+    if (error > tol) {
+      /* Begin Iterations */
+      for (k=0; k<n; k++) {
+        for (i=0; i<n; i++)
+          z[i] = r[i]*M[i];
+
+        rho = sdot(n, r, z);
+
+        if (k == 0)
+          scopy(n, z, p);
+        else {
+          if (rho_1 != 0.0)
+            beta = rho/rho_1;
+          else
+            beta = 0.0;
+          for (i=0; i<n; i++)
+            p[i] = z[i] + beta*p[i];
+        }
+
+        mvMult2(A, p, q); /* q = A*p */
+
+        tmp = sdot(n, p, q);
+        if (tmp != 0.0)
+          alpha = rho/tmp;
+        else
+          alpha = 0.0;
+        saxpy(n, alpha, p, x);    /* x = x + alpha*p */
+        saxpy(n, -alpha, q, r);   /* r = r - alpha*q */
+        error = snorm2(n, r) / bnrm2;
+        if (error < tol)
+          break;
+
+        rho_1 = rho;
+      }
+    }
+  }
+}
+
+
+/*************************************************************************
+* This function performs Matrix-Vector multiplication
+**************************************************************************/
+void mvMult2(MatrixType *A, float *v, float *w)
+{
+  int i, j;
+
+  for (i = 0; i < A->nrows; i++)
+    w[i] = 0.0;
+
+  for (i = 0; i < A->nrows; i++)
+    for (j = A->rowptr[i]; j < A->rowptr[i+1]; j++)
+      w[i] += A->values[j] * v[A->colind[j]];
+
+  return;
+  }
+
+
+/*************************************************************************
+* This function sets up the transfer vectors
+**************************************************************************/
+void ComputeTransferVector(int ncon, MatrixType *matrix, float *solution,
+  float *transfer, int index)
+{
+  int j, k;
+  int nrows;
+  idxtype *rowptr, *colind;
+
+  nrows = matrix->nrows;
+  rowptr = matrix->rowptr;
+  colind = matrix->colind;
+
+  for (j=0; j<nrows; j++) {
+    for (k=rowptr[j]+1; k<rowptr[j+1]; k++) {
+      if (solution[j] > solution[colind[k]]) {
+        transfer[k*ncon+index] = solution[j] - solution[colind[k]];
+      }
+      else {
+        transfer[k*ncon+index] = 0.0;
+      }
+    }
+  }
+}
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/fpqueue.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/fpqueue.c
new file mode 100644
index 0000000..11c617f
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/fpqueue.c
@@ -0,0 +1,440 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * fpqueue.c
+ *
+ * This file contains functions for manipulating the bucket list
+ * representation of the gains associated with each vertex in a graph.
+ * These functions are used by the refinement algorithms
+ *
+ * Started 9/2/94
+ * George
+ *
+ * $Id: fpqueue.c,v 1.2 2003/07/21 17:18:48 karypis Exp $
+ *
+ */
+
+#include <parmetislib.h>
+
+
+/*************************************************************************
+* This function initializes the data structures of the priority queue
+**************************************************************************/
+void FPQueueInit(FPQueueType *queue, int maxnodes)
+{
+  queue->nnodes = 0;
+  queue->maxnodes = maxnodes;
+  queue->heap = NULL;
+  queue->locator = NULL;
+
+  queue->heap = (FKeyValueType *) malloc(sizeof(FKeyValueType)*maxnodes);
+  queue->locator = (idxtype *) malloc(sizeof(idxtype)*maxnodes);
+
+  idxset(maxnodes, -1, queue->locator);
+
+}
+
+
+/*************************************************************************
+* This function resets the buckets
+**************************************************************************/
+void FPQueueReset(FPQueueType *queue)
+{
+  queue->nnodes = 0;
+
+  idxset(queue->maxnodes, -1, queue->locator);
+
+}
+
+
+/*************************************************************************
+* This function frees the buckets
+**************************************************************************/
+void FPQueueFree(FPQueueType *queue)
+{
+
+  free(queue->heap);
+  free(queue->locator);
+
+  queue->maxnodes = 0;
+}
+
+
+/*************************************************************************
+* This function returns the number of nodes in the queue
+**************************************************************************/
+int FPQueueGetSize(FPQueueType *queue)
+{
+  return queue->nnodes;
+}
+
+
+/*************************************************************************
+* This function adds a node of certain gain into a partition
+**************************************************************************/
+int FPQueueInsert(FPQueueType *queue, int node, float gain)
+{
+  int i, j;
+  idxtype *locator;
+  FKeyValueType *heap;
+
+  ASSERTS(CheckHeapFloat(queue));
+
+  heap = queue->heap;
+  locator = queue->locator;
+
+  ASSERTS(locator[node] == -1);
+
+  i = queue->nnodes++;
+  while (i > 0) {
+    j = (i-1)/2;
+    if (heap[j].key < gain) {
+      heap[i] = heap[j];
+      locator[heap[i].val] = i;
+      i = j;
+    }
+    else 
+      break;
+  }
+  ASSERTS(i >= 0);
+  heap[i].key = gain;
+  heap[i].val = node;
+  locator[node] = i;
+
+  ASSERTS(CheckHeapFloat(queue));
+
+  return 0;
+}
+
+
+/*************************************************************************
+* This function deletes a node from a partition and reinserts it with
+* an updated gain
+**************************************************************************/
+int FPQueueDelete(FPQueueType *queue, int node)
+{
+  int i, j;
+  float newgain, oldgain;
+  idxtype *locator;
+  FKeyValueType *heap;
+
+  heap = queue->heap;
+  locator = queue->locator;
+
+  ASSERTS(locator[node] != -1);
+  ASSERTS(heap[locator[node]].val == node);
+
+  ASSERTS(CheckHeapFloat(queue));
+
+  i = locator[node];
+  locator[node] = -1;
+
+  if (--queue->nnodes > 0 && heap[queue->nnodes].val != node) {
+    node = heap[queue->nnodes].val;
+    newgain = heap[queue->nnodes].key;
+    oldgain = heap[i].key;
+
+    if (oldgain < newgain) {
+      /* Filter-up */
+      while (i > 0) {
+        j = (i-1)>>1;
+        if (heap[j].key < newgain) {
+          heap[i] = heap[j];
+          locator[heap[i].val] = i;
+          i = j;
+        }
+        else 
+          break;
+      }
+    }
+    else {
+      /* Filter down */
+      while ((j=2*i+1) < queue->nnodes) {
+        if (heap[j].key > newgain) {
+          if (j+1 < queue->nnodes && heap[j+1].key > heap[j].key)
+            j = j+1;
+          heap[i] = heap[j];
+          locator[heap[i].val] = i;
+          i = j;
+        }
+        else if (j+1 < queue->nnodes && heap[j+1].key > newgain) {
+          j = j+1;
+          heap[i] = heap[j];
+          locator[heap[i].val] = i;
+          i = j;
+        }
+        else
+          break;
+      }
+    }
+
+    heap[i].key = newgain;
+    heap[i].val = node;
+    locator[node] = i;
+  }
+
+  ASSERTS(CheckHeapFloat(queue));
+
+  return 0;
+}
+
+
+
+/*************************************************************************
+* This function deletes a node from a partition and reinserts it with
+* an updated gain
+**************************************************************************/
+int FPQueueUpdate(FPQueueType *queue, int node, float oldgain, float newgain)
+{
+  int i, j;
+  idxtype *locator;
+  FKeyValueType *heap;
+
+  if (oldgain == newgain) 
+    return 0;
+
+  heap = queue->heap;
+  locator = queue->locator;
+
+  ASSERTS(locator[node] != -1);
+  ASSERTS(heap[locator[node]].val == node);
+  ASSERTS(fabs(heap[locator[node]].key - oldgain) < SMALLFLOAT);
+  ASSERTS(CheckHeapFloat(queue));
+
+  i = locator[node];
+
+  if (oldgain < newgain) {
+    /* Filter-up */
+    while (i > 0) {
+      j = (i-1)>>1;
+      if (heap[j].key < newgain) {
+        heap[i] = heap[j];
+        locator[heap[i].val] = i;
+        i = j;
+      }
+      else 
+        break;
+    }
+  }
+  else {
+    /* Filter down */
+    while ((j=2*i+1) < queue->nnodes) {
+      if (heap[j].key > newgain) {
+        if (j+1 < queue->nnodes && heap[j+1].key > heap[j].key)
+          j = j+1;
+        heap[i] = heap[j];
+        locator[heap[i].val] = i;
+        i = j;
+      }
+      else if (j+1 < queue->nnodes && heap[j+1].key > newgain) {
+        j = j+1;
+        heap[i] = heap[j];
+        locator[heap[i].val] = i;
+        i = j;
+      }
+      else
+        break;
+    }
+  }
+
+  heap[i].key = newgain;
+  heap[i].val = node;
+  locator[node] = i;
+
+  ASSERTS(CheckHeapFloat(queue));
+
+  return 0;
+}
+
+
+
+/*************************************************************************
+* This function deletes a node from a partition and reinserts it with
+* an updated gain
+**************************************************************************/
+void FPQueueUpdateUp(FPQueueType *queue, int node, float oldgain, float newgain)
+{
+  int i, j;
+  idxtype *locator;
+  FKeyValueType *heap;
+
+  if (oldgain == newgain) 
+    return;
+
+  heap = queue->heap;
+  locator = queue->locator;
+
+  ASSERTS(locator[node] != -1);
+  ASSERTS(heap[locator[node]].val == node);
+  ASSERTS(heap[locator[node]].key == oldgain);
+  ASSERTS(CheckHeapFloat(queue));
+
+
+  /* Here we are just filtering up since the newgain is greater than the oldgain */
+  i = locator[node];
+  while (i > 0) {
+    j = (i-1)>>1;
+    if (heap[j].key < newgain) {
+      heap[i] = heap[j];
+      locator[heap[i].val] = i;
+      i = j;
+    }
+    else 
+      break;
+  }
+
+  heap[i].key = newgain;
+  heap[i].val = node;
+  locator[node] = i;
+
+  ASSERTS(CheckHeapFloat(queue));
+
+}
+
+
+/*************************************************************************
+* This function returns the vertex with the largest gain from a partition
+* and removes the node from the bucket list
+**************************************************************************/
+int FPQueueGetMax(FPQueueType *queue)
+{
+  int vtx, i, j, node;
+  float gain;
+  idxtype *locator;
+  FKeyValueType *heap;
+
+  if (queue->nnodes == 0)
+    return -1;
+
+  queue->nnodes--;
+
+  heap = queue->heap;
+  locator = queue->locator;
+
+  vtx = heap[0].val;
+  locator[vtx] = -1;
+
+  if ((i = queue->nnodes) > 0) {
+    gain = heap[i].key;
+    node = heap[i].val;
+    i = 0;
+    while ((j=2*i+1) < queue->nnodes) {
+      if (heap[j].key > gain) {
+        if (j+1 < queue->nnodes && heap[j+1].key > heap[j].key)
+          j = j+1;
+        heap[i] = heap[j];
+        locator[heap[i].val] = i;
+        i = j;
+      }
+      else if (j+1 < queue->nnodes && heap[j+1].key > gain) {
+        j = j+1;
+        heap[i] = heap[j];
+        locator[heap[i].val] = i;
+        i = j;
+      }
+      else
+        break;
+    }
+
+    heap[i].key = gain;
+    heap[i].val = node;
+    locator[node] = i;
+  }
+
+  ASSERTS(CheckHeapFloat(queue));
+  return vtx;
+}
+      
+
+/*************************************************************************
+* This function returns the vertex with the largest gain from a partition
+**************************************************************************/
+int FPQueueSeeMaxVtx(FPQueueType *queue)
+{
+  int vtx;
+
+  if (queue->nnodes == 0)
+    return -1;
+
+  vtx = queue->heap[0].val;
+
+  return vtx;
+}
+      
+
+/*************************************************************************
+* This function returns the vertex with the largest gain from a partition
+**************************************************************************/
+float FPQueueSeeMaxGain(FPQueueType *queue)
+{
+  float gain;
+
+  if (queue->nnodes == 0)
+    return 0.0;
+
+  gain = queue->heap[0].key;
+
+  return gain;
+}
+
+
+/*************************************************************************
+* This function returns the vertex with the largest gain from a partition
+**************************************************************************/
+float FPQueueGetKey(FPQueueType *queue)
+{
+  int key;
+
+  if (queue->nnodes == 0)
+    return -1;
+
+  key = queue->heap[0].key;
+
+  return key;
+}
+      
+/*************************************************************************
+* This function returns the number of nodes in the queue
+**************************************************************************/
+int FPQueueGetQSize(FPQueueType *queue)
+{
+  return queue->nnodes;
+}
+
+
+
+
+
+
+/*************************************************************************
+* This functions checks the consistency of the heap
+**************************************************************************/
+int CheckHeapFloat(FPQueueType *queue)
+{
+  int i, j, nnodes;
+  idxtype *locator;
+  FKeyValueType *heap;
+
+  heap = queue->heap;
+  locator = queue->locator;
+  nnodes = queue->nnodes;
+
+  if (nnodes == 0)
+    return 1;
+
+  ASSERTS(locator[heap[0].val] == 0);
+  for (i=1; i<nnodes; i++) {
+    ASSERTS(locator[heap[i].val] == i);
+    ASSERTS(heap[i].key <= heap[(i-1)/2].key);
+  }
+  for (i=1; i<nnodes; i++)
+    ASSERTS(heap[i].key <= heap[0].key);
+
+  for (j=i=0; i<queue->maxnodes; i++) {
+    if (locator[i] != -1)
+      j++;
+  }
+  ASSERTS(j == nnodes);
+
+  return 1;
+}
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/frename.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/frename.c
new file mode 100644
index 0000000..551dd59
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/frename.c
@@ -0,0 +1,322 @@
+/*
+ * frename.c
+ *
+ * This file contains some renaming routines to deal with different
+ * Fortran compilers.
+ *
+ * Started 6/1/98
+ * George
+ *
+ * $Id: frename.c,v 1.4 2003/07/30 18:37:58 karypis Exp $
+ *
+ */
+
+#include <parmetislib.h>
+
+
+
+/*************************************************************************
+* Renaming macro (at least to save some typing :))  
+**************************************************************************/
+#define FRENAME(name0, name1, name2, name3, name4, dargs, cargs)   \
+  void name1 dargs { name0 cargs; }                          \
+  void name2 dargs { name0 cargs; }                          \
+  void name3 dargs { name0 cargs; }                          \
+  void name4 dargs { name0 cargs; }
+
+
+
+
+
+
+
+
+/*************************************************************************
+* Renames for Release 3.0 API
+**************************************************************************/
+FRENAME(ParMETIS_V3_AdaptiveRepart, 
+        PARMETIS_V3_ADAPTIVEREPART,
+	parmetis_v3_adaptiverepart,
+	parmetis_v3_adaptiverepart_,
+	parmetis_v3_adaptiverepart__,
+	(idxtype *vtxdist, idxtype *xadj, idxtype *adjncy, idxtype *vwgt,
+	 idxtype *vsize, idxtype *adjwgt, int *wgtflag, int *numflag, int *ncon,
+	 int *nparts, float *tpwgts, float *ubvec, float *ipc2redist,
+	 int *options, int *edgecut, idxtype *part, MPI_Comm *comm),
+	(vtxdist, xadj, adjncy, vwgt, vsize, adjwgt, wgtflag, numflag, ncon,
+	 nparts, tpwgts, ubvec, ipc2redist, options, edgecut, part, comm)
+)
+
+FRENAME(ParMETIS_V3_PartGeomKway,
+        PARMETIS_V3_PARTGEOMKWAY,
+	parmetis_v3_partgeomkway,
+	parmetis_v3_partgeomkway_,
+	parmetis_v3_partgeomkway__,
+        (idxtype *vtxdist, idxtype *xadj, idxtype *adjncy, idxtype *vwgt, 
+	 idxtype *adjwgt, int *wgtflag, int *numflag, int *ndims, float *xyz, 
+	 int *ncon, int *nparts, float *tpwgts, float *ubvec, int *options, 
+	 int *edgecut, idxtype *part, MPI_Comm *comm),
+        (vtxdist, xadj, adjncy, vwgt, adjwgt, wgtflag, numflag, ndims, xyz, 
+	 ncon, nparts, tpwgts, ubvec, options, edgecut, part, comm)
+)
+	 
+FRENAME(ParMETIS_V3_PartGeom,
+        PARMETIS_V3_PARTGEOM,
+	parmetis_v3_partgeom,
+	parmetis_v3_partgeom_,
+	parmetis_v3_partgeom__,
+	(idxtype *vtxdist, int *ndims, float *xyz, idxtype *part, MPI_Comm *comm),
+	(vtxdist, ndims, xyz, part, comm)
+)
+
+FRENAME(ParMETIS_V3_PartKway,
+        PARMETIS_V3_PARTKWAY,
+	parmetis_v3_partkway,
+	parmetis_v3_partkway_,
+	parmetis_v3_partkway__,
+	(idxtype *vtxdist, idxtype *xadj, idxtype *adjncy, idxtype *vwgt, idxtype *adjwgt, 
+	 int *wgtflag, int *numflag, int *ncon, int *nparts, float *tpwgts, float *ubvec, 
+	 int *options, int *edgecut, idxtype *part, MPI_Comm *comm),
+	(vtxdist, xadj, adjncy, vwgt, adjwgt, wgtflag, numflag, ncon, nparts, tpwgts, 
+	 ubvec, options, edgecut, part, comm)
+)
+
+FRENAME(ParMETIS_V3_Mesh2Dual,
+        PARMETIS_V3_MESH2DUAL,
+	parmetis_v3_mesh2dual,
+	parmetis_v3_mesh2dual_,
+	parmetis_v3_mesh2dual__,
+	(idxtype *elmdist, idxtype *eptr, idxtype *eind, int *numflag, int *ncommonnodes, 
+	 idxtype **xadj, idxtype **adjncy, MPI_Comm *comm),
+	(elmdist, eptr, eind, numflag, ncommonnodes, xadj, adjncy, comm)
+)
+
+FRENAME(ParMETIS_V3_PartMeshKway,
+        PARMETIS_V3_PARTMESHKWAY, 
+	parmetis_v3_partmeshkway,
+	parmetis_v3_partmeshkway_,
+	parmetis_v3_partmeshkway__,
+	(idxtype *elmdist, idxtype *eptr, idxtype *eind, idxtype *elmwgt, int *wgtflag, 
+	 int *numflag, int *ncon, int *ncommonnodes, int *nparts, float *tpwgts, 
+	 float *ubvec, int *options, int *edgecut, idxtype *part, MPI_Comm *comm),
+	(elmdist, eptr, eind, elmwgt, wgtflag, numflag, ncon, ncommonnodes, nparts, tpwgts, 
+	 ubvec, options, edgecut, part, comm)
+)
+	 
+FRENAME(ParMETIS_V3_NodeND,
+        PARMETIS_V3_NODEND,
+        parmetis_v3_nodend,
+        parmetis_v3_nodend_,
+        parmetis_v3_nodend__,
+        (idxtype *vtxdist, idxtype *xadj, idxtype *adjncy, int *numflag, int *options, 
+	 idxtype *order, idxtype *sizes, MPI_Comm *comm),
+        (vtxdist, xadj, adjncy, numflag, options, order, sizes, comm)
+)
+
+FRENAME(ParMETIS_V3_RefineKway,
+        PARMETIS_V3_REFINEKWAY,
+        parmetis_v3_refinekway,
+        parmetis_v3_refinekway_,
+        parmetis_v3_refinekway__,
+        (idxtype *vtxdist, idxtype *xadj, idxtype *adjncy, idxtype *vwgt, idxtype *adjwgt, 
+	 int *wgtflag, int *numflag, int *ncon, int *nparts, float *tpwgts, float *ubvec, 
+	 int *options, int *edgecut, idxtype *part, MPI_Comm *comm),
+        (vtxdist, xadj, adjncy, vwgt, adjwgt, wgtflag, numflag, ncon, nparts, tpwgts, 
+	 ubvec, options, edgecut, part, comm)
+)
+
+
+/*************************************************************************
+* Renames for Release 2.0 API
+**************************************************************************/
+FRENAME(ParMETIS_PartKway,
+        PARMETIS_PARTKWAY,
+	parmetis_partkway,
+	parmetis_partkway_,
+	parmetis_partkway__,
+        (idxtype *vtxdist, idxtype *xadj, idxtype *adjncy, idxtype *vwgt, idxtype *adjwgt, 
+	 int *wgtflag, int *numflag, int *nparts, int *options, int *edgecut, idxtype *part, 
+	 MPI_Comm *comm),
+        (vtxdist, xadj, adjncy, vwgt, adjwgt, wgtflag, numflag, nparts, options, edgecut, 
+	 part, comm)
+)
+	 
+FRENAME(ParMETIS_PartGeomKway,
+        PARMETIS_PARTGEOMKWAY,
+        parmetis_partgeomkway,
+        parmetis_partgeomkway_,
+        parmetis_partgeomkway__,
+        (idxtype *vtxdist, idxtype *xadj, idxtype *adjncy, idxtype *vwgt, idxtype *adjwgt, 
+	 int *wgtflag, int *numflag, int *ndims, float *xyz, int *nparts, int *options, 
+	 int *edgecut, idxtype *part, MPI_Comm *comm),
+        (vtxdist, xadj, adjncy, vwgt, adjwgt, wgtflag, numflag, ndims, xyz, nparts, options, 
+	 edgecut, part, comm)
+) 
+	 
+FRENAME(ParMETIS_PartGeom,
+        PARMETIS_PARTGEOM,
+	parmetis_partgeom,
+	parmetis_partgeom_,
+	parmetis_partgeom__,
+	(idxtype *vtxdist, int *ndims, float *xyz, idxtype *part, MPI_Comm *comm),
+	(vtxdist, ndims, xyz, part, comm)
+)
+
+FRENAME(ParMETIS_PartGeomRefine,
+        PARMETIS_PARTGEOMREFINE,
+	parmetis_partgeomrefine,
+	parmetis_partgeomrefine_,
+	parmetis_partgeomrefine__,
+        (idxtype *vtxdist, idxtype *xadj, idxtype *adjncy, idxtype *vwgt, idxtype *adjwgt, 
+	 int *wgtflag, int *numflag, int *ndims, float *xyz, int *options, int *edgecut, 
+	 idxtype *part, MPI_Comm *comm),
+        (vtxdist, xadj, adjncy, vwgt, adjwgt, wgtflag, numflag, ndims, xyz, options, 
+	 edgecut, part, comm)
+)
+
+FRENAME(ParMETIS_RefineKway,
+        PARMETIS_REFINEKWAY,
+	parmetis_refinekway,
+	parmetis_refinekway_,
+	parmetis_refinekway__,
+        (idxtype *vtxdist, idxtype *xadj, idxtype *adjncy, idxtype *vwgt, idxtype *adjwgt, 
+	 int *wgtflag, int *numflag, int *options, int *edgecut, idxtype *part, MPI_Comm *comm),
+        (vtxdist, xadj, adjncy, vwgt, adjwgt, wgtflag, numflag, options, edgecut, part, comm)
+)
+
+FRENAME(ParMETIS_RepartLDiffusion,
+        PARMETIS_REPARTLDIFUSSION,
+	parmetis_repartldiffusion,
+	parmetis_repartldiffusion_,
+	parmetis_repartldiffusion__,
+	(idxtype *vtxdist, idxtype *xadj, idxtype *adjncy, idxtype *vwgt, idxtype *adjwgt, 
+	 int *wgtflag, int *numflag, int *options, int *edgecut, idxtype *part, MPI_Comm *comm),
+	(vtxdist, xadj, adjncy, vwgt, adjwgt, wgtflag, numflag, options, edgecut, part, comm)
+)
+
+FRENAME(ParMETIS_RepartGDiffusion,
+        PARMETIS_REPARTGDIFFUSION,
+        parmetis_repartgdiffusion,
+        parmetis_repartgdiffusion_,
+        parmetis_repartgdiffusion__,
+	(idxtype *vtxdist, idxtype *xadj, idxtype *adjncy, idxtype *vwgt, idxtype *adjwgt, 
+	 int *wgtflag, int *numflag, int *options, int *edgecut, idxtype *part, MPI_Comm *comm),
+	(vtxdist, xadj, adjncy, vwgt, adjwgt, wgtflag, numflag, options, edgecut, part, comm)
+)
+
+FRENAME(ParMETIS_RepartRemap,
+        PARMETIS_REPARTREMAP,
+        parmetis_repartremap,
+        parmetis_repartremap_,
+        parmetis_repartremap__,
+        (idxtype *vtxdist, idxtype *xadj, idxtype *adjncy, idxtype *vwgt, idxtype *adjwgt, 
+	 int *wgtflag, int *numflag, int *options, int *edgecut, idxtype *part, MPI_Comm *comm),
+        (vtxdist, xadj, adjncy, vwgt, adjwgt, wgtflag, numflag, options, edgecut, part, comm)
+)
+
+FRENAME(ParMETIS_RepartMLRemap,
+        PARMETIS_REPARTMLREMAP,
+        parmetis_repartmlremap,
+        parmetis_repartmlremap_,
+        parmetis_repartmlremap__,
+        (idxtype *vtxdist, idxtype *xadj, idxtype *adjncy, idxtype *vwgt, idxtype *adjwgt, 
+	 int *wgtflag, int *numflag, int *options, int *edgecut, idxtype *part, MPI_Comm *comm),
+        (vtxdist, xadj, adjncy, vwgt, adjwgt, wgtflag, numflag, options, edgecut, part, comm)
+)
+
+FRENAME(ParMETIS_NodeND,
+        PARMETIS_NODEND,
+        parmetis_nodend,
+        parmetis_nodend_,
+        parmetis_nodend__,
+	(idxtype *vtxdist, idxtype *xadj, idxtype *adjncy, int *numflag, int *options, 
+	 idxtype *order, idxtype *sizes, MPI_Comm *comm),
+	(vtxdist, xadj, adjncy, numflag, options, order, sizes, comm)
+)
+
+FRENAME(ParMETIS_SerialNodeND,
+        PARMETIS_SERIALNODEND,
+	parmetis_serialnodend,
+	parmetis_serialnodend_,
+	parmetis_serialnodend__,
+	(idxtype *vtxdist, idxtype *xadj, idxtype *adjncy, int *numflag, int *options, 
+	 idxtype *order, idxtype *sizes, MPI_Comm *comm),
+	(vtxdist, xadj, adjncy, numflag, options, order, sizes, comm)
+)
+
+
+
+
+/*************************************************************************
+* Renames for Release 1.0 API
+**************************************************************************/
+FRENAME(PARKMETIS,
+        PARKMETIS_,
+	parkmetis,
+	parkmetis_,
+	parkmetis__,
+	(idxtype *vtxdist, idxtype *xadj, idxtype *vwgt, idxtype *adjncy, idxtype *adjwgt, 
+	 idxtype *part, int *options, MPI_Comm comm),
+	(vtxdist, xadj, vwgt, adjncy, adjwgt, part, options, comm)
+)
+
+FRENAME(PARGKMETIS,
+        PARGKMETIS_,
+        pargkmetis,
+        pargkmetis_,
+        pargkmetis__,
+        (idxtype *vtxdist, idxtype *xadj, idxtype *vwgt, idxtype *adjncy, idxtype *adjwgt,
+         int ndims, float *xyz, idxtype *part, int *options, MPI_Comm comm),
+        (vtxdist, xadj, vwgt, adjncy, adjwgt, ndims, xyz, part, options, comm)
+)	
+
+FRENAME(PARGRMETIS,
+        PARGRMETIS_,
+	pargrmetis,
+	pargrmetis_,
+	pargrmetis__,
+        (idxtype *vtxdist, idxtype *xadj, idxtype *vwgt, idxtype *adjncy, idxtype *adjwgt,
+         int ndims, float *xyz, idxtype *part, int *options, MPI_Comm comm),
+        (vtxdist, xadj, vwgt, adjncy, adjwgt, ndims, xyz, part, options, comm)
+)
+
+FRENAME(PARGMETIS,
+        PARGMETIS_,
+	pargmetis,
+	pargmetis_,
+	pargmetis__,
+	(idxtype *vtxdist, idxtype *xadj, idxtype *adjncy, int ndims, float *xyz,
+         idxtype *part, int *options, MPI_Comm comm),
+	(vtxdist, xadj, adjncy, ndims, xyz, part, options, comm)
+)
+
+FRENAME(PARRMETIS,
+        PARRMETIS_,
+	parrmetis,
+	parrmetis_,
+	parrmetis__,
+	(idxtype *vtxdist, idxtype *xadj, idxtype *vwgt, idxtype *adjncy, idxtype *adjwgt, 
+	 idxtype *part, int *options, MPI_Comm comm),
+	(vtxdist, xadj, vwgt, adjncy, adjwgt, part, options, comm)
+)
+
+FRENAME(PARUAMETIS,
+        PARUAMETIS_,
+	paruametis,
+	paruametis_,
+	paruametis__,
+	(idxtype *vtxdist, idxtype *xadj, idxtype *vwgt, idxtype *adjncy, idxtype *adjwgt, 
+	 idxtype *part, int *options, MPI_Comm comm),
+	(vtxdist, xadj, vwgt, adjncy, adjwgt, part, options, comm)
+)
+
+FRENAME(PARDAMETIS,
+        PARDAMETIS_,
+	pardametis,
+	pardametis_,
+	pardametis__,
+        (idxtype *vtxdist, idxtype *xadj, idxtype *vwgt, idxtype *adjncy, idxtype *adjwgt,
+         idxtype *part, int *options, MPI_Comm comm),
+        (vtxdist, xadj, vwgt, adjncy, adjwgt, part, options, comm)
+)
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/gkmetis.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/gkmetis.c
new file mode 100644
index 0000000..8eb2382
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/gkmetis.c
@@ -0,0 +1,331 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * gkmetis.c
+ *
+ * This is the entry point of parallel geometry based partitioning
+ * routines
+ *
+ * Started 10/19/96
+ * George
+ *
+ * $Id: gkmetis.c,v 1.8 2003/07/31 16:23:30 karypis Exp $
+ *
+ */
+
+#include <parmetislib.h>
+
+
+
+
+/***********************************************************************************
+* This function is the entry point of the parallel kmetis algorithm that uses
+* coordinates to compute an initial graph distribution.
+************************************************************************************/
+void ParMETIS_V3_PartGeomKway(idxtype *vtxdist, idxtype *xadj, idxtype *adjncy,
+              idxtype *vwgt, idxtype *adjwgt, int *wgtflag, int *numflag, int *ndims, 
+	      float *xyz, int *ncon, int *nparts, float *tpwgts, float *ubvec, 
+	      int *options, int *edgecut, idxtype *part, MPI_Comm *comm)
+{
+  int h, i, j;
+  int nvtxs = -1, npes, mype;
+  int uwgtflag, cut, gcut, maxnvtxs;
+  int ltvwgts[MAXNCON];
+  int moptions[10];
+  CtrlType ctrl;
+  idxtype *uvwgt;
+  WorkSpaceType wspace;
+  GraphType *graph, *mgraph;
+  float avg, maximb, balance, *mytpwgts;
+  int seed, dbglvl = 0;
+  int iwgtflag, inumflag, incon, inparts, ioptions[10];
+  float *itpwgts, iubvec[MAXNCON];
+
+  MPI_Comm_size(*comm, &npes);
+  MPI_Comm_rank(*comm, &mype);
+
+  /********************************/
+  /* Try and take care bad inputs */
+  /********************************/
+  if (options != NULL && options[0] == 1)
+    dbglvl = options[PMV3_OPTION_DBGLVL];
+
+  CheckInputs(STATIC_PARTITION, npes, dbglvl, wgtflag, &iwgtflag, numflag, &inumflag,
+              ncon, &incon, nparts, &inparts, tpwgts, &itpwgts, ubvec, iubvec, 
+	      NULL, NULL, options, ioptions, part, comm);
+
+
+  /*********************************/
+  /* Take care the nparts = 1 case */
+  /*********************************/
+  if (inparts <= 1) {
+    idxset(vtxdist[mype+1]-vtxdist[mype], 0, part);
+    *edgecut = 0;
+    return;
+  }
+
+  /******************************/
+  /* Take care of npes = 1 case */
+  /******************************/
+  if (npes == 1 && inparts > 1) {
+    moptions[0] = 0;
+    nvtxs = vtxdist[1];
+
+    if (incon == 1) {
+      METIS_WPartGraphKway(&nvtxs, xadj, adjncy, vwgt, adjwgt, &iwgtflag, &inumflag, 
+            &inparts, itpwgts, moptions, edgecut, part);
+    }
+    else {
+      /* ADD: this is because METIS does not support tpwgts for all constraints */
+      mytpwgts = fmalloc(inparts, "mytpwgts");
+      for (i=0; i<inparts; i++)
+        mytpwgts[i] = itpwgts[i*incon];
+
+      moptions[7] = -1;
+      METIS_mCPartGraphRecursive2(&nvtxs, &incon, xadj, adjncy, vwgt, adjwgt, &iwgtflag, 
+            &inumflag, &inparts, mytpwgts, moptions, edgecut, part);
+
+      free(mytpwgts);
+    }
+
+    return;
+  }
+
+
+  if (inumflag == 1)
+    ChangeNumbering(vtxdist, xadj, adjncy, part, npes, mype, 1);
+
+  /*****************************/
+  /* Set up control structures */
+  /*****************************/
+  if (ioptions[0] == 1) {
+    dbglvl = ioptions[PMV3_OPTION_DBGLVL];
+    seed = ioptions[PMV3_OPTION_SEED];
+  }
+  else {
+    dbglvl = GLOBAL_DBGLVL;
+    seed = GLOBAL_SEED;
+  }
+  SetUpCtrl(&ctrl, npes, dbglvl, *comm);
+  ctrl.CoarsenTo = amin(vtxdist[npes]+1, 25*incon*amax(npes, inparts));
+  ctrl.seed = (seed == 0) ? mype : seed*mype;
+  ctrl.sync = GlobalSEMax(&ctrl, seed);
+  ctrl.partType = STATIC_PARTITION;
+  ctrl.ps_relation = -1;
+  ctrl.tpwgts = itpwgts;
+  scopy(incon, iubvec, ctrl.ubvec);
+
+  uwgtflag = iwgtflag|2;
+  uvwgt = idxsmalloc(vtxdist[mype+1]-vtxdist[mype], 1, "uvwgt");
+  graph = Moc_SetUpGraph(&ctrl, 1, vtxdist, xadj, uvwgt, adjncy, adjwgt, &uwgtflag);
+  free(graph->nvwgt); graph->nvwgt = NULL;
+
+  PreAllocateMemory(&ctrl, graph, &wspace);
+
+  /*=================================================================
+   * Compute the initial npes-way partitioning geometric partitioning
+   =================================================================*/
+  IFSET(ctrl.dbglvl, DBG_TIME, InitTimers(&ctrl));
+  IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
+  IFSET(ctrl.dbglvl, DBG_TIME, starttimer(ctrl.TotalTmr));
+
+  Coordinate_Partition(&ctrl, graph, *ndims, xyz, 1, &wspace);
+
+  IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
+  IFSET(ctrl.dbglvl, DBG_TIME, stoptimer(ctrl.TotalTmr));
+  IFSET(ctrl.dbglvl, DBG_TIME, PrintTimingInfo(&ctrl));
+
+  /*=================================================================
+   * Move the graph according to the partitioning
+   =================================================================*/
+  IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
+  IFSET(ctrl.dbglvl, DBG_TIME, starttimer(ctrl.MoveTmr));
+
+  free(uvwgt);
+  graph->vwgt = ((iwgtflag&2) != 0) ? vwgt : idxsmalloc(graph->nvtxs*incon, 1, "vwgt");
+  graph->ncon = incon;
+  j = ctrl.nparts;
+  ctrl.nparts = ctrl.npes;
+  mgraph = Moc_MoveGraph(&ctrl, graph, &wspace);
+  ctrl.nparts = j;
+
+  /**********************************************************/
+  /* Do the same functionality as Moc_SetUpGraph for mgraph */
+  /**********************************************************/
+  /* compute tvwgts */
+  for (j=0; j<incon; j++)
+    ltvwgts[j] = 0;
+
+  for (i=0; i<graph->nvtxs; i++)
+    for (j=0; j<incon; j++)
+      ltvwgts[j] += mgraph->vwgt[i*incon+j];
+
+  for (j=0; j<incon; j++)
+    ctrl.tvwgts[j] = GlobalSESum(&ctrl, ltvwgts[j]);
+
+  /* check for zero wgt constraints */
+  for (i=0; i<incon; i++) {
+    /* ADD: take care of the case in which tvwgts is zero */
+    if (ctrl.tvwgts[i] == 0) {
+      if (ctrl.mype == 0) printf("ERROR: sum weight for constraint %d is zero\n", i);
+      MPI_Finalize();
+      exit(-1);
+    }
+  }
+
+  /* compute nvwgt */
+  mgraph->nvwgt = fmalloc(mgraph->nvtxs*incon, "mgraph->nvwgt");
+  for (i=0; i<mgraph->nvtxs; i++)
+    for (j=0; j<incon; j++)
+      mgraph->nvwgt[i*incon+j] = (float)(mgraph->vwgt[i*incon+j]) / (float)(ctrl.tvwgts[j]);
+
+
+  IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
+  IFSET(ctrl.dbglvl, DBG_TIME, stoptimer(ctrl.MoveTmr));
+
+  if (ctrl.dbglvl&DBG_INFO) {
+    cut = 0;
+    for (i=0; i<graph->nvtxs; i++)
+      for (j=graph->xadj[i]; j<graph->xadj[i+1]; j++)
+        if (graph->where[i] != graph->where[graph->adjncy[j]])
+          cut += graph->adjwgt[j];
+    gcut = GlobalSESum(&ctrl, cut)/2;
+    maxnvtxs = GlobalSEMax(&ctrl, mgraph->nvtxs);
+    balance = (float)(maxnvtxs)/((float)(graph->gnvtxs)/(float)(npes));
+    rprintf(&ctrl, "XYZ Cut: %6d \tBalance: %6.3f [%d %d %d]\n",
+      gcut, balance, maxnvtxs, graph->gnvtxs, npes);
+
+  }
+
+  /*=================================================================
+   * Set up the newly moved graph
+   =================================================================*/
+  IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
+  IFSET(ctrl.dbglvl, DBG_TIME, starttimer(ctrl.TotalTmr));
+
+  ctrl.nparts = inparts;
+  FreeWSpace(&wspace);
+  PreAllocateMemory(&ctrl, mgraph, &wspace);
+
+  /*=======================================================
+   * Now compute the partition of the moved graph
+   =======================================================*/
+  if (vtxdist[npes] < SMALLGRAPH || vtxdist[npes] < npes*20 || GlobalSESum(&ctrl, mgraph->nedges) == 0) {
+    IFSET(ctrl.dbglvl, DBG_INFO, rprintf(&ctrl, "Partitioning a graph of size %d serially\n", vtxdist[npes]));
+    PartitionSmallGraph(&ctrl, mgraph, &wspace);
+  }
+  else {
+    Moc_Global_Partition(&ctrl, mgraph, &wspace);
+  }
+  ParallelReMapGraph(&ctrl, mgraph, &wspace);
+
+  /* Invert the ordering back to the original graph */
+  ctrl.nparts = npes;
+  ProjectInfoBack(&ctrl, graph, part, mgraph->where, &wspace);
+
+  *edgecut = mgraph->mincut;
+
+  IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
+  IFSET(ctrl.dbglvl, DBG_TIME, stoptimer(ctrl.TotalTmr));
+
+  /*******************/
+  /* Print out stats */
+  /*******************/
+  IFSET(ctrl.dbglvl, DBG_TIME, PrintTimingInfo(&ctrl));
+  IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
+
+  if (ctrl.dbglvl&DBG_INFO) {
+    rprintf(&ctrl, "Final %d-way CUT: %6d \tBalance: ", inparts, mgraph->mincut);
+    avg = 0.0;
+    for (h=0; h<incon; h++) {
+      maximb = 0.0;
+      for (i=0; i<inparts; i++)
+        maximb = amax(maximb, mgraph->gnpwgts[i*incon+h]/itpwgts[i*incon+h]);
+      avg += maximb;
+      rprintf(&ctrl, "%.3f ", maximb);
+    }
+    rprintf(&ctrl, "  avg: %.3f\n", avg/(float)incon);
+  }
+
+  GKfree((void **)&itpwgts, LTERM);
+  FreeGraph(mgraph);
+  FreeInitialGraphAndRemap(graph, iwgtflag);
+  FreeWSpace(&wspace);
+  FreeCtrl(&ctrl);
+
+  if (inumflag == 1)
+    ChangeNumbering(vtxdist, xadj, adjncy, part, npes, mype, 0);
+
+}
+
+
+
+/***********************************************************************************
+* This function is the entry point of the parallel ordering algorithm.
+* This function assumes that the graph is already nice partitioned among the
+* processors and then proceeds to perform recursive bisection.
+************************************************************************************/
+void ParMETIS_V3_PartGeom(idxtype *vtxdist, int *ndims, float *xyz, idxtype *part, MPI_Comm *comm)
+{
+  int i, npes, mype, nvtxs, firstvtx, dbglvl;
+  idxtype *xadj, *adjncy;
+  CtrlType ctrl;
+  WorkSpaceType wspace;
+  GraphType *graph;
+  int zeroflg = 0;
+
+  MPI_Comm_size(*comm, &npes);
+  MPI_Comm_rank(*comm, &mype);
+
+  if (npes == 1) {
+    idxset(vtxdist[mype+1]-vtxdist[mype], 0, part);
+    return;
+  }
+
+  /* Setup a fake graph to allow the rest of the code to work unchanged */
+  dbglvl = 0;
+
+  nvtxs = vtxdist[mype+1]-vtxdist[mype];
+  firstvtx = vtxdist[mype];
+  xadj = idxmalloc(nvtxs+1, "ParMETIS_PartGeom: xadj");
+  adjncy = idxmalloc(nvtxs, "ParMETIS_PartGeom: adjncy");
+  for (i=0; i<nvtxs; i++) {
+    xadj[i] = i;
+    adjncy[i] = firstvtx + (i+1)%nvtxs;
+  }
+  xadj[nvtxs] = nvtxs;
+
+  /* Proceed with the rest of the code */
+  SetUpCtrl(&ctrl, npes, dbglvl, *comm);
+  ctrl.seed      = mype;
+  ctrl.CoarsenTo = amin(vtxdist[npes]+1, 25*npes);
+
+  graph = Moc_SetUpGraph(&ctrl, 1, vtxdist, xadj, NULL, adjncy, NULL, &zeroflg);
+
+  PreAllocateMemory(&ctrl, graph, &wspace);
+
+  /*=======================================================
+   * Compute the initial geometric partitioning
+   =======================================================*/
+  IFSET(ctrl.dbglvl, DBG_TIME, InitTimers(&ctrl));
+  IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
+  IFSET(ctrl.dbglvl, DBG_TIME, starttimer(ctrl.TotalTmr));
+
+  Coordinate_Partition(&ctrl, graph, *ndims, xyz, 0, &wspace);
+
+  idxcopy(graph->nvtxs, graph->where, part);
+
+  IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
+  IFSET(ctrl.dbglvl, DBG_TIME, stoptimer(ctrl.TotalTmr));
+  IFSET(ctrl.dbglvl, DBG_TIME, PrintTimingInfo(&ctrl));
+
+  FreeInitialGraphAndRemap(graph, 0);
+  FreeWSpace(&wspace);
+  FreeCtrl(&ctrl);
+
+  GKfree((void **)&xadj, (void **)&adjncy, LTERM);
+}
+
+
+
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/grsetup.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/grsetup.c
new file mode 100644
index 0000000..7f10a8f
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/grsetup.c
@@ -0,0 +1,274 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * mgrsetup.c
+ *
+ * This file contain various graph setting up routines
+ *
+ * Started 10/19/96
+ * George
+ *
+ * $Id: grsetup.c,v 1.7 2003/07/23 00:54:55 karypis Exp $
+ *
+ */
+
+#include <parmetislib.h>
+
+
+
+/*************************************************************************
+* This function setsup the CtrlType structure
+**************************************************************************/
+GraphType *Moc_SetUpGraph(CtrlType *ctrl, int ncon, idxtype *vtxdist, idxtype *xadj, 
+                          idxtype *vwgt, idxtype *adjncy, idxtype *adjwgt, int *wgtflag)
+{
+  int i, j;
+  GraphType *graph;
+  int ltvwgts[MAXNCON];
+
+  graph = CreateGraph();
+  graph->level   = 0;
+  graph->gnvtxs  = vtxdist[ctrl->npes];
+  graph->nvtxs   = vtxdist[ctrl->mype+1]-vtxdist[ctrl->mype];
+  graph->ncon    = ncon;
+  graph->nedges  = xadj[graph->nvtxs];
+  graph->xadj    = xadj;
+  graph->vwgt    = vwgt;
+  graph->adjncy  = adjncy;
+  graph->adjwgt  = adjwgt;
+  graph->vtxdist = vtxdist;
+
+
+  if (((*wgtflag)&2) == 0) 
+    graph->vwgt = idxsmalloc(graph->nvtxs*ncon, 1, "Par_KMetis: vwgt");
+
+  if (((*wgtflag)&1) == 0) 
+    graph->adjwgt = idxsmalloc(graph->nedges, 1, "Par_KMetis: adjwgt");
+
+  /* compute tvwgts */
+  for (j=0; j<ncon; j++)
+    ltvwgts[j] = 0;
+
+  for (i=0; i<graph->nvtxs; i++)
+    for (j=0; j<ncon; j++)
+      ltvwgts[j] += graph->vwgt[i*ncon+j];
+
+  for (j=0; j<ncon; j++)
+    ctrl->tvwgts[j] = GlobalSESum(ctrl, ltvwgts[j]);
+
+  /* check for zero wgt constraints */
+  for (i=0; i<ncon; i++) {
+    /* ADD: take care of the case in which tvwgts is zero */
+    if (ctrl->tvwgts[i] == 0) {
+      rprintf(ctrl, "ERROR: sum weight for constraint %d is zero\n", i);
+      MPI_Finalize();
+      exit(-1);
+    }
+  }
+
+  /* compute nvwgts */
+  graph->nvwgt = fmalloc(graph->nvtxs*ncon, "graph->nvwgt");
+  for (i=0; i<graph->nvtxs; i++) {
+    for (j=0; j<ncon; j++)
+      graph->nvwgt[i*ncon+j] = (float)(graph->vwgt[i*ncon+j]) / (float)(ctrl->tvwgts[j]);
+  }
+
+  srand(ctrl->seed);
+
+  return graph;
+}
+
+
+/*************************************************************************
+* This function setsup the CtrlType structure
+**************************************************************************/
+void SetUpCtrl(CtrlType *ctrl, int nparts, int dbglvl, MPI_Comm comm)
+{
+
+  MPI_Comm_dup(comm, &(ctrl->gcomm));
+  MPI_Comm_rank(ctrl->gcomm, &ctrl->mype);
+  MPI_Comm_size(ctrl->gcomm, &ctrl->npes);
+
+  ctrl->dbglvl  = dbglvl;
+  ctrl->nparts  = nparts;    /* Set the # of partitions is de-coupled from the # of domains */
+  ctrl->comm    = ctrl->gcomm;
+  ctrl->xyztype = XYZ_SPFILL;
+
+  srand(ctrl->mype);
+}
+
+
+/*************************************************************************
+* This function changes the numbering from 1 to 0 or 0 to 1
+**************************************************************************/
+void ChangeNumbering(idxtype *vtxdist, idxtype *xadj, idxtype *adjncy, idxtype *part, int npes, int mype, int from)
+{
+  int i, nvtxs, nedges;
+
+  if (from == 1) {  /* Change it from 1 to 0 */
+    for (i=0; i<npes+1; i++)
+      vtxdist[i]--;
+
+    nvtxs = vtxdist[mype+1]-vtxdist[mype];
+    for (i=0; i<nvtxs+1; i++) 
+      xadj[i]--;
+
+    nedges = xadj[nvtxs];
+    for (i=0; i<nedges; i++) 
+      adjncy[i]--;
+  }
+  else {  /* Change it from 0 to 1 */
+    nvtxs = vtxdist[mype+1]-vtxdist[mype];
+    nedges = xadj[nvtxs];
+
+    for (i=0; i<npes+1; i++) 
+      vtxdist[i]++;
+
+    for (i=0; i<nvtxs+1; i++) 
+      xadj[i]++; 
+
+    for (i=0; i<nedges; i++) 
+      adjncy[i]++; 
+
+    for (i=0; i<nvtxs; i++)
+      part[i]++;
+
+  }
+}
+
+
+/*************************************************************************
+* This function changes the numbering from 1 to 0 or 0 to 1
+**************************************************************************/
+void ChangeNumberingMesh(idxtype *elmdist, idxtype *elements, idxtype *xadj, 
+                         idxtype *adjncy, idxtype *part, int npes, int mype, 
+			 int elmntlen, int from)
+{
+  int i, nelms, nedges;
+
+  if (from == 1) {  /* Change it from 1 to 0 */
+    for (i=0; i<npes+1; i++)
+      elmdist[i]--;
+
+    for (i=0; i<elmntlen; i++) 
+      elements[i]--;
+  }
+  else {  /* Change it from 0 to 1 */
+    nelms = elmdist[mype+1]-elmdist[mype];
+    nedges = xadj[nelms];
+
+    for (i=0; i<npes+1; i++) 
+      elmdist[i]++;
+
+    for (i=0; i<elmntlen; i++) 
+      elements[i]++;
+
+    for (i=0; i<nelms+1; i++) 
+      xadj[i]++; 
+
+    for (i=0; i<nedges; i++) 
+      adjncy[i]++; 
+
+    if (part != NULL)
+      for (i=0; i<nelms; i++)
+        part[i]++;
+  }
+}
+
+
+/*************************************************************************
+* This function changes the numbering from 1 to 0 or 0 to 1
+**************************************************************************/
+void ChangeNumberingMesh2(idxtype *elmdist, idxtype *eptr, idxtype *eind, 
+                          idxtype *xadj, idxtype *adjncy, idxtype *part, 
+			  int npes, int mype, int from)
+{
+  int i, nelms;
+
+  nelms = elmdist[mype+1]-elmdist[mype];
+
+  if (from == 1) {  /* Change it from 1 to 0 */
+    for (i=0; i<npes+1; i++)
+      elmdist[i]--;
+
+    for (i=0; i<nelms+1; i++) 
+      eptr[i]--;
+
+    for (i=0; i<eptr[nelms]; i++) 
+      eind[i]--;
+  }
+  else {  /* Change it from 0 to 1 */
+    for (i=0; i<npes+1; i++) 
+      elmdist[i]++;
+
+    for (i=0; i<nelms+1; i++) 
+      eptr[i]++;
+
+    for (i=0; i<eptr[nelms]; i++) 
+      eind[i]++;
+
+    for (i=0; i<nelms+1; i++) 
+      xadj[i]++; 
+
+    for (i=0; i<xadj[nelms]; i++) 
+      adjncy[i]++; 
+
+    if (part != NULL)
+      for (i=0; i<nelms; i++)
+        part[i]++;
+  }
+}
+
+
+
+
+/*************************************************************************
+* This function randomly permutes the locally stored adjacency lists
+**************************************************************************/
+void GraphRandomPermute(GraphType *graph) 
+{
+  int i, j, k, tmp;
+
+  for (i=0; i<graph->nvtxs; i++) {
+    for (j=graph->xadj[i]; j<graph->xadj[i+1]; j++) {
+      k = graph->xadj[i] + RandomInRange(graph->xadj[i+1]-graph->xadj[i]);
+      SWAP(graph->adjncy[j], graph->adjncy[k], tmp);
+      SWAP(graph->adjwgt[j], graph->adjwgt[k], tmp);
+    }
+  }
+}
+
+
+/*************************************************************************
+* This function computes movement statistics for adaptive refinement
+* schemes
+**************************************************************************/
+void ComputeMoveStatistics(CtrlType *ctrl, GraphType *graph, int *nmoved, int *maxin, int *maxout)
+{
+  int i, j, nvtxs;
+  idxtype *vwgt, *where;
+  idxtype *lpvtxs, *gpvtxs;
+
+  nvtxs = graph->nvtxs;
+  vwgt = graph->vwgt;
+  where = graph->where;
+
+  lpvtxs = idxsmalloc(ctrl->nparts, 0, "ComputeMoveStatistics: lpvtxs");
+  gpvtxs = idxsmalloc(ctrl->nparts, 0, "ComputeMoveStatistics: gpvtxs");
+
+  for (j=i=0; i<nvtxs; i++) {
+    lpvtxs[where[i]]++;
+    if (where[i] != ctrl->mype)
+      j++;
+  }
+
+  /* PrintVector(ctrl, ctrl->npes, 0, lpvtxs, "Lpvtxs: "); */
+
+  MPI_Allreduce((void *)lpvtxs, (void *)gpvtxs, ctrl->nparts, IDX_DATATYPE, MPI_SUM, ctrl->comm);
+
+  *nmoved = GlobalSESum(ctrl, j);
+  *maxout = GlobalSEMax(ctrl, j);
+  *maxin = GlobalSEMax(ctrl, gpvtxs[ctrl->mype]-(nvtxs-j));
+
+  GKfree((void **)&lpvtxs, (void **)&gpvtxs, LTERM);
+}
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/iidxsort.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/iidxsort.c
new file mode 100644
index 0000000..869748c
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/iidxsort.c
@@ -0,0 +1,152 @@
+#include <parmetislib.h>
+
+
+/* Byte-wise swap two items of size SIZE. */
+#define QSSWAP(a, b, stmp) do { stmp = (a); (a) = (b); (b) = stmp; } while (0)
+
+/* Discontinue quicksort algorithm when partition gets below this size.
+   This particular magic number was chosen to work best on a Sun 4/260. */
+#define MAX_THRESH 20
+
+/* Stack node declarations used to store unfulfilled partition obligations. */
+typedef struct {
+  idxtype *lo;
+  idxtype *hi;
+} stack_node;
+
+
+/* The next 4 #defines implement a very fast in-line stack abstraction. */
+#define STACK_SIZE	(8 * sizeof(unsigned long int))
+#define PUSH(low, high)	((void) ((top->lo = (low)), (top->hi = (high)), ++top))
+#define	POP(low, high)	((void) (--top, (low = top->lo), (high = top->hi)))
+#define	STACK_NOT_EMPTY	(stack < top)
+
+
+void iidxsort(int total_elems, idxtype *pbase)
+{
+  idxtype pivot, stmp;
+
+  if (total_elems == 0)
+    /* Avoid lossage with unsigned arithmetic below.  */
+    return;
+
+  if (total_elems > MAX_THRESH) {
+    idxtype *lo = pbase;
+    idxtype *hi = &lo[total_elems - 1];
+    stack_node stack[STACK_SIZE]; /* Largest size needed for 32-bit int!!! */
+    stack_node *top = stack + 1;
+
+    while (STACK_NOT_EMPTY) {
+      idxtype *left_ptr;
+      idxtype *right_ptr;
+      idxtype *mid = lo + ((hi - lo) >> 1);
+
+
+      if (*mid < *lo) 
+        QSSWAP(*mid, *lo, stmp);
+      if (*hi < *mid)
+        QSSWAP(*mid, *hi, stmp);
+      else
+        goto jump_over;
+      if (*mid < *lo)
+        QSSWAP(*mid, *lo, stmp);
+
+jump_over:;
+      pivot = *mid;
+      left_ptr  = lo + 1;
+      right_ptr = hi - 1;
+
+      /* Here's the famous ``collapse the walls'' section of quicksort.
+	 Gotta like those tight inner loops!  They are the main reason
+	 that this algorithm runs much faster than others. */
+      do {
+	while (*left_ptr < pivot)
+	  left_ptr++;
+
+	while (pivot < *right_ptr)
+	  right_ptr--;
+
+	if (left_ptr < right_ptr) {
+	  QSSWAP (*left_ptr, *right_ptr, stmp);
+	  left_ptr++;
+	  right_ptr--;
+	}
+	else if (left_ptr == right_ptr) {
+	  left_ptr++;
+	  right_ptr--;
+	  break;
+	}
+      } while (left_ptr <= right_ptr);
+
+      /* Set up pointers for next iteration.  First determine whether
+         left and right partitions are below the threshold size.  If so,
+         ignore one or both.  Otherwise, push the larger partition's
+         bounds on the stack and continue sorting the smaller one. */
+
+      if ((size_t) (right_ptr - lo) <= MAX_THRESH) {
+        if ((size_t) (hi - left_ptr) <= MAX_THRESH)
+	  /* Ignore both small partitions. */
+          POP (lo, hi);
+        else
+	  /* Ignore small left partition. */
+          lo = left_ptr;
+      }
+      else if ((size_t) (hi - left_ptr) <= MAX_THRESH)
+	/* Ignore small right partition. */
+        hi = right_ptr;
+      else if ((right_ptr - lo) > (hi - left_ptr)) {
+       /* Push larger left partition indices. */
+       PUSH (lo, right_ptr);
+       lo = left_ptr;
+      }
+      else {
+	/* Push larger right partition indices. */
+        PUSH (left_ptr, hi);
+        hi = right_ptr;
+      }
+    }
+  }
+
+  /* Once the BASE_PTR array is partially sorted by quicksort the rest
+     is completely sorted using insertion sort, since this is efficient
+     for partitions below MAX_THRESH size. BASE_PTR points to the beginning
+     of the array to sort, and END_PTR points at the very last element in
+     the array (*not* one beyond it!). */
+
+  {
+    idxtype *end_ptr = &pbase[total_elems - 1];
+    idxtype *tmp_ptr = pbase;
+    idxtype *thresh = (end_ptr < pbase + MAX_THRESH ? end_ptr : pbase + MAX_THRESH);
+    register idxtype *run_ptr;
+
+    /* Find smallest element in first threshold and place it at the
+       array's beginning.  This is the smallest array element,
+       and the operation speeds up insertion sort's inner loop. */
+
+
+    for (run_ptr = tmp_ptr + 1; run_ptr <= thresh; run_ptr++)
+      if (*run_ptr < *tmp_ptr)
+        tmp_ptr = run_ptr;
+
+    if (tmp_ptr != pbase)
+      QSSWAP(*tmp_ptr, *pbase, stmp);
+
+    /* Insertion sort, running from left-hand-side up to right-hand-side.  */
+    run_ptr = pbase + 1;
+    while (++run_ptr <= end_ptr) {
+      tmp_ptr = run_ptr - 1;
+      while (*run_ptr < *tmp_ptr)
+        tmp_ptr--;
+
+      tmp_ptr++;
+      if (tmp_ptr != run_ptr) {
+        idxtype elmnt = *run_ptr;
+        idxtype *mptr;
+
+        for (mptr=run_ptr; mptr>tmp_ptr; mptr--)
+          *mptr = *(mptr-1);
+        *mptr = elmnt;
+      }
+    }
+  }
+}
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/iintsort.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/iintsort.c
new file mode 100644
index 0000000..fb0e840
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/iintsort.c
@@ -0,0 +1,157 @@
+#include <parmetislib.h>
+
+
+/* Byte-wise swap two items of size SIZE. */
+#define QSSWAP(a, b, stmp) do { stmp = (a); (a) = (b); (b) = stmp; } while (0)
+
+/* Discontinue quicksort algorithm when partition gets below this size.
+   This particular magic number was chosen to work best on a Sun 4/260. */
+#define MAX_THRESH 20
+
+/* Stack node declarations used to store unfulfilled partition obligations. */
+typedef struct {
+  int *lo;
+  int *hi;
+} stack_node;
+
+
+/* The next 4 #defines implement a very fast in-line stack abstraction. */
+#define STACK_SIZE	(8 * sizeof(unsigned long int))
+#define PUSH(low, high)	((void) ((top->lo = (low)), (top->hi = (high)), ++top))
+#define	POP(low, high)	((void) (--top, (low = top->lo), (high = top->hi)))
+#define	STACK_NOT_EMPTY	(stack < top)
+
+
+void iintsort(int total_elems, int *pbase)
+{
+  int pivot, stmp;
+
+  if (total_elems == 0)
+    /* Avoid lossage with unsigned arithmetic below.  */
+    return;
+
+  if (total_elems > MAX_THRESH) {
+    int *lo = pbase;
+    int *hi = &lo[total_elems - 1];
+    stack_node stack[STACK_SIZE]; /* Largest size needed for 32-bit int!!! */
+    stack_node *top = stack + 1;
+
+    while (STACK_NOT_EMPTY) {
+      int *left_ptr;
+      int *right_ptr;
+
+      /* Select median value from among LO, MID, and HI. Rearrange
+	 LO and HI so the three values are sorted. This lowers the
+	 probability of picking a pathological pivot value and
+	 skips a comparison for both the LEFT_PTR and RIGHT_PTR. */
+
+      int *mid = lo + ((hi - lo) >> 1);
+
+      if (*mid < *lo) 
+        QSSWAP(*mid, *lo, stmp);
+      if (*hi < *mid)
+        QSSWAP(*mid, *hi, stmp);
+      else
+        goto jump_over;
+      if (*mid < *lo)
+        QSSWAP(*mid, *lo, stmp);
+
+jump_over:;
+      pivot = *mid;
+      left_ptr  = lo + 1;
+      right_ptr = hi - 1;
+
+      /* Here's the famous ``collapse the walls'' section of quicksort.
+	 Gotta like those tight inner loops!  They are the main reason
+	 that this algorithm runs much faster than others. */
+      do {
+	while (*left_ptr < pivot)
+	  left_ptr++;
+
+	while (pivot < *right_ptr)
+	  right_ptr--;
+
+	if (left_ptr < right_ptr) {
+	  QSSWAP (*left_ptr, *right_ptr, stmp);
+	  left_ptr++;
+	  right_ptr--;
+	}
+	else if (left_ptr == right_ptr) {
+	  left_ptr++;
+	  right_ptr--;
+	  break;
+	}
+      } while (left_ptr <= right_ptr);
+
+      /* Set up pointers for next iteration.  First determine whether
+         left and right partitions are below the threshold size.  If so,
+         ignore one or both.  Otherwise, push the larger partition's
+         bounds on the stack and continue sorting the smaller one. */
+
+      if ((size_t) (right_ptr - lo) <= MAX_THRESH) {
+        if ((size_t) (hi - left_ptr) <= MAX_THRESH)
+	  /* Ignore both small partitions. */
+          POP (lo, hi);
+        else
+	  /* Ignore small left partition. */
+          lo = left_ptr;
+      }
+      else if ((size_t) (hi - left_ptr) <= MAX_THRESH)
+	/* Ignore small right partition. */
+        hi = right_ptr;
+      else if ((right_ptr - lo) > (hi - left_ptr)) {
+       /* Push larger left partition indices. */
+       PUSH (lo, right_ptr);
+       lo = left_ptr;
+      }
+      else {
+	/* Push larger right partition indices. */
+        PUSH (left_ptr, hi);
+        hi = right_ptr;
+      }
+    }
+  }
+
+  /* Once the BASE_PTR array is partially sorted by quicksort the rest
+     is completely sorted using insertion sort, since this is efficient
+     for partitions below MAX_THRESH size. BASE_PTR points to the beginning
+     of the array to sort, and END_PTR points at the very last element in
+     the array (*not* one beyond it!). */
+
+  {
+    int *end_ptr = &pbase[total_elems - 1];
+    int *tmp_ptr = pbase;
+    int *thresh = (end_ptr < pbase + MAX_THRESH ? end_ptr : pbase + MAX_THRESH);
+    register int *run_ptr;
+
+    /* Find smallest element in first threshold and place it at the
+       array's beginning.  This is the smallest array element,
+       and the operation speeds up insertion sort's inner loop. */
+
+
+    for (run_ptr = tmp_ptr + 1; run_ptr <= thresh; run_ptr++)
+      if (*run_ptr < *tmp_ptr)
+        tmp_ptr = run_ptr;
+
+    if (tmp_ptr != pbase)
+      QSSWAP(*tmp_ptr, *pbase, stmp);
+
+    /* Insertion sort, running from left-hand-side up to right-hand-side.  */
+    run_ptr = pbase + 1;
+    while (++run_ptr <= end_ptr) {
+      tmp_ptr = run_ptr - 1;
+      while (*run_ptr < *tmp_ptr)
+        tmp_ptr--;
+
+      tmp_ptr++;
+      if (tmp_ptr != run_ptr) {
+        int elmnt = *run_ptr;
+        int *mptr;
+
+        for (mptr=run_ptr; mptr>tmp_ptr; mptr--)
+          *mptr = *(mptr-1);
+        *mptr = elmnt;
+      }
+    }
+  }
+}
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/ikeysort.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/ikeysort.c
new file mode 100644
index 0000000..14b4241
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/ikeysort.c
@@ -0,0 +1,151 @@
+#include <parmetislib.h>
+
+
+/* Byte-wise swap two items of size SIZE. */
+#define QSSWAP(a, b, stmp) do { stmp = (a); (a) = (b); (b) = stmp; } while (0)
+
+/* Discontinue quicksort algorithm when partition gets below this size.
+   This particular magic number was chosen to work best on a Sun 4/260. */
+#define MAX_THRESH 20
+
+/* Stack node declarations used to store unfulfilled partition obligations. */
+typedef struct {
+  KeyValueType *lo;
+  KeyValueType *hi;
+} stack_node;
+
+
+/* The next 4 #defines implement a very fast in-line stack abstraction. */
+#define STACK_SIZE	(8 * sizeof(unsigned long int))
+#define PUSH(low, high)	((void) ((top->lo = (low)), (top->hi = (high)), ++top))
+#define	POP(low, high)	((void) (--top, (low = top->lo), (high = top->hi)))
+#define	STACK_NOT_EMPTY	(stack < top)
+
+
+void ikeysort(int total_elems, KeyValueType *pbase)
+{
+  KeyValueType pivot, stmp;
+
+  if (total_elems == 0)
+    /* Avoid lossage with unsigned arithmetic below.  */
+    return;
+
+  if (total_elems > MAX_THRESH) {
+    KeyValueType *lo = pbase;
+    KeyValueType *hi = &lo[total_elems - 1];
+    stack_node stack[STACK_SIZE]; /* Largest size needed for 32-bit int!!! */
+    stack_node *top = stack + 1;
+
+    while (STACK_NOT_EMPTY) {
+      KeyValueType *left_ptr;
+      KeyValueType *right_ptr;
+      KeyValueType *mid = lo + ((hi - lo) >> 1);
+
+      if (mid->key < lo->key) 
+        QSSWAP(*mid, *lo, stmp);
+      if (hi->key < mid->key)
+        QSSWAP(*mid, *hi, stmp);
+      else
+        goto jump_over;
+      if (mid->key < lo->key)
+        QSSWAP(*mid, *lo, stmp);
+
+jump_over:;
+      pivot = *mid;
+      left_ptr  = lo + 1;
+      right_ptr = hi - 1;
+
+      /* Here's the famous ``collapse the walls'' section of quicksort.
+	 Gotta like those tight inner loops!  They are the main reason
+	 that this algorithm runs much faster than others. */
+      do {
+	while (left_ptr->key < pivot.key)
+	  left_ptr++;
+
+	while (pivot.key < right_ptr->key)
+	  right_ptr--;
+
+	if (left_ptr < right_ptr) {
+	  QSSWAP (*left_ptr, *right_ptr, stmp);
+	  left_ptr++;
+	  right_ptr--;
+	}
+	else if (left_ptr == right_ptr) {
+	  left_ptr++;
+	  right_ptr--;
+	  break;
+	}
+      } while (left_ptr <= right_ptr);
+
+      /* Set up pointers for next iteration.  First determine whether
+         left and right partitions are below the threshold size.  If so,
+         ignore one or both.  Otherwise, push the larger partition's
+         bounds on the stack and continue sorting the smaller one. */
+
+      if ((size_t) (right_ptr - lo) <= MAX_THRESH) {
+        if ((size_t) (hi - left_ptr) <= MAX_THRESH)
+	  /* Ignore both small partitions. */
+          POP (lo, hi);
+        else
+	  /* Ignore small left partition. */
+          lo = left_ptr;
+      }
+      else if ((size_t) (hi - left_ptr) <= MAX_THRESH)
+	/* Ignore small right partition. */
+        hi = right_ptr;
+      else if ((right_ptr - lo) > (hi - left_ptr)) {
+       /* Push larger left partition indices. */
+       PUSH (lo, right_ptr);
+       lo = left_ptr;
+      }
+      else {
+	/* Push larger right partition indices. */
+        PUSH (left_ptr, hi);
+        hi = right_ptr;
+      }
+    }
+  }
+
+  /* Once the BASE_PTR array is partially sorted by quicksort the rest
+     is completely sorted using insertion sort, since this is efficient
+     for partitions below MAX_THRESH size. BASE_PTR points to the beginning
+     of the array to sort, and END_PTR points at the very last element in
+     the array (*not* one beyond it!). */
+
+  {
+    KeyValueType *end_ptr = &pbase[total_elems - 1];
+    KeyValueType *tmp_ptr = pbase;
+    KeyValueType *thresh = (end_ptr < pbase + MAX_THRESH ? end_ptr : pbase + MAX_THRESH);
+    register KeyValueType *run_ptr;
+
+    /* Find smallest element in first threshold and place it at the
+       array's beginning.  This is the smallest array element,
+       and the operation speeds up insertion sort's inner loop. */
+
+    for (run_ptr = tmp_ptr + 1; run_ptr <= thresh; run_ptr++)
+      if (run_ptr->key < tmp_ptr->key)
+        tmp_ptr = run_ptr;
+
+    if (tmp_ptr != pbase)
+      QSSWAP(*tmp_ptr, *pbase, stmp);
+
+    /* Insertion sort, running from left-hand-side up to right-hand-side.  */
+    run_ptr = pbase + 1;
+    while (++run_ptr <= end_ptr) {
+      tmp_ptr = run_ptr - 1;
+      while (run_ptr->key < tmp_ptr->key)
+        tmp_ptr--;
+
+      tmp_ptr++;
+      if (tmp_ptr != run_ptr) {
+        KeyValueType elmnt = *run_ptr;
+        KeyValueType *mptr;
+
+        for (mptr=run_ptr; mptr>tmp_ptr; mptr--)
+          *mptr = *(mptr-1);
+        *mptr = elmnt;
+      }
+    }
+  }
+}
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/ikeyvalsort.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/ikeyvalsort.c
new file mode 100644
index 0000000..59dce52
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/ikeyvalsort.c
@@ -0,0 +1,151 @@
+#include <parmetislib.h>
+
+
+/* Byte-wise swap two items of size SIZE. */
+#define QSSWAP(a, b, stmp) do { stmp = (a); (a) = (b); (b) = stmp; } while (0)
+
+/* Discontinue quicksort algorithm when partition gets below this size.
+   This particular magic number was chosen to work best on a Sun 4/260. */
+#define MAX_THRESH 20
+
+/* Stack node declarations used to store unfulfilled partition obligations. */
+typedef struct {
+  KeyValueType *lo;
+  KeyValueType *hi;
+} stack_node;
+
+
+/* The next 4 #defines implement a very fast in-line stack abstraction. */
+#define STACK_SIZE	(8 * sizeof(unsigned long int))
+#define PUSH(low, high)	((void) ((top->lo = (low)), (top->hi = (high)), ++top))
+#define	POP(low, high)	((void) (--top, (low = top->lo), (high = top->hi)))
+#define	STACK_NOT_EMPTY	(stack < top)
+
+
+void ikeyvalsort(int total_elems, KeyValueType *pbase)
+{
+  KeyValueType pivot, stmp;
+
+  if (total_elems == 0)
+    /* Avoid lossage with unsigned arithmetic below.  */
+    return;
+
+  if (total_elems > MAX_THRESH) {
+    KeyValueType *lo = pbase;
+    KeyValueType *hi = &lo[total_elems - 1];
+    stack_node stack[STACK_SIZE]; /* Largest size needed for 32-bit int!!! */
+    stack_node *top = stack + 1;
+
+    while (STACK_NOT_EMPTY) {
+      KeyValueType *left_ptr;
+      KeyValueType *right_ptr;
+      KeyValueType *mid = lo + ((hi - lo) >> 1);
+
+      if (mid->key < lo->key || (mid->key == lo->key && mid->val < lo->val)) 
+        QSSWAP(*mid, *lo, stmp);
+      if (hi->key < mid->key || (hi->key == mid->key && hi->val < mid->val))
+        QSSWAP(*mid, *hi, stmp);
+      else
+        goto jump_over;
+      if (mid->key < lo->key || (mid->key == lo->key && mid->val < lo->val))
+        QSSWAP(*mid, *lo, stmp);
+
+jump_over:;
+      pivot = *mid;
+      left_ptr  = lo + 1;
+      right_ptr = hi - 1;
+
+      /* Here's the famous ``collapse the walls'' section of quicksort.
+	 Gotta like those tight inner loops!  They are the main reason
+	 that this algorithm runs much faster than others. */
+      do {
+	while (left_ptr->key < pivot.key || (left_ptr->key == pivot.key && left_ptr->val < pivot.val))
+	  left_ptr++;
+
+	while (pivot.key < right_ptr->key || (pivot.key == right_ptr->key && pivot.val < right_ptr->val))
+	  right_ptr--;
+
+	if (left_ptr < right_ptr) {
+	  QSSWAP (*left_ptr, *right_ptr, stmp);
+	  left_ptr++;
+	  right_ptr--;
+	}
+	else if (left_ptr == right_ptr) {
+	  left_ptr++;
+	  right_ptr--;
+	  break;
+	}
+      } while (left_ptr <= right_ptr);
+
+      /* Set up pointers for next iteration.  First determine whether
+         left and right partitions are below the threshold size.  If so,
+         ignore one or both.  Otherwise, push the larger partition's
+         bounds on the stack and continue sorting the smaller one. */
+
+      if ((size_t) (right_ptr - lo) <= MAX_THRESH) {
+        if ((size_t) (hi - left_ptr) <= MAX_THRESH)
+	  /* Ignore both small partitions. */
+          POP (lo, hi);
+        else
+	  /* Ignore small left partition. */
+          lo = left_ptr;
+      }
+      else if ((size_t) (hi - left_ptr) <= MAX_THRESH)
+	/* Ignore small right partition. */
+        hi = right_ptr;
+      else if ((right_ptr - lo) > (hi - left_ptr)) {
+       /* Push larger left partition indices. */
+       PUSH (lo, right_ptr);
+       lo = left_ptr;
+      }
+      else {
+	/* Push larger right partition indices. */
+        PUSH (left_ptr, hi);
+        hi = right_ptr;
+      }
+    }
+  }
+
+  /* Once the BASE_PTR array is partially sorted by quicksort the rest
+     is completely sorted using insertion sort, since this is efficient
+     for partitions below MAX_THRESH size. BASE_PTR points to the beginning
+     of the array to sort, and END_PTR points at the very last element in
+     the array (*not* one beyond it!). */
+
+  {
+    KeyValueType *end_ptr = &pbase[total_elems - 1];
+    KeyValueType *tmp_ptr = pbase;
+    KeyValueType *thresh = (end_ptr < pbase + MAX_THRESH ? end_ptr : pbase + MAX_THRESH);
+    register KeyValueType *run_ptr;
+
+    /* Find smallest element in first threshold and place it at the
+       array's beginning.  This is the smallest array element,
+       and the operation speeds up insertion sort's inner loop. */
+
+    for (run_ptr = tmp_ptr + 1; run_ptr <= thresh; run_ptr++)
+      if (run_ptr->key < tmp_ptr->key || (run_ptr->key == tmp_ptr->key && run_ptr->val < tmp_ptr->val))
+        tmp_ptr = run_ptr;
+
+    if (tmp_ptr != pbase)
+      QSSWAP(*tmp_ptr, *pbase, stmp);
+
+    /* Insertion sort, running from left-hand-side up to right-hand-side.  */
+    run_ptr = pbase + 1;
+    while (++run_ptr <= end_ptr) {
+      tmp_ptr = run_ptr - 1;
+      while (run_ptr->key < tmp_ptr->key || (run_ptr->key == tmp_ptr->key && run_ptr->val < tmp_ptr->val))
+        tmp_ptr--;
+
+      tmp_ptr++;
+      if (tmp_ptr != run_ptr) {
+        KeyValueType elmnt = *run_ptr;
+        KeyValueType *mptr;
+
+        for (mptr=run_ptr; mptr>tmp_ptr; mptr--)
+          *mptr = *(mptr-1);
+        *mptr = elmnt;
+      }
+    }
+  }
+}
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/initbalance.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/initbalance.c
new file mode 100644
index 0000000..9ec7a31
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/initbalance.c
@@ -0,0 +1,498 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * initbalance.c
+ *
+ * This file contains code that computes an initial partitioning
+ *
+ * Started 3/4/96
+ * George
+ *
+ * $Id: initbalance.c,v 1.4 2003/07/30 21:18:52 karypis Exp $
+ */
+
+#include <parmetislib.h>
+
+
+/*************************************************************************
+* This function is the entry point of the initial balancing algorithm.
+* This algorithm assembles the graph to all the processors and preceeds
+* with the balancing step.
+**************************************************************************/
+void Balance_Partition(CtrlType *ctrl, GraphType *graph, WorkSpaceType *wspace)
+{
+  int i, j, mype, npes, nvtxs, nedges, ncon;
+  idxtype *vtxdist, *xadj, *adjncy, *adjwgt, *vwgt, *vsize;
+  idxtype *part, *lwhere, *home;
+  GraphType *agraph, cgraph;
+  CtrlType myctrl;
+  int lnparts, fpart, fpe, lnpes, ngroups, srnpes, srmype; 
+  int twoparts=2, numflag = 0, wgtflag = 3, moptions[10], edgecut, max_cut;
+  int sr_pe, gd_pe, sr, gd, who_wins, *rcounts, *rdispls;
+  float my_cut, my_totalv, my_cost = -1.0, my_balance = -1.0, wsum;
+  float rating, max_rating, your_cost = -1.0, your_balance = -1.0;
+  float lbvec[MAXNCON], lbsum, min_lbsum, *mytpwgts, mytpwgts2[2], buffer[2];
+  MPI_Status status;
+  MPI_Comm ipcomm, srcomm;
+  struct {
+    float cost;
+    int rank;
+  } lpecost, gpecost;
+
+  IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->InitPartTmr));
+
+  vtxdist = graph->vtxdist;
+  agraph = Moc_AssembleAdaptiveGraph(ctrl, graph, wspace);
+  nvtxs = cgraph.nvtxs = agraph->nvtxs;
+  nedges = cgraph.nedges = agraph->nedges;
+  ncon = cgraph.ncon = agraph->ncon;
+
+  xadj = cgraph.xadj = idxmalloc(nvtxs*(5+ncon)+1+nedges*2, "U_IP: xadj");
+  vwgt = cgraph.vwgt = xadj + nvtxs+1;
+  vsize = cgraph.vsize = xadj + nvtxs*(1+ncon)+1;
+  cgraph.where = agraph->where = part = xadj + nvtxs*(2+ncon)+1;
+  lwhere = xadj + nvtxs*(3+ncon)+1;
+  home = xadj + nvtxs*(4+ncon)+1;
+  adjncy = cgraph.adjncy = xadj + nvtxs*(5+ncon)+1;
+  adjwgt = cgraph.adjwgt = xadj + nvtxs*(5+ncon)+1 + nedges;
+
+  /* ADD: this assumes that tpwgts for all constraints is the same */
+  /* ADD: this is necessary because serial metis does not support the general case */
+  mytpwgts = fsmalloc(ctrl->nparts, 0.0, "mytpwgts");
+  for (i=0; i<ctrl->nparts; i++)
+    for (j=0; j<ncon; j++)
+      mytpwgts[i] += ctrl->tpwgts[i*ncon+j];
+  for (i=0; i<ctrl->nparts; i++)
+    mytpwgts[i] /= (float)ncon;
+
+  idxcopy(nvtxs+1, agraph->xadj, xadj);
+  idxcopy(nvtxs*ncon, agraph->vwgt, vwgt);
+  idxcopy(nvtxs, agraph->vsize, vsize);
+  idxcopy(nedges, agraph->adjncy, adjncy);
+  idxcopy(nedges, agraph->adjwgt, adjwgt);
+
+  /****************************************/
+  /****************************************/
+  if (ctrl->ps_relation == DISCOUPLED) {
+    rcounts = imalloc(ctrl->npes, "rcounts");
+    rdispls = imalloc(ctrl->npes+1, "rdispls");
+
+    for (i=0; i<ctrl->npes; i++) {
+      rdispls[i] = rcounts[i] = vtxdist[i+1]-vtxdist[i];
+    }
+    MAKECSR(i, ctrl->npes, rdispls);
+
+    MPI_Allgatherv((void *)graph->home, graph->nvtxs, IDX_DATATYPE,
+    (void *)part, rcounts, rdispls, IDX_DATATYPE, ctrl->comm);
+
+    for (i=0; i<agraph->nvtxs; i++)
+      home[i] = part[i];
+
+    GKfree((void **)&rcounts, (void **)&rdispls, LTERM);
+  }
+  else {
+    for (i=0; i<ctrl->npes; i++)
+      for (j=vtxdist[i]; j<vtxdist[i+1]; j++)
+        part[j] = home[j] = i;
+  }
+
+  /* Ensure that the initial partitioning is legal */
+  for (i=0; i<agraph->nvtxs; i++) {
+    if (part[i] >= ctrl->nparts)
+      part[i] = home[i] = part[i] % ctrl->nparts;
+    if (part[i] < 0)
+      part[i] = home[i] = (-1*part[i]) % ctrl->nparts;
+  }
+  /****************************************/
+  /****************************************/
+
+  IFSET(ctrl->dbglvl, DBG_REFINEINFO, Moc_ComputeSerialBalance(ctrl, agraph, agraph->where, lbvec));
+  IFSET(ctrl->dbglvl, DBG_REFINEINFO, rprintf(ctrl, "input cut: %d, balance: ", ComputeSerialEdgeCut(agraph)));
+  for (i=0; i<agraph->ncon; i++)
+    IFSET(ctrl->dbglvl, DBG_REFINEINFO, rprintf(ctrl, "%.3f ", lbvec[i]));
+  IFSET(ctrl->dbglvl, DBG_REFINEINFO, rprintf(ctrl, "\n"));
+
+  /****************************************/
+  /* Split the processors into two groups */
+  /****************************************/
+  sr = (ctrl->mype % 2 == 0) ? 1 : 0;
+  gd = (ctrl->mype % 2 == 1) ? 1 : 0;
+
+  if (graph->ncon > MAX_NCON_FOR_DIFFUSION || ctrl->npes == 1) {
+    sr = 1;
+    gd = 0;
+  }
+
+  sr_pe = 0;
+  gd_pe = 1;
+
+  MPI_Comm_split(ctrl->gcomm, sr, 0, &ipcomm);
+  MPI_Comm_rank(ipcomm, &mype);
+  MPI_Comm_size(ipcomm, &npes);
+
+  myctrl.dbglvl = 0;
+  myctrl.mype = mype;
+  myctrl.npes = npes;
+  myctrl.comm = ipcomm;
+  myctrl.sync = ctrl->sync;
+  myctrl.seed = ctrl->seed;
+  myctrl.nparts = ctrl->nparts;
+  myctrl.ipc_factor = ctrl->ipc_factor;
+  myctrl.redist_factor = ctrl->redist_base;
+  myctrl.partType = ADAPTIVE_PARTITION;
+  myctrl.ps_relation = DISCOUPLED;
+  myctrl.tpwgts = ctrl->tpwgts;
+  icopy(ncon, ctrl->tvwgts, myctrl.tvwgts);
+  icopy(ncon, ctrl->ubvec, myctrl.ubvec);
+
+  if (sr == 1) {
+    /*******************************************/
+    /* Half of the processors do scratch-remap */
+    /*******************************************/
+    ngroups = amax(amin(RIP_SPLIT_FACTOR, npes), 1);
+    MPI_Comm_split(ipcomm, mype % ngroups, 0, &srcomm);
+    MPI_Comm_rank(srcomm, &srmype);
+    MPI_Comm_size(srcomm, &srnpes);
+
+    moptions[0] = 0;
+    moptions[7] = ctrl->sync + (mype % ngroups) + 1;
+
+    idxset(nvtxs, 0, lwhere);
+    lnparts = ctrl->nparts;
+    fpart = fpe = 0;
+    lnpes = srnpes;
+    while (lnpes > 1 && lnparts > 1) {
+      ASSERT(ctrl, agraph->nvtxs > 1);
+      /* Determine the weights of the partitions */
+      mytpwgts2[0] = ssum(lnparts/2, mytpwgts+fpart);
+      mytpwgts2[1] = 1.0-mytpwgts2[0];
+
+
+      if (agraph->ncon == 1) {
+        METIS_WPartGraphKway2(&agraph->nvtxs, agraph->xadj, agraph->adjncy, agraph->vwgt, 
+	      agraph->adjwgt, &wgtflag, &numflag, &twoparts, mytpwgts2, moptions, &edgecut, 
+	      part);
+      }
+      else {
+        METIS_mCPartGraphRecursive2(&agraph->nvtxs, &ncon, agraph->xadj, agraph->adjncy, 
+	      agraph->vwgt, agraph->adjwgt, &wgtflag, &numflag, &twoparts, mytpwgts2, 
+	      moptions, &edgecut, part);
+      }
+
+      wsum = ssum(lnparts/2, mytpwgts+fpart);
+      sscale(lnparts/2, 1.0/wsum, mytpwgts+fpart);
+      sscale(lnparts-lnparts/2, 1.0/(1.0-wsum), mytpwgts+fpart+lnparts/2);
+
+      /* I'm picking the left branch */
+      if (srmype < fpe+lnpes/2) {
+        Moc_KeepPart(agraph, wspace, part, 0);
+        lnpes = lnpes/2;
+        lnparts = lnparts/2;
+      }
+      else {
+        Moc_KeepPart(agraph, wspace, part, 1);
+        fpart = fpart + lnparts/2;
+        fpe = fpe + lnpes/2;
+        lnpes = lnpes - lnpes/2;
+        lnparts = lnparts - lnparts/2;
+      }
+    }
+
+    /* In case srnpes is greater than or equal to nparts */
+    if (lnparts == 1) {
+      /* Only the first process will assign labels (for the reduction to work) */
+      if (srmype == fpe) {
+        for (i=0; i<agraph->nvtxs; i++) 
+          lwhere[agraph->label[i]] = fpart;
+      }
+    }
+    /* In case srnpes is smaller than nparts */
+    else {
+      if (ncon == 1)
+        METIS_WPartGraphKway2(&agraph->nvtxs, agraph->xadj, agraph->adjncy, agraph->vwgt, 
+	      agraph->adjwgt, &wgtflag, &numflag, &lnparts, mytpwgts+fpart, moptions, 
+	      &edgecut, part);
+      else
+        METIS_mCPartGraphRecursive2(&agraph->nvtxs, &ncon, agraph->xadj, agraph->adjncy, 
+	      agraph->vwgt, agraph->adjwgt, &wgtflag, &numflag, &lnparts, mytpwgts+fpart, 
+	      moptions, &edgecut, part);
+
+      for (i=0; i<agraph->nvtxs; i++) 
+        lwhere[agraph->label[i]] = fpart + part[i];
+    }
+
+    MPI_Allreduce((void *)lwhere, (void *)part, nvtxs, IDX_DATATYPE, MPI_SUM, srcomm);
+
+    edgecut = ComputeSerialEdgeCut(&cgraph);
+    Moc_ComputeSerialBalance(ctrl, &cgraph, part, lbvec);
+    lbsum = ssum(ncon, lbvec);
+    MPI_Allreduce((void *)&edgecut, (void *)&max_cut, 1, MPI_INT, MPI_MAX, ipcomm);
+    MPI_Allreduce((void *)&lbsum, (void *)&min_lbsum, 1, MPI_FLOAT, MPI_MIN, ipcomm);
+    lpecost.rank = ctrl->mype;
+    lpecost.cost = lbsum;
+    if (min_lbsum < UNBALANCE_FRACTION * (float)(ncon)) {
+      if (lbsum < UNBALANCE_FRACTION * (float)(ncon))
+        lpecost.cost = (float)edgecut;
+      else
+        lpecost.cost = (float)max_cut + lbsum;
+    }
+    MPI_Allreduce((void *)&lpecost, (void *)&gpecost, 1, MPI_FLOAT_INT, MPI_MINLOC, ipcomm);
+
+    if (ctrl->mype == gpecost.rank && ctrl->mype != sr_pe) {
+      MPI_Send((void *)part, nvtxs, IDX_DATATYPE, sr_pe, 1, ctrl->comm);
+    }
+
+    if (ctrl->mype != gpecost.rank && ctrl->mype == sr_pe) {
+      MPI_Recv((void *)part, nvtxs, IDX_DATATYPE, gpecost.rank, 1, ctrl->comm, &status);
+    }
+
+    if (ctrl->mype == sr_pe) {
+      idxcopy(nvtxs, part, lwhere);
+      SerialRemap(&cgraph, ctrl->nparts, home, lwhere, part, ctrl->tpwgts);
+    }
+
+    MPI_Comm_free(&srcomm);
+  }
+  /**************************************/
+  /* The other half do global diffusion */
+  /**************************************/
+  else {
+    /******************************************************************/
+    /* The next stmt is required to balance out the sr MPI_Comm_split */
+    /******************************************************************/
+    MPI_Comm_split(ipcomm, MPI_UNDEFINED, 0, &srcomm);
+
+    if (ncon == 1) {
+      rating = WavefrontDiffusion(&myctrl, agraph, home);
+      Moc_ComputeSerialBalance(ctrl, &cgraph, part, lbvec);
+      lbsum = ssum(ncon, lbvec);
+
+      /* Determine which PE computed the best partitioning */
+      MPI_Allreduce((void *)&rating, (void *)&max_rating, 1, MPI_FLOAT, MPI_MAX, ipcomm);
+      MPI_Allreduce((void *)&lbsum, (void *)&min_lbsum, 1, MPI_FLOAT, MPI_MIN, ipcomm);
+
+      lpecost.rank = ctrl->mype;
+      lpecost.cost = lbsum;
+      if (min_lbsum < UNBALANCE_FRACTION * (float)(ncon)) {
+        if (lbsum < UNBALANCE_FRACTION * (float)(ncon))
+          lpecost.cost = rating;
+        else
+          lpecost.cost = max_rating + lbsum;
+      }
+
+      MPI_Allreduce((void *)&lpecost, (void *)&gpecost, 1, MPI_FLOAT_INT, MPI_MINLOC, ipcomm);
+
+      /* Now send this to the coordinating processor */
+      if (ctrl->mype == gpecost.rank && ctrl->mype != gd_pe)
+        MPI_Send((void *)part, nvtxs, IDX_DATATYPE, gd_pe, 1, ctrl->comm);
+
+      if (ctrl->mype != gpecost.rank && ctrl->mype == gd_pe)
+        MPI_Recv((void *)part, nvtxs, IDX_DATATYPE, gpecost.rank, 1, ctrl->comm, &status);
+
+      if (ctrl->mype == gd_pe) {
+        idxcopy(nvtxs, part, lwhere);
+        SerialRemap(&cgraph, ctrl->nparts, home, lwhere, part, ctrl->tpwgts);
+      }
+    }
+    else {
+      Moc_Diffusion(&myctrl, agraph, graph->vtxdist, agraph->where, home, wspace, N_MOC_GD_PASSES);
+    }
+  }
+
+  if (graph->ncon <= MAX_NCON_FOR_DIFFUSION) {
+    if (ctrl->mype == sr_pe  || ctrl->mype == gd_pe) {
+      /********************************************************************/
+      /* The coordinators from each group decide on the best partitioning */
+      /********************************************************************/
+      my_cut = (float) ComputeSerialEdgeCut(&cgraph);
+      my_totalv = (float) Mc_ComputeSerialTotalV(&cgraph, home);
+      Moc_ComputeSerialBalance(ctrl, &cgraph, part, lbvec);
+      my_balance = ssum(cgraph.ncon, lbvec);
+      my_balance /= (float) cgraph.ncon;
+      my_cost = ctrl->ipc_factor * my_cut + REDIST_WGT * ctrl->redist_base * my_totalv;
+
+      IFSET(ctrl->dbglvl, DBG_REFINEINFO, printf("%s initial cut: %.1f, totalv: %.1f, balance: %.3f\n",
+      (ctrl->mype == sr_pe ? "scratch-remap" : "diffusion"), my_cut, my_totalv, my_balance));
+
+      if (ctrl->mype == gd_pe) {
+        buffer[0] = my_cost;
+        buffer[1] = my_balance;
+        MPI_Send((void *)buffer, 2, MPI_FLOAT, sr_pe, 1, ctrl->comm);
+      }
+      else {
+        MPI_Recv((void *)buffer, 2, MPI_FLOAT, gd_pe, 1, ctrl->comm, &status);
+        your_cost = buffer[0];
+        your_balance = buffer[1];
+      }
+    }
+
+    if (ctrl->mype == sr_pe) {
+      who_wins = gd_pe;
+      if ((my_balance < 1.1 && your_balance > 1.1) ||
+          (my_balance < 1.1 && your_balance < 1.1 && my_cost < your_cost) ||
+          (my_balance > 1.1 && your_balance > 1.1 && my_balance < your_balance)) {
+        who_wins = sr_pe;
+      }
+    }
+
+    MPI_Bcast((void *)&who_wins, 1, MPI_INT, sr_pe, ctrl->comm);
+  }
+  else {
+    who_wins = sr_pe;
+  }
+
+  MPI_Bcast((void *)part, nvtxs, IDX_DATATYPE, who_wins, ctrl->comm);
+  idxcopy(graph->nvtxs, part+vtxdist[ctrl->mype], graph->where);
+
+  MPI_Comm_free(&ipcomm);
+  GKfree((void **)&xadj, (void **)&mytpwgts, LTERM);
+
+/* For whatever reason, FreeGraph crashes here...so explicitly free the memory.
+  FreeGraph(agraph);
+*/
+  GKfree((void **)&agraph->xadj, (void **)&agraph->adjncy, (void **)&agraph->vwgt, (void **)&agraph->nvwgt, LTERM);
+  GKfree((void **)&agraph->vsize, (void **)&agraph->adjwgt, (void **)&agraph->label, LTERM);
+  GKfree((void **)&agraph, LTERM);
+
+  IFSET(ctrl->dbglvl, DBG_TIME, stoptimer(ctrl->InitPartTmr));
+
+}
+
+
+/* NOTE: this subroutine should work for static, adaptive, single-, and multi-contraint */
+/*************************************************************************
+* This function assembles the graph into a single processor
+**************************************************************************/
+GraphType *Moc_AssembleAdaptiveGraph(CtrlType *ctrl, GraphType *graph, WorkSpaceType *wspace)
+{
+  int i, j, k, l, gnvtxs, nvtxs, ncon, gnedges, nedges, gsize;
+  idxtype *xadj, *vwgt, *vsize, *adjncy, *adjwgt, *vtxdist, *imap;
+  idxtype *axadj, *aadjncy, *aadjwgt, *avwgt, *avsize = NULL, *alabel;
+  idxtype *mygraph, *ggraph;
+  int *rcounts, *rdispls, mysize;
+  float *anvwgt;
+  GraphType *agraph;
+
+  gnvtxs  = graph->gnvtxs;
+  nvtxs   = graph->nvtxs;
+  ncon    = graph->ncon;
+  nedges  = graph->xadj[nvtxs];
+  xadj    = graph->xadj;
+  vwgt    = graph->vwgt;
+  vsize   = graph->vsize;
+  adjncy  = graph->adjncy;
+  adjwgt  = graph->adjwgt;
+  vtxdist = graph->vtxdist;
+  imap    = graph->imap;
+
+  /*************************************************************/
+  /* Determine the # of idxtype to receive from each processor */
+  /*************************************************************/
+  rcounts = imalloc(ctrl->npes, "AssembleGraph: rcounts");
+  switch (ctrl->partType) {
+    case STATIC_PARTITION:
+      mysize = (1+ncon)*nvtxs + 2*nedges;
+      break;
+    case ADAPTIVE_PARTITION:
+    case REFINE_PARTITION:
+      mysize = (2+ncon)*nvtxs + 2*nedges;
+      break;
+    default:
+      printf("WARNING: bad value for ctrl->partType %d\n", ctrl->partType);
+      break;
+  }
+  MPI_Allgather((void *)(&mysize), 1, MPI_INT, (void *)rcounts, 1, MPI_INT, ctrl->comm);
+
+  rdispls = imalloc(ctrl->npes+1, "AssembleGraph: rdispls");
+  rdispls[0] = 0;
+  for (i=1; i<ctrl->npes+1; i++)
+    rdispls[i] = rdispls[i-1] + rcounts[i-1];
+
+  /* Construct the one-array storage format of the assembled graph */
+  mygraph = (mysize <= wspace->maxcore ? wspace->core : idxmalloc(mysize, "AssembleGraph: mygraph"));
+  for (k=i=0; i<nvtxs; i++) {
+    mygraph[k++] = xadj[i+1]-xadj[i];
+    for (j=0; j<ncon; j++)
+      mygraph[k++] = vwgt[i*ncon+j];
+    if (ctrl->partType == ADAPTIVE_PARTITION || ctrl->partType == REFINE_PARTITION)
+      mygraph[k++] = vsize[i];
+    for (j=xadj[i]; j<xadj[i+1]; j++) {
+      mygraph[k++] = imap[adjncy[j]];
+      mygraph[k++] = adjwgt[j];
+    }
+  }
+  ASSERT(ctrl, mysize == k);
+
+  /**************************************/
+  /* Assemble and send the entire graph */
+  /**************************************/
+  gsize = rdispls[ctrl->npes];
+  ggraph = (gsize <= wspace->maxcore-mysize ? wspace->core+mysize : idxmalloc(gsize, "AssembleGraph: ggraph"));
+  MPI_Allgatherv((void *)mygraph, mysize, IDX_DATATYPE, (void *)ggraph, rcounts, rdispls, IDX_DATATYPE, ctrl->comm);
+
+  GKfree((void **)&rcounts, (void **)&rdispls, LTERM);
+  if (mysize > wspace->maxcore)
+    free(mygraph);
+
+  agraph = CreateGraph();
+  agraph->nvtxs = gnvtxs;
+  switch (ctrl->partType) {
+    case STATIC_PARTITION:
+      agraph->nedges = gnedges = (gsize-(1+ncon)*gnvtxs)/2;
+      break;
+    case ADAPTIVE_PARTITION:
+    case REFINE_PARTITION:
+      agraph->nedges = gnedges = (gsize-(2+ncon)*gnvtxs)/2;
+      break;
+    default:
+      printf("WARNING: bad value for ctrl->partType %d\n", ctrl->partType);
+      agraph->nedges = gnedges = -1;
+      break;
+  }
+
+  agraph->ncon = ncon;
+
+  /*******************************************/
+  /* Allocate memory for the assembled graph */
+  /*******************************************/
+  axadj = agraph->xadj = idxmalloc(gnvtxs+1, "AssembleGraph: axadj");
+  avwgt = agraph->vwgt = idxmalloc(gnvtxs*ncon, "AssembleGraph: avwgt");
+  anvwgt = agraph->nvwgt = fmalloc(gnvtxs*ncon, "AssembleGraph: anvwgt");
+  aadjncy = agraph->adjncy = idxmalloc(gnedges, "AssembleGraph: adjncy");
+  aadjwgt = agraph->adjwgt = idxmalloc(gnedges, "AssembleGraph: adjwgt");
+  alabel = agraph->label = idxmalloc(gnvtxs, "AssembleGraph: alabel");
+  if (ctrl->partType == ADAPTIVE_PARTITION || ctrl->partType == REFINE_PARTITION)
+    avsize = agraph->vsize = idxmalloc(gnvtxs, "AssembleGraph: avsize");
+
+  for (k=j=i=0; i<gnvtxs; i++) {
+    axadj[i] = ggraph[k++];
+    for (l=0; l<ncon; l++)
+      avwgt[i*ncon+l] = ggraph[k++];
+    if (ctrl->partType == ADAPTIVE_PARTITION || ctrl->partType == REFINE_PARTITION)
+      avsize[i] = ggraph[k++];
+    for (l=0; l<axadj[i]; l++) {
+      aadjncy[j] = ggraph[k++];
+      aadjwgt[j] = ggraph[k++];
+      j++;
+    }
+  }
+
+  /*********************************/
+  /* Now fix up the received graph */
+  /*********************************/
+  MAKECSR(i, gnvtxs, axadj);
+
+  for (i=0; i<gnvtxs; i++)
+    for (j=0; j<ncon; j++)
+      anvwgt[i*ncon+j] = (float)(agraph->vwgt[i*ncon+j]) / (float)(ctrl->tvwgts[j]);
+
+  for (i=0; i<gnvtxs; i++)
+    alabel[i] = i;
+
+  if (gsize > wspace->maxcore-mysize)
+    free(ggraph);
+
+  return agraph;
+}
+
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/initmsection.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/initmsection.c
new file mode 100644
index 0000000..63c7c35
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/initmsection.c
@@ -0,0 +1,242 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * initmsection.c
+ *
+ * This file contains code that performs the k-way multisection
+ *
+ * Started 6/3/97
+ * George
+ *
+ * $Id: initmsection.c,v 1.2 2003/07/21 17:18:49 karypis Exp $
+ */
+
+#include <parmetislib.h>
+
+
+#define DEBUG_IPART_
+
+
+
+/*************************************************************************
+* This function is the entry point of the initial partitioning algorithm.
+* This algorithm assembles the graph to all the processors and preceed
+* serially.
+**************************************************************************/
+void InitMultisection(CtrlType *ctrl, GraphType *graph, WorkSpaceType *wspace)
+{
+  int i, lpecut[2], gpecut[2], mypart, moptions[10];
+  idxtype *vtxdist, *gwhere = NULL, *part, *label;
+  GraphType *agraph;
+  int *sendcounts, *displs;
+  MPI_Comm newcomm, labelcomm;
+
+  IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->InitPartTmr));
+
+  /* Assemble the graph and do the necessary pre-processing */
+  agraph = AssembleMultisectedGraph(ctrl, graph, wspace);
+  part = agraph->where;
+  agraph->where = NULL;
+
+  /* Split the processors into groups so that each one can do a bisection */
+  mypart = ctrl->mype%(ctrl->nparts/2);
+  MPI_Comm_split(ctrl->comm, mypart, 0, &newcomm);
+
+  /* Each processor keeps the graphs that it only needs and bisects it */
+  agraph->ncon = 1; /* needed for Moc_KeepPart */
+  Moc_KeepPart(agraph, wspace, part, mypart);
+  label = agraph->label;  /* Save this because ipart may need it */
+  agraph->label = NULL;
+
+  /* Bisect the graph and construct the separator */
+  switch (ctrl->ipart) {
+    case ISEP_EDGE:
+      moptions[0] = 1;
+      moptions[1] = 3;
+      moptions[2] = 1;
+      moptions[3] = 1;
+      moptions[4] = 0;
+      moptions[7] = ctrl->mype;
+
+      agraph->where = idxmalloc(agraph->nvtxs, "InitMultisection: agraph->where");
+
+      METIS_EdgeComputeSeparator(&agraph->nvtxs, agraph->xadj, agraph->adjncy,
+        agraph->vwgt, agraph->adjwgt, moptions, &agraph->mincut, agraph->where);
+      break;
+    case ISEP_NODE:
+      moptions[0] = 1;
+      moptions[1] = 3;
+      moptions[2] = 1;
+      moptions[3] = 2;
+      moptions[4] = 0;
+      moptions[7] = ctrl->mype;
+
+      agraph->where = idxmalloc(agraph->nvtxs, "InitMultisection: agraph->where");
+
+      METIS_NodeComputeSeparator(&agraph->nvtxs, agraph->xadj, agraph->adjncy, agraph->vwgt, 
+            agraph->adjwgt, moptions, &agraph->mincut, agraph->where);
+      break;
+    default:
+      errexit("Unknown ISEP type!\n");
+  }
+
+  for (i=0; i<agraph->nvtxs; i++) {
+    ASSERT(ctrl, agraph->where[i]>=0 && agraph->where[i]<=2);
+    if (agraph->where[i] == 2)
+      agraph->where[i] = ctrl->nparts+2*mypart;
+    else
+      agraph->where[i] += 2*mypart;
+  }
+
+  /* Determine which PE got the minimum cut */
+  lpecut[0] = agraph->mincut;
+  MPI_Comm_rank(newcomm, lpecut+1);
+  MPI_Allreduce(lpecut, gpecut, 1, MPI_2INT, MPI_MINLOC, newcomm);
+
+  /* myprintf(ctrl, "Nvtxs: %d, Mincut: %d, GMincut: %d, %d\n", agraph->nvtxs, agraph->mincut, gpecut[0], gpecut[1]); */
+
+  /* Send the best where to the root processor of this partition */
+  if (lpecut[1] == gpecut[1] && gpecut[1] != 0) 
+    MPI_Send((void *)agraph->where, agraph->nvtxs, IDX_DATATYPE, 0, 1, newcomm);
+  if (lpecut[1] == 0 && gpecut[1] != 0)
+    MPI_Recv((void *)agraph->where, agraph->nvtxs, IDX_DATATYPE, gpecut[1], 1, newcomm, &ctrl->status);
+
+  /* Create a communicator that stores all the i-th processors of the newcomm */
+  MPI_Comm_split(ctrl->comm, lpecut[1], 0, &labelcomm);
+
+  /* Map the separator back to agraph. This is inefficient! */
+  if (lpecut[1] == 0) {
+    gwhere = idxsmalloc(graph->gnvtxs, 0, "InitMultisection: gwhere");
+    for (i=0; i<agraph->nvtxs; i++)
+      gwhere[label[i]] = agraph->where[i];
+  }
+
+  free(agraph->where);
+  agraph->where = part;
+
+  if (lpecut[1] == 0) {
+    MPI_Reduce((void *)gwhere, (void *)agraph->where, graph->gnvtxs, IDX_DATATYPE, MPI_SUM, 0, labelcomm);
+    free(gwhere);
+  }
+
+  /* The minimum PE performs the Scatter */
+  vtxdist = graph->vtxdist;
+  ASSERT(ctrl, graph->where != NULL);
+  free(graph->where);  /* Remove the propagated down where info */
+  graph->where = idxmalloc(graph->nvtxs+graph->nrecv, "InitPartition: where");
+
+  sendcounts = imalloc(ctrl->npes, "InitPartitionNew: sendcounts");
+  displs = imalloc(ctrl->npes, "InitPartitionNew: displs");
+
+  for (i=0; i<ctrl->npes; i++) {
+    sendcounts[i] = vtxdist[i+1]-vtxdist[i];
+    displs[i] = vtxdist[i];
+  }
+
+  MPI_Scatterv((void *)agraph->where, sendcounts, displs, IDX_DATATYPE, 
+               (void *)graph->where, graph->nvtxs, IDX_DATATYPE, 0, ctrl->comm);
+
+  GKfree((void **)&sendcounts, (void **)&displs, (void **)&label, LTERM);
+
+  FreeGraph(agraph);
+
+  MPI_Comm_free(&newcomm);
+  MPI_Comm_free(&labelcomm);
+
+  IFSET(ctrl->dbglvl, DBG_TIME, stoptimer(ctrl->InitPartTmr));
+
+}
+
+
+
+
+/*************************************************************************
+* This function assembles the graph into a single processor
+**************************************************************************/
+GraphType *AssembleMultisectedGraph(CtrlType *ctrl, GraphType *graph, WorkSpaceType *wspace)
+{
+  int i, j, k, l, gnvtxs, nvtxs, gnedges, nedges, gsize;
+  idxtype *xadj, *vwgt, *where, *adjncy, *adjwgt, *vtxdist, *imap;
+  idxtype *axadj, *aadjncy, *aadjwgt, *avwgt, *awhere, *alabel;
+  idxtype *mygraph, *ggraph;
+  int *recvcounts, *displs, mysize;
+  GraphType *agraph;
+
+  gnvtxs = graph->gnvtxs;
+  nvtxs = graph->nvtxs;
+  nedges = graph->xadj[nvtxs];
+  xadj = graph->xadj;
+  vwgt = graph->vwgt;
+  where = graph->where;
+  adjncy = graph->adjncy;
+  adjwgt = graph->adjwgt;
+  vtxdist = graph->vtxdist;
+  imap = graph->imap;
+
+  /* Determine the # of idxtype to receive from each processor */
+  recvcounts = imalloc(ctrl->npes, "AssembleGraph: recvcounts");
+  mysize = 3*nvtxs + 2*nedges;
+  MPI_Allgather((void *)(&mysize), 1, MPI_INT, (void *)recvcounts, 1, MPI_INT, ctrl->comm);
+  
+  displs = imalloc(ctrl->npes+1, "AssembleGraph: displs");
+  displs[0] = 0;
+  for (i=1; i<ctrl->npes+1; i++) 
+    displs[i] = displs[i-1] + recvcounts[i-1];
+
+  /* Construct the one-array storage format of the assembled graph */
+  mygraph = (mysize <= wspace->maxcore ? wspace->core : idxmalloc(mysize, "AssembleGraph: mygraph"));
+  for (k=i=0; i<nvtxs; i++) {
+    mygraph[k++] = xadj[i+1]-xadj[i];
+    mygraph[k++] = vwgt[i];
+    mygraph[k++] = where[i];
+    for (j=xadj[i]; j<xadj[i+1]; j++) {
+      mygraph[k++] = imap[adjncy[j]];
+      mygraph[k++] = adjwgt[j];
+    }
+  }
+  ASSERT(ctrl, mysize == k);
+
+  /* Assemble the entire graph */
+  gsize = displs[ctrl->npes];
+  ggraph = (gsize <= wspace->maxcore-mysize ? wspace->core+mysize : idxmalloc(gsize, "AssembleGraph: ggraph"));
+  MPI_Allgatherv((void *)mygraph, mysize, IDX_DATATYPE, (void *)ggraph, recvcounts, displs, IDX_DATATYPE, ctrl->comm);
+
+  GKfree((void **)&recvcounts, (void **)&displs, LTERM);
+  if (mysize > wspace->maxcore)
+    free(mygraph);
+
+  agraph = CreateGraph();
+  agraph->nvtxs = gnvtxs;
+  agraph->nedges = gnedges = (gsize-3*gnvtxs)/2;
+
+  /* Allocate memory for the assembled graph */
+  axadj = agraph->xadj = idxmalloc(gnvtxs+1, "AssembleGraph: axadj");
+  avwgt = agraph->vwgt = idxmalloc(gnvtxs, "AssembleGraph: avwgt");
+  awhere = agraph->where = idxmalloc(gnvtxs, "AssembleGraph: awhere");
+  aadjncy = agraph->adjncy = idxmalloc(gnedges, "AssembleGraph: adjncy");
+  aadjwgt = agraph->adjwgt = idxmalloc(gnedges, "AssembleGraph: adjwgt");
+  alabel = agraph->label = idxmalloc(gnvtxs, "AssembleGraph: alabel");
+
+  for (k=j=i=0; i<gnvtxs; i++) {
+    axadj[i] = ggraph[k++];
+    avwgt[i] = ggraph[k++];
+    awhere[i] = ggraph[k++];
+    for (l=0; l<axadj[i]; l++) {
+      aadjncy[j] = ggraph[k++];
+      aadjwgt[j] = ggraph[k++];
+      j++;
+    }
+  }
+
+  /* Now fix up the received graph */
+  MAKECSR(i, gnvtxs, axadj);
+
+  for (i=0; i<gnvtxs; i++)
+    alabel[i] = i;
+
+  if (gsize > wspace->maxcore-mysize)
+    free(ggraph);
+
+  return agraph;
+}
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/initpart.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/initpart.c
new file mode 100644
index 0000000..40b8d95
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/initpart.c
@@ -0,0 +1,252 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * initpart.c
+ *
+ * This file contains code that performs log(p) parallel multilevel
+ * recursive bissection
+ *
+ * Started 3/4/96
+ * George
+ *
+ * $Id: initpart.c,v 1.2 2003/07/21 17:18:49 karypis Exp $
+ */
+
+#include <parmetislib.h>
+
+
+#define DEBUG_IPART_
+
+
+
+/*************************************************************************
+* This function is the entry point of the initial partition algorithm
+* that does recursive bissection.
+* This algorithm assembles the graph to all the processors and preceeds
+* by parallelizing the recursive bisection step.
+**************************************************************************/
+void Moc_InitPartition_RB(CtrlType *ctrl, GraphType *graph, WorkSpaceType *wspace)
+{
+  int i, j;
+  int ncon, mype, npes, gnvtxs, ngroups;
+  idxtype *xadj, *adjncy, *adjwgt, *vwgt;
+  idxtype *part, *gwhere0, *gwhere1;
+  idxtype *tmpwhere, *tmpvwgt, *tmpxadj, *tmpadjncy, *tmpadjwgt;
+  GraphType *agraph;
+  int lnparts, fpart, fpe, lnpes; 
+  int twoparts=2, numflag = 0, wgtflag = 3, moptions[10], edgecut, max_cut;
+  float *mytpwgts, mytpwgts2[2], lbvec[MAXNCON], lbsum, min_lbsum, wsum;
+  MPI_Comm ipcomm;
+  struct {
+    float sum;
+    int rank;
+  } lpesum, gpesum;
+
+  ncon = graph->ncon;
+  ngroups = amax(amin(RIP_SPLIT_FACTOR, ctrl->npes), 1);
+
+  IFSET(ctrl->dbglvl, DBG_TIME, MPI_Barrier(ctrl->comm));
+  IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->InitPartTmr));
+
+  agraph = Moc_AssembleAdaptiveGraph(ctrl, graph, wspace);
+  part = idxmalloc(agraph->nvtxs, "Moc_IP_RB: part");
+  xadj = idxmalloc(agraph->nvtxs+1, "Moc_IP_RB: xadj");
+  adjncy = idxmalloc(agraph->nedges, "Moc_IP_RB: adjncy");
+  adjwgt = idxmalloc(agraph->nedges, "Moc_IP_RB: adjwgt");
+  vwgt = idxmalloc(agraph->nvtxs*ncon, "Moc_IP_RB: vwgt");
+
+  idxcopy(agraph->nvtxs*ncon, agraph->vwgt, vwgt);
+  idxcopy(agraph->nvtxs+1, agraph->xadj, xadj);
+  idxcopy(agraph->nedges, agraph->adjncy, adjncy);
+  idxcopy(agraph->nedges, agraph->adjwgt, adjwgt);
+
+  MPI_Comm_split(ctrl->gcomm, ctrl->mype % ngroups, 0, &ipcomm);
+  MPI_Comm_rank(ipcomm, &mype);
+  MPI_Comm_size(ipcomm, &npes);
+
+  gnvtxs = agraph->nvtxs;
+
+  gwhere0 = idxsmalloc(gnvtxs, 0, "Moc_IP_RB: gwhere0");
+  gwhere1 = idxmalloc(gnvtxs, "Moc_IP_RB: gwhere1");
+
+  /* ADD: this assumes that tpwgts for all constraints is the same */
+  /* ADD: this is necessary because serial metis does not support the general case */
+  mytpwgts = fsmalloc(ctrl->nparts, 0.0, "mytpwgts");
+  for (i=0; i<ctrl->nparts; i++)
+    for (j=0; j<ncon; j++)
+      mytpwgts[i] += ctrl->tpwgts[i*ncon+j];
+  for (i=0; i<ctrl->nparts; i++)
+    mytpwgts[i] /= (float)ncon;
+
+  /* Go into the recursive bisection */
+  /* ADD: consider changing this to breadth-first type bisection */
+  moptions[0] = 0;
+  moptions[7] = ctrl->sync + (ctrl->mype % ngroups) + 1;
+
+  lnparts = ctrl->nparts;
+  fpart = fpe = 0;
+  lnpes = npes;
+  while (lnpes > 1 && lnparts > 1) {
+    /* Determine the weights of the partitions */
+    mytpwgts2[0] = ssum(lnparts/2, mytpwgts+fpart);
+    mytpwgts2[1] = 1.0-mytpwgts2[0];
+
+    if (ncon == 1)
+      METIS_WPartGraphKway2(&agraph->nvtxs, agraph->xadj, agraph->adjncy,
+        agraph->vwgt, agraph->adjwgt, &wgtflag, &numflag, &twoparts, mytpwgts2,
+        moptions, &edgecut, part);
+    else {
+      METIS_mCPartGraphRecursive2(&agraph->nvtxs, &ncon, agraph->xadj,
+        agraph->adjncy, agraph->vwgt, agraph->adjwgt, &wgtflag, &numflag,
+        &twoparts, mytpwgts2, moptions, &edgecut, part);
+    }
+
+    wsum = ssum(lnparts/2, mytpwgts+fpart);
+    sscale(lnparts/2, 1.0/wsum, mytpwgts+fpart);
+    sscale(lnparts-lnparts/2, 1.0/(1.0-wsum), mytpwgts+fpart+lnparts/2);
+
+    /* I'm picking the left branch */
+    if (mype < fpe+lnpes/2) {
+      Moc_KeepPart(agraph, wspace, part, 0);
+      lnpes = lnpes/2;
+      lnparts = lnparts/2;
+    }
+    else {
+      Moc_KeepPart(agraph, wspace, part, 1);
+      fpart = fpart + lnparts/2;
+      fpe = fpe + lnpes/2;
+      lnpes = lnpes - lnpes/2;
+      lnparts = lnparts - lnparts/2;
+    }
+  }
+
+  /* In case npes is greater than or equal to nparts */
+  if (lnparts == 1) {
+    /* Only the first process will assign labels (for the reduction to work) */
+    if (mype == fpe) {
+      for (i=0; i<agraph->nvtxs; i++) 
+        gwhere0[agraph->label[i]] = fpart;
+    }
+  }
+  /* In case npes is smaller than nparts */
+  else {
+    if (ncon == 1)
+      METIS_WPartGraphKway2(&agraph->nvtxs, agraph->xadj, agraph->adjncy,
+      agraph->vwgt, agraph->adjwgt, &wgtflag, &numflag, &lnparts, mytpwgts+fpart,
+      moptions, &edgecut, part);
+    else
+      METIS_mCPartGraphRecursive2(&agraph->nvtxs, &ncon, agraph->xadj,
+      agraph->adjncy, agraph->vwgt, agraph->adjwgt, &wgtflag, &numflag,
+      &lnparts, mytpwgts+fpart, moptions, &edgecut, part);
+
+    for (i=0; i<agraph->nvtxs; i++) 
+      gwhere0[agraph->label[i]] = fpart + part[i];
+  }
+
+  MPI_Allreduce((void *)gwhere0, (void *)gwhere1, gnvtxs, IDX_DATATYPE, MPI_SUM, ipcomm);
+
+  if (ngroups > 1) {
+    tmpxadj = agraph->xadj;
+    tmpadjncy = agraph->adjncy;
+    tmpadjwgt = agraph->adjwgt;
+    tmpvwgt = agraph->vwgt;
+    tmpwhere = agraph->where;
+    agraph->xadj = xadj;
+    agraph->adjncy = adjncy;
+    agraph->adjwgt = adjwgt;
+    agraph->vwgt = vwgt;
+    agraph->where = gwhere1;
+    agraph->vwgt = vwgt;
+    agraph->nvtxs = gnvtxs;
+    Moc_ComputeSerialBalance(ctrl, agraph, gwhere1, lbvec);
+    lbsum = ssum(ncon, lbvec);
+
+    edgecut = ComputeSerialEdgeCut(agraph);
+    MPI_Allreduce((void *)&edgecut, (void *)&max_cut, 1, MPI_INT, MPI_MAX, ctrl->gcomm);
+    MPI_Allreduce((void *)&lbsum, (void *)&min_lbsum, 1, MPI_FLOAT, MPI_MIN, ctrl->gcomm);
+
+    lpesum.sum = lbsum;
+    if (min_lbsum < UNBALANCE_FRACTION * (float)(ncon)) {
+      if (lbsum < UNBALANCE_FRACTION * (float)(ncon))
+        lpesum.sum = (float) (edgecut);
+      else
+        lpesum.sum = (float) (max_cut);
+    } 
+    
+    MPI_Comm_rank(ctrl->gcomm, &(lpesum.rank));
+    MPI_Allreduce((void *)&lpesum, (void *)&gpesum, 1, MPI_FLOAT_INT, MPI_MINLOC, ctrl->gcomm);
+    MPI_Bcast((void *)gwhere1, gnvtxs, IDX_DATATYPE, gpesum.rank, ctrl->gcomm);
+
+    agraph->xadj = tmpxadj;
+    agraph->adjncy = tmpadjncy;
+    agraph->adjwgt = tmpadjwgt;
+    agraph->vwgt = tmpvwgt;
+    agraph->where = tmpwhere;
+  }
+
+  idxcopy(graph->nvtxs, gwhere1+graph->vtxdist[ctrl->mype], graph->where);
+
+  FreeGraph(agraph);
+  MPI_Comm_free(&ipcomm);
+  GKfree((void **)&gwhere0, (void **)&gwhere1, (void **)&mytpwgts, (void **)&part, (void **)&xadj, (void **)&adjncy, (void **)&adjwgt, (void **)&vwgt, LTERM);
+
+  IFSET(ctrl->dbglvl, DBG_TIME, MPI_Barrier(ctrl->comm));
+  IFSET(ctrl->dbglvl, DBG_TIME, stoptimer(ctrl->InitPartTmr));
+
+}
+
+
+/*************************************************************************
+* This function keeps one parts
+**************************************************************************/
+void Moc_KeepPart(GraphType *graph, WorkSpaceType *wspace, idxtype *part, int mypart)
+{
+  int h, i, j, k;
+  int nvtxs, ncon, mynvtxs, mynedges;
+  idxtype *xadj, *vwgt, *adjncy, *adjwgt, *label;
+  idxtype *rename;
+
+  nvtxs = graph->nvtxs;
+  ncon = graph->ncon;
+  xadj = graph->xadj;
+  vwgt = graph->vwgt;
+  adjncy = graph->adjncy;
+  adjwgt = graph->adjwgt;
+  label = graph->label;
+
+  rename = idxmalloc(nvtxs, "Moc_KeepPart: rename");
+ 
+  for (mynvtxs=0, i=0; i<nvtxs; i++) {
+    if (part[i] == mypart)
+      rename[i] = mynvtxs++;
+  }
+
+  for (mynvtxs=0, mynedges=0, j=xadj[0], i=0; i<nvtxs; i++) {
+    if (part[i] == mypart) {
+      for (; j<xadj[i+1]; j++) {
+        k = adjncy[j];
+        if (part[k] == mypart) {
+          adjncy[mynedges] = rename[k];
+          adjwgt[mynedges++] = adjwgt[j];
+        }
+      }
+      j = xadj[i+1];  /* Save xadj[i+1] for later use */
+
+      for (h=0; h<ncon; h++)
+        vwgt[mynvtxs*ncon+h] = vwgt[i*ncon+h];
+      label[mynvtxs] = label[i];
+      xadj[++mynvtxs] = mynedges;
+
+    }
+    else {
+      j = xadj[i+1];  /* Save xadj[i+1] for later use */
+    }
+  }
+
+  graph->nvtxs = mynvtxs;
+  graph->nedges = mynedges;
+
+  free(rename);
+}
+
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/kmetis.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/kmetis.c
new file mode 100644
index 0000000..2ad20e3
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/kmetis.c
@@ -0,0 +1,274 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * kmetis.c
+ *
+ * This is the entry point of Moc_PARMETIS_PartGraphKway
+ *
+ * Started 10/19/96
+ * George
+ *
+ * $Id: kmetis.c,v 1.9 2003/07/31 16:27:27 karypis Exp $
+ *
+ */
+
+#include <parmetislib.h>
+
+
+/***********************************************************************************
+* This function is the entry point of the parallel k-way multilevel partitionioner. 
+* This function assumes nothing about the graph distribution.
+* It is the general case.
+************************************************************************************/
+void ParMETIS_V3_PartKway(idxtype *vtxdist, idxtype *xadj, idxtype *adjncy, idxtype *vwgt,
+              idxtype *adjwgt, int *wgtflag, int *numflag, int *ncon, int *nparts, 
+	      float *tpwgts, float *ubvec, int *options, int *edgecut, idxtype *part, 
+	      MPI_Comm *comm)
+{
+  int h, i;
+  int nvtxs = -1, npes, mype;
+  CtrlType ctrl;
+  WorkSpaceType wspace;
+  GraphType *graph;
+  float avg, maximb, *mytpwgts;
+  int moptions[10];
+  int seed, dbglvl = 0;
+  int iwgtflag, inumflag, incon, inparts, ioptions[10];
+  float *itpwgts, iubvec[MAXNCON];
+
+  MPI_Comm_size(*comm, &npes);
+  MPI_Comm_rank(*comm, &mype);
+
+
+  /********************************/
+  /* Try and take care bad inputs */
+  /********************************/
+  if (options != NULL && options[0] == 1)
+    dbglvl = options[PMV3_OPTION_DBGLVL];
+
+  CheckInputs(STATIC_PARTITION, npes, dbglvl, wgtflag, &iwgtflag, numflag, &inumflag, ncon, 
+              &incon, nparts, &inparts, tpwgts, &itpwgts, ubvec, iubvec, NULL, NULL, 
+	      options, ioptions, part, comm);
+
+
+  /*********************************/
+  /* Take care the nparts = 1 case */
+  /*********************************/
+  if (inparts <= 1) {
+    idxset(vtxdist[mype+1]-vtxdist[mype], 0, part); 
+    *edgecut = 0;
+    return;
+  }
+
+  /******************************/
+  /* Take care of npes = 1 case */
+  /******************************/
+  if (npes == 1 && inparts > 1) {
+    moptions[0] = 0;
+    nvtxs = vtxdist[1];
+
+    if (incon == 1) {
+      METIS_WPartGraphKway(&nvtxs, xadj, adjncy, vwgt, adjwgt, &iwgtflag, &inumflag, 
+            &inparts, itpwgts, moptions, edgecut, part);
+    }
+    else {
+      /* ADD: this is because METIS does not support tpwgts for all constraints */
+      mytpwgts = fmalloc(inparts, "mytpwgts");
+      for (i=0; i<inparts; i++)
+        mytpwgts[i] = itpwgts[i*incon];
+
+      moptions[7] = -1;
+      METIS_mCPartGraphRecursive2(&nvtxs, &incon, xadj, adjncy, vwgt, adjwgt, &iwgtflag, 
+            &inumflag, &inparts, mytpwgts, moptions, edgecut, part);
+
+      free(mytpwgts);
+    }
+ 
+    return;
+  }
+
+
+  if (inumflag == 1) 
+    ChangeNumbering(vtxdist, xadj, adjncy, part, npes, mype, 1);
+
+  /*****************************/
+  /* Set up control structures */
+  /*****************************/
+  if (ioptions[0] == 1) {
+    dbglvl = ioptions[PMV3_OPTION_DBGLVL];
+    seed = ioptions[PMV3_OPTION_SEED];
+  }
+  else {
+    dbglvl = GLOBAL_DBGLVL;
+    seed = GLOBAL_SEED;
+  }
+  SetUpCtrl(&ctrl, inparts, dbglvl, *comm);
+  ctrl.CoarsenTo = amin(vtxdist[npes]+1, 25*incon*amax(npes, inparts));
+  ctrl.seed = (seed == 0) ? mype : seed*mype;
+  ctrl.sync = GlobalSEMax(&ctrl, seed);
+  ctrl.partType = STATIC_PARTITION;
+  ctrl.ps_relation = -1;
+  ctrl.tpwgts = itpwgts;
+  scopy(incon, iubvec, ctrl.ubvec);
+
+  graph = Moc_SetUpGraph(&ctrl, incon, vtxdist, xadj, vwgt, adjncy, adjwgt, &iwgtflag);
+
+  PreAllocateMemory(&ctrl, graph, &wspace);
+
+  IFSET(ctrl.dbglvl, DBG_TIME, InitTimers(&ctrl));
+  IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
+  IFSET(ctrl.dbglvl, DBG_TIME, starttimer(ctrl.TotalTmr));
+
+  /*******************************************/
+  /* Check for funny cases                   */
+  /* 	-graph with no edges                 */
+  /* 	-graph with self edges               */
+  /* 	-graph with poor vertex distribution */
+  /* 	-graph with less than 2*npe nodes    */
+  /*******************************************/
+  if (vtxdist[npes] < SMALLGRAPH || vtxdist[npes] < npes*20 || GlobalSESum(&ctrl, graph->nedges) == 0) {
+    IFSET(ctrl.dbglvl, DBG_INFO, rprintf(&ctrl, "Partitioning a graph of size %d serially\n", vtxdist[npes]));
+    PartitionSmallGraph(&ctrl, graph, &wspace);
+  }
+  else {
+    /***********************/
+    /* Partition the graph */
+    /***********************/
+    Moc_Global_Partition(&ctrl, graph, &wspace);
+    ParallelReMapGraph(&ctrl, graph, &wspace);
+  }
+
+  IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
+  IFSET(ctrl.dbglvl, DBG_TIME, stoptimer(ctrl.TotalTmr));
+
+  idxcopy(graph->nvtxs, graph->where, part);
+  *edgecut = graph->mincut;
+
+  /*******************/
+  /* Print out stats */
+  /*******************/
+  IFSET(ctrl.dbglvl, DBG_TIME, PrintTimingInfo(&ctrl));
+  IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
+
+  if (ctrl.dbglvl&DBG_INFO) {
+    rprintf(&ctrl, "Final %d-way CUT: %6d \tBalance: ", inparts, graph->mincut);
+    avg = 0.0;
+    for (h=0; h<incon; h++) {
+      maximb = 0.0;
+      for (i=0; i<inparts; i++)
+        maximb = amax(maximb, graph->gnpwgts[i*incon+h]/itpwgts[i*incon+h]);
+      avg += maximb;
+      rprintf(&ctrl, "%.3f ", maximb);
+    }
+    rprintf(&ctrl, "  avg: %.3f\n", avg/(float)incon);
+  }
+
+  GKfree((void **)&itpwgts, (void **)&graph->lnpwgts, (void **)&graph->gnpwgts, (void **)&graph->nvwgt, LTERM);
+  FreeInitialGraphAndRemap(graph, iwgtflag);
+  FreeWSpace(&wspace);
+  FreeCtrl(&ctrl);
+
+  if (inumflag == 1) 
+    ChangeNumbering(vtxdist, xadj, adjncy, part, npes, mype, 0);
+
+}
+
+
+
+/*************************************************************************
+* This function is the driver to the multi-constraint partitioning algorithm.
+**************************************************************************/
+void Moc_Global_Partition(CtrlType *ctrl, GraphType *graph, WorkSpaceType *wspace)
+{
+  int i, ncon, nparts;
+  float ftmp, ubavg, lbavg, lbvec[MAXNCON];
+ 
+  ncon = graph->ncon;
+  nparts = ctrl->nparts;
+  ubavg = savg(graph->ncon, ctrl->ubvec);
+
+  SetUp(ctrl, graph, wspace);
+
+  if (ctrl->dbglvl&DBG_PROGRESS) {
+    rprintf(ctrl, "[%6d %8d %5d %5d] [%d] [", graph->gnvtxs, GlobalSESum(ctrl, graph->nedges),
+	    GlobalSEMin(ctrl, graph->nvtxs), GlobalSEMax(ctrl, graph->nvtxs), ctrl->CoarsenTo);
+    for (i=0; i<ncon; i++)
+      rprintf(ctrl, " %.3f", GlobalSEMinFloat(ctrl,graph->nvwgt[samin_strd(graph->nvtxs, graph->nvwgt+i, ncon)*ncon+i]));  
+    rprintf(ctrl, "] [");
+    for (i=0; i<ncon; i++)
+      rprintf(ctrl, " %.3f", GlobalSEMaxFloat(ctrl, graph->nvwgt[samax_strd(graph->nvtxs, graph->nvwgt+i, ncon)*ncon+i]));  
+    rprintf(ctrl, "]\n");
+  }
+
+  if (graph->gnvtxs < 1.3*ctrl->CoarsenTo ||
+	(graph->finer != NULL &&
+	graph->gnvtxs > graph->finer->gnvtxs*COARSEN_FRACTION)) {
+
+    /* Done with coarsening. Find a partition */
+    graph->where = idxmalloc(graph->nvtxs+graph->nrecv, "graph->where");
+    Moc_InitPartition_RB(ctrl, graph, wspace);
+
+    if (ctrl->dbglvl&DBG_PROGRESS) {
+      Moc_ComputeParallelBalance(ctrl, graph, graph->where, lbvec);
+      rprintf(ctrl, "nvtxs: %10d, balance: ", graph->gnvtxs);
+      for (i=0; i<graph->ncon; i++) 
+        rprintf(ctrl, "%.3f ", lbvec[i]);
+      rprintf(ctrl, "\n");
+    }
+
+    /* In case no coarsening took place */
+    if (graph->finer == NULL) {
+      Moc_ComputePartitionParams(ctrl, graph, wspace);
+      Moc_KWayFM(ctrl, graph, wspace, NGR_PASSES);
+    }
+  }
+  else {
+    Moc_GlobalMatch_Balance(ctrl, graph, wspace);
+
+    Moc_Global_Partition(ctrl, graph->coarser, wspace);
+
+    Moc_ProjectPartition(ctrl, graph, wspace);
+    Moc_ComputePartitionParams(ctrl, graph, wspace);
+
+    if (graph->ncon > 1 && graph->level < 3) {
+      for (i=0; i<ncon; i++) {
+        ftmp = ssum_strd(nparts, graph->gnpwgts+i, ncon);
+        if (ftmp != 0.0)
+          lbvec[i] = (float)(nparts) *
+          graph->gnpwgts[samax_strd(nparts, graph->gnpwgts+i, ncon)*ncon+i]/ftmp;
+        else
+          lbvec[i] = 1.0;
+      }
+      lbavg = savg(graph->ncon, lbvec);
+
+      if (lbavg > ubavg + 0.035) {
+        if (ctrl->dbglvl&DBG_PROGRESS) {
+          Moc_ComputeParallelBalance(ctrl, graph, graph->where, lbvec);
+          rprintf(ctrl, "nvtxs: %10d, cut: %8d, balance: ", graph->gnvtxs, graph->mincut);
+          for (i=0; i<graph->ncon; i++) 
+            rprintf(ctrl, "%.3f ", lbvec[i]);
+          rprintf(ctrl, "\n");
+	}
+
+        Moc_KWayBalance(ctrl, graph, wspace, graph->ncon);
+      }
+    }
+
+    Moc_KWayFM(ctrl, graph, wspace, NGR_PASSES);
+
+    if (ctrl->dbglvl&DBG_PROGRESS) {
+      Moc_ComputeParallelBalance(ctrl, graph, graph->where, lbvec);
+      rprintf(ctrl, "nvtxs: %10d, cut: %8d, balance: ", graph->gnvtxs, graph->mincut);
+      for (i=0; i<graph->ncon; i++) 
+        rprintf(ctrl, "%.3f ", lbvec[i]);
+      rprintf(ctrl, "\n");
+    }
+
+    if (graph->level != 0)
+      GKfree((void **)&graph->lnpwgts, (void **)&graph->gnpwgts, LTERM);
+  }
+
+  return;
+}
+
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/kwaybalance.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/kwaybalance.c
new file mode 100644
index 0000000..003ec7c
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/kwaybalance.c
@@ -0,0 +1,456 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * mkwaybalance.c
+ *
+ * This file contains code that performs the k-way refinement
+ *
+ * Started 3/1/96
+ * George
+ *
+ * $Id: kwaybalance.c,v 1.2 2003/07/21 17:18:49 karypis Exp $
+ */
+
+#include <parmetislib.h>
+
+#define ProperSide(c, from, other) \
+              (((c) == 0 && (from)-(other) < 0) || ((c) == 1 && (from)-(other) > 0))
+
+/*************************************************************************
+* This function performs k-way refinement
+**************************************************************************/
+void Moc_KWayBalance(CtrlType *ctrl, GraphType *graph, WorkSpaceType *wspace, int npasses)
+{
+  int h, i, ii, iii, j, k, c;
+  int pass, nvtxs, nedges, ncon;
+  int nmoves, nmoved, nswaps;
+/*  int gnswaps; */
+  int me, firstvtx, lastvtx, yourlastvtx;
+  int from, to = -1, oldto, oldcut, mydomain, yourdomain, imbalanced;
+  int npes = ctrl->npes, mype = ctrl->mype, nparts = ctrl->nparts;
+  int nlupd, nsupd, nnbrs, nchanged;
+  idxtype *xadj, *ladjncy, *adjwgt, *vtxdist;
+  idxtype *where, *tmp_where, *moved;
+  float *lnpwgts, *gnpwgts;
+  idxtype *update, *supdate, *rupdate, *pe_updates;
+  idxtype *changed, *perm, *pperm, *htable;
+  idxtype *peind, *recvptr, *sendptr;
+  KeyValueType *swchanges, *rwchanges;
+  RInfoType *rinfo, *myrinfo, *tmp_myrinfo, *tmp_rinfo;
+  EdgeType *tmp_edegrees, *my_edegrees, *your_edegrees;
+  float lbvec[MAXNCON], *nvwgt, *badmaxpwgt, *ubvec, *tpwgts, lbavg, ubavg;
+  int *nupds_pe;
+/*  int ndirty, nclean, dptr; */
+
+  IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->KWayTmr));
+
+  /*************************/
+  /* set up common aliases */
+  /*************************/
+  nvtxs = graph->nvtxs;
+  nedges = graph->nedges;
+  ncon = graph->ncon;
+
+  vtxdist = graph->vtxdist;
+  xadj = graph->xadj;
+  ladjncy = graph->adjncy;
+  adjwgt = graph->adjwgt;
+
+  firstvtx = vtxdist[mype];
+  lastvtx = vtxdist[mype+1];
+
+  where = graph->where;
+  rinfo = graph->rinfo;
+  lnpwgts = graph->lnpwgts;
+  gnpwgts = graph->gnpwgts;
+  ubvec = ctrl->ubvec;
+  tpwgts = ctrl->tpwgts;
+
+  nnbrs = graph->nnbrs;
+  peind = graph->peind;
+  recvptr = graph->recvptr;
+  sendptr = graph->sendptr;
+
+  changed = idxmalloc(nvtxs, "KWR: changed");
+  rwchanges = wspace->pairs;
+  swchanges = rwchanges + recvptr[nnbrs];
+
+  /************************************/
+  /* set up important data structures */
+  /************************************/
+  perm = idxmalloc(nvtxs, "KWR: perm");
+  pperm = idxmalloc(nparts, "KWR: pperm");
+
+  update = idxmalloc(nvtxs, "KWR: update");
+  supdate = wspace->indices;
+  rupdate = supdate + recvptr[nnbrs];
+  nupds_pe = imalloc(npes, "KWR: nupds_pe");
+  htable = idxsmalloc(nvtxs+graph->nrecv, 0, "KWR: lhtable");
+  badmaxpwgt = fmalloc(nparts*ncon, "badmaxpwgt");
+
+  for (i=0; i<nparts; i++) {
+    for (h=0; h<ncon; h++) {
+      badmaxpwgt[i*ncon+h] = ubvec[h]*tpwgts[i*ncon+h];
+    }
+  }
+
+  moved = idxmalloc(nvtxs, "KWR: moved");
+  tmp_where = idxmalloc(nvtxs+graph->nrecv, "KWR: tmp_where");
+  tmp_rinfo = (RInfoType *)GKmalloc(sizeof(RInfoType)*nvtxs, "KWR: tmp_rinfo");
+  tmp_edegrees = (EdgeType *)GKmalloc(sizeof(EdgeType)*nedges, "KWR: tmp_edegrees");
+
+  idxcopy(nvtxs+graph->nrecv, where, tmp_where);
+  for (i=0; i<nvtxs; i++) {
+    tmp_rinfo[i].id = rinfo[i].id;
+    tmp_rinfo[i].ed = rinfo[i].ed;
+    tmp_rinfo[i].ndegrees = rinfo[i].ndegrees;
+    tmp_rinfo[i].degrees = tmp_edegrees+xadj[i];
+
+    for (j=0; j<rinfo[i].ndegrees; j++) {
+      tmp_rinfo[i].degrees[j].edge = rinfo[i].degrees[j].edge;
+      tmp_rinfo[i].degrees[j].ewgt = rinfo[i].degrees[j].ewgt;
+    }
+  }
+
+  nswaps = 0;
+  /*********************************************************/
+  /* perform a small number of passes through the vertices */
+  /*********************************************************/
+  for (pass=0; pass<npasses; pass++) {
+    oldcut = graph->mincut;
+    if (mype == 0)
+      RandomPermute(nparts, pperm, 1);
+    MPI_Bcast((void *)pperm, nparts, IDX_DATATYPE, 0, ctrl->comm);
+    FastRandomPermute(nvtxs, perm, 1);
+
+    /*****************************/
+    /* move dirty vertices first */
+    /*****************************/
+/*
+    ndirty = 0;
+    for (i=0; i<nvtxs; i++)
+      if (where[i] != mype)
+        ndirty++;
+
+    dptr = 0; 
+    for (i=0; i<nvtxs; i++)
+      if (where[i] != mype)
+        perm[dptr++] = i;
+      else
+        perm[ndirty++] = i;
+ 
+    ASSERT(ctrl, ndirty == nvtxs);
+    ndirty = dptr;
+    nclean = nvtxs-dptr;
+    FastRandomPermute(ndirty, perm, 0);
+    FastRandomPermute(nclean, perm+ndirty, 0);
+*/
+
+    /* check to see if the partitioning is imbalanced */
+    Moc_ComputeParallelBalance(ctrl, graph, graph->where, lbvec);
+    ubavg = savg(ncon, ubvec);
+    lbavg = savg(ncon, lbvec);
+    imbalanced = (lbavg > ubavg) ? 1 : 0;
+
+    for (c=0; c<2; c++) {
+      nmoved = 0;
+
+      /**********************************************/
+      /* PASS ONE -- record stats for desired moves */
+      /**********************************************/
+      for (iii=0; iii<nvtxs; iii++) {
+        i = perm[iii];
+        from = tmp_where[i];
+        nvwgt = graph->nvwgt+i*ncon;
+
+        for (h=0; h<ncon; h++)
+          if (fabs(nvwgt[h]-gnpwgts[from*ncon+h]) < SMALLFLOAT)
+            break;
+
+        if (h < ncon) {
+          continue;
+        }
+
+        /* check for a potential improvement */
+        if (tmp_rinfo[i].ed >= tmp_rinfo[i].id) {
+          my_edegrees = tmp_rinfo[i].degrees;
+
+          for (k=0; k<tmp_rinfo[i].ndegrees; k++) {
+            to = my_edegrees[k].edge;
+            if (ProperSide(c, pperm[from], pperm[to]) &&
+            IsHBalanceBetterFT(ncon, gnpwgts+from*ncon, gnpwgts+to*ncon, nvwgt, ubvec)) {
+              break;
+            }
+          }
+          oldto = to;
+
+          /* check if a subdomain was found that fits */
+          if (k < tmp_rinfo[i].ndegrees) {
+            for (j=k+1; j<tmp_rinfo[i].ndegrees; j++) {
+              to = my_edegrees[j].edge;
+              if (ProperSide(c, pperm[from], pperm[to]) &&
+              IsHBalanceBetterTT(ncon, gnpwgts+oldto*ncon, gnpwgts+to*ncon, nvwgt, ubvec)){
+                k = j;
+                oldto = my_edegrees[k].edge;
+              }
+            }
+            to = oldto;
+
+            if (iii % npes == 0) {
+              /****************************************/
+              /* Update tmp arrays of the moved vertex */
+              /****************************************/
+              tmp_where[i] = to;
+              moved[nmoved++] = i;
+              for (h=0; h<ncon; h++) {
+                lnpwgts[to*ncon+h] += nvwgt[h];
+                lnpwgts[from*ncon+h] -= nvwgt[h];
+                gnpwgts[to*ncon+h] += nvwgt[h];
+                gnpwgts[from*ncon+h] -= nvwgt[h];
+              }
+
+              tmp_rinfo[i].ed += tmp_rinfo[i].id-my_edegrees[k].ewgt;
+              SWAP(tmp_rinfo[i].id, my_edegrees[k].ewgt, j);
+              if (my_edegrees[k].ewgt == 0) {
+                tmp_rinfo[i].ndegrees--;
+                my_edegrees[k].edge = my_edegrees[tmp_rinfo[i].ndegrees].edge;
+                my_edegrees[k].ewgt = my_edegrees[tmp_rinfo[i].ndegrees].ewgt;
+              }
+              else {
+                my_edegrees[k].edge = from;
+              }
+
+              /* Update the degrees of adjacent vertices */
+              for (j=xadj[i]; j<xadj[i+1]; j++) {
+                /* no need to bother about vertices on different pe's */
+                if (ladjncy[j] >= nvtxs)
+                  continue;
+
+                me = ladjncy[j];
+                mydomain = tmp_where[me];
+
+                myrinfo = tmp_rinfo+me;
+                your_edegrees = myrinfo->degrees;
+
+                if (mydomain == from) {
+                  INC_DEC(myrinfo->ed, myrinfo->id, adjwgt[j]);
+                }
+                else {
+                  if (mydomain == to) {
+                    INC_DEC(myrinfo->id, myrinfo->ed, adjwgt[j]);
+                  }
+                }
+
+                /* Remove contribution from the .ed of 'from' */
+                if (mydomain != from) {
+                  for (k=0; k<myrinfo->ndegrees; k++) {
+                    if (your_edegrees[k].edge == from) {
+                      if (your_edegrees[k].ewgt == adjwgt[j]) {
+                        myrinfo->ndegrees--;
+                        your_edegrees[k].edge = your_edegrees[myrinfo->ndegrees].edge;
+                        your_edegrees[k].ewgt = your_edegrees[myrinfo->ndegrees].ewgt;
+                      }
+                      else {
+                        your_edegrees[k].ewgt -= adjwgt[j];
+                      }
+                      break;
+                    }
+                  }
+                }
+
+                /* Add contribution to the .ed of 'to' */
+                if (mydomain != to) {
+                  for (k=0; k<myrinfo->ndegrees; k++) {
+                    if (your_edegrees[k].edge == to) {
+                      your_edegrees[k].ewgt += adjwgt[j];
+                      break;
+                    }
+                  }
+                  if (k == myrinfo->ndegrees) {
+                    your_edegrees[myrinfo->ndegrees].edge = to;
+                    your_edegrees[myrinfo->ndegrees++].ewgt = adjwgt[j];
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+
+      /*************************************************/
+      /* PASS TWO -- commit the remainder of the moves */
+      /*************************************************/
+      nlupd = nsupd = nmoves = nchanged = 0;
+      for (iii=0; iii<nmoved; iii++) {
+        i = moved[iii];
+        if (i == -1)
+          continue;
+
+        where[i] = tmp_where[i];
+
+        /* Make sure to update the vertex information */
+        if (htable[i] == 0) {
+          /* make sure you do the update */
+          htable[i] = 1;
+          update[nlupd++] = i;
+        }
+
+        /* Put the vertices adjacent to i into the update array */
+        for (j=xadj[i]; j<xadj[i+1]; j++) {
+          k = ladjncy[j];
+          if (htable[k] == 0) {
+            htable[k] = 1;
+            if (k<nvtxs)
+              update[nlupd++] = k;
+            else
+              supdate[nsupd++] = k;
+          }
+        }
+        nmoves++;
+        nswaps++;
+
+        /* check number of zero-gain moves */
+        for (k=0; k<rinfo[i].ndegrees; k++)
+          if (rinfo[i].degrees[k].edge == to)
+            break;
+
+        if (graph->pexadj[i+1]-graph->pexadj[i] > 0)
+          changed[nchanged++] = i;
+      }
+
+      /* Tell interested pe's the new where[] info for the interface vertices */
+      CommChangedInterfaceData(ctrl, graph, nchanged, changed, where,
+      swchanges, rwchanges, wspace->pv4); 
+
+
+      IFSET(ctrl->dbglvl, DBG_RMOVEINFO,
+      rprintf(ctrl, "\t[%d %d], [%.4f],  [%d %d %d]\n",
+      pass, c, badmaxpwgt[0],
+      GlobalSESum(ctrl, nmoves),
+      GlobalSESum(ctrl, nsupd),
+      GlobalSESum(ctrl, nlupd)));
+
+      /*-------------------------------------------------------------
+      / Time to communicate with processors to send the vertices
+      / whose degrees need to be update.
+      /-------------------------------------------------------------*/
+      /* Issue the receives first */
+      for (i=0; i<nnbrs; i++) {
+        MPI_Irecv((void *)(rupdate+sendptr[i]), sendptr[i+1]-sendptr[i], IDX_DATATYPE,
+                  peind[i], 1, ctrl->comm, ctrl->rreq+i);
+      }
+
+      /* Issue the sends next. This needs some preporcessing */
+      for (i=0; i<nsupd; i++) {
+        htable[supdate[i]] = 0;
+        supdate[i] = graph->imap[supdate[i]];
+      }
+      iidxsort(nsupd, supdate);
+
+      for (j=i=0; i<nnbrs; i++) {
+        yourlastvtx = vtxdist[peind[i]+1];
+        for (k=j; k<nsupd && supdate[k] < yourlastvtx; k++); 
+        MPI_Isend((void *)(supdate+j), k-j, IDX_DATATYPE, peind[i], 1, ctrl->comm, ctrl->sreq+i);
+        j = k;
+      }
+
+      /* OK, now get into the loop waiting for the send/recv operations to finish */
+      MPI_Waitall(nnbrs, ctrl->rreq, ctrl->statuses);
+      for (i=0; i<nnbrs; i++) 
+        MPI_Get_count(ctrl->statuses+i, IDX_DATATYPE, nupds_pe+i);
+      MPI_Waitall(nnbrs, ctrl->sreq, ctrl->statuses);
+
+
+      /*-------------------------------------------------------------
+      / Place the recieved to-be updated vertices into update[] 
+      /-------------------------------------------------------------*/
+      for (i=0; i<nnbrs; i++) {
+        pe_updates = rupdate+sendptr[i];
+        for (j=0; j<nupds_pe[i]; j++) {
+          k = pe_updates[j];
+          if (htable[k-firstvtx] == 0) {
+            htable[k-firstvtx] = 1;
+            update[nlupd++] = k-firstvtx;
+          }
+        }
+      }
+
+
+      /*-------------------------------------------------------------
+      / Update the rinfo of the vertices in the update[] array
+      /-------------------------------------------------------------*/
+      for (ii=0; ii<nlupd; ii++) {
+        i = update[ii];
+        ASSERT(ctrl, htable[i] == 1);
+
+        htable[i] = 0;
+
+        mydomain = where[i];
+        myrinfo = rinfo+i;
+        tmp_myrinfo = tmp_rinfo+i;
+        my_edegrees = myrinfo->degrees;
+        your_edegrees = tmp_myrinfo->degrees;
+
+        graph->lmincut -= myrinfo->ed;
+        myrinfo->ndegrees = 0;
+        myrinfo->id = 0;
+        myrinfo->ed = 0;
+
+        for (j=xadj[i]; j<xadj[i+1]; j++) {
+          yourdomain = where[ladjncy[j]];
+          if (mydomain != yourdomain) {
+            myrinfo->ed += adjwgt[j];
+
+            for (k=0; k<myrinfo->ndegrees; k++) {
+              if (my_edegrees[k].edge == yourdomain) {
+                my_edegrees[k].ewgt += adjwgt[j];
+                your_edegrees[k].ewgt += adjwgt[j];
+                break;
+              }
+            }
+            if (k == myrinfo->ndegrees) {
+              my_edegrees[k].edge = yourdomain;
+              my_edegrees[k].ewgt = adjwgt[j];
+              your_edegrees[k].edge = yourdomain;
+              your_edegrees[k].ewgt = adjwgt[j];
+              myrinfo->ndegrees++;
+            }
+            ASSERT(ctrl, myrinfo->ndegrees <= xadj[i+1]-xadj[i]);
+            ASSERT(ctrl, tmp_myrinfo->ndegrees <= xadj[i+1]-xadj[i]);
+
+          }
+          else {
+            myrinfo->id += adjwgt[j];
+          }
+        }
+        graph->lmincut += myrinfo->ed;
+
+        tmp_myrinfo->id = myrinfo->id;
+        tmp_myrinfo->ed = myrinfo->ed;
+        tmp_myrinfo->ndegrees = myrinfo->ndegrees;
+      }
+
+      /* finally, sum-up the partition weights */
+      MPI_Allreduce((void *)lnpwgts, (void *)gnpwgts, nparts*ncon,
+      MPI_FLOAT, MPI_SUM, ctrl->comm);
+    }
+    graph->mincut = GlobalSESum(ctrl, graph->lmincut)/2;
+
+    if (graph->mincut == oldcut)
+      break;
+  }
+
+/*
+  gnswaps = GlobalSESum(ctrl, nswaps);
+  if (mype == 0)
+    printf("niters: %d, nswaps: %d\n", pass+1, gnswaps);
+*/
+
+  GKfree((void **)&badmaxpwgt, (void **)&update, (void **)&nupds_pe, (void **)&htable, LTERM);
+  GKfree((void **)&changed, (void **)&pperm, (void **)&perm, (void **)&moved, LTERM);
+  GKfree((void **)&tmp_where, (void **)&tmp_rinfo, (void **)&tmp_edegrees, LTERM);
+
+  IFSET(ctrl->dbglvl, DBG_TIME, stoptimer(ctrl->KWayTmr));
+}
+
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/kwayfm.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/kwayfm.c
new file mode 100644
index 0000000..4d0849b
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/kwayfm.c
@@ -0,0 +1,599 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * mkwayfm.c
+ *
+ * This file contains code that performs the k-way refinement
+ *
+ * Started 3/1/96
+ * George
+ *
+ * $Id: kwayfm.c,v 1.3 2003/07/22 20:29:05 karypis Exp $
+ */
+
+#include <parmetislib.h>
+
+#define ProperSide(c, from, other) \
+              (((c) == 0 && (from)-(other) < 0) || ((c) == 1 && (from)-(other) > 0))
+
+/*************************************************************************
+* This function performs k-way refinement
+**************************************************************************/
+void Moc_KWayFM(CtrlType *ctrl, GraphType *graph, WorkSpaceType *wspace, int npasses)
+{
+  int h, i, ii, iii, j, k, c;
+  int pass, nvtxs, nedges, ncon;
+  int nmoves, nmoved, nswaps, nzgswaps;
+/*  int gnswaps, gnzgswaps; */
+  int me, firstvtx, lastvtx, yourlastvtx;
+  int from, to = -1, oldto, oldcut, mydomain, yourdomain, imbalanced, overweight;
+  int npes = ctrl->npes, mype = ctrl->mype, nparts = ctrl->nparts;
+  int nlupd, nsupd, nnbrs, nchanged;
+  idxtype *xadj, *ladjncy, *adjwgt, *vtxdist;
+  idxtype *where, *tmp_where, *moved;
+  float *lnpwgts, *gnpwgts, *ognpwgts, *pgnpwgts, *movewgts, *overfill;
+  idxtype *update, *supdate, *rupdate, *pe_updates;
+  idxtype *changed, *perm, *pperm, *htable;
+  idxtype *peind, *recvptr, *sendptr;
+  KeyValueType *swchanges, *rwchanges;
+  RInfoType *rinfo, *myrinfo, *tmp_myrinfo, *tmp_rinfo;
+  EdgeType *tmp_edegrees, *my_edegrees, *your_edegrees;
+  float lbvec[MAXNCON], *nvwgt, *badmaxpwgt, *ubvec, *tpwgts, lbavg, ubavg;
+  int *nupds_pe;
+
+  IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->KWayTmr));
+
+  /*************************/
+  /* set up common aliases */
+  /*************************/
+  nvtxs = graph->nvtxs;
+  nedges = graph->nedges;
+  ncon = graph->ncon;
+
+  vtxdist = graph->vtxdist;
+  xadj = graph->xadj;
+  ladjncy = graph->adjncy;
+  adjwgt = graph->adjwgt;
+
+  firstvtx = vtxdist[mype];
+  lastvtx = vtxdist[mype+1];
+
+  where   = graph->where;
+  rinfo   = graph->rinfo;
+  lnpwgts = graph->lnpwgts;
+  gnpwgts = graph->gnpwgts;
+  ubvec   = ctrl->ubvec;
+  tpwgts  = ctrl->tpwgts;
+
+  nnbrs = graph->nnbrs;
+  peind = graph->peind;
+  recvptr = graph->recvptr;
+  sendptr = graph->sendptr;
+
+  changed = idxmalloc(nvtxs, "KWR: changed");
+  rwchanges = wspace->pairs;
+  swchanges = rwchanges + recvptr[nnbrs];
+
+  /************************************/
+  /* set up important data structures */
+  /************************************/
+  perm = idxmalloc(nvtxs, "KWR: perm");
+  pperm = idxmalloc(nparts, "KWR: pperm");
+
+  update = idxmalloc(nvtxs, "KWR: update");
+  supdate = wspace->indices;
+  rupdate = supdate + recvptr[nnbrs];
+  nupds_pe = imalloc(npes, "KWR: nupds_pe");
+  htable = idxsmalloc(nvtxs+graph->nrecv, 0, "KWR: lhtable");
+  badmaxpwgt = fmalloc(nparts*ncon, "badmaxpwgt");
+
+  for (i=0; i<nparts; i++) {
+    for (h=0; h<ncon; h++) {
+      badmaxpwgt[i*ncon+h] = ubvec[h]*tpwgts[i*ncon+h];
+    }
+  }
+
+  movewgts = fmalloc(nparts*ncon, "KWR: movewgts");
+  ognpwgts = fmalloc(nparts*ncon, "KWR: ognpwgts");
+  pgnpwgts = fmalloc(nparts*ncon, "KWR: pgnpwgts");
+  overfill = fmalloc(nparts*ncon, "KWR: overfill");
+  moved = idxmalloc(nvtxs, "KWR: moved");
+  tmp_where = idxmalloc(nvtxs+graph->nrecv, "KWR: tmp_where");
+  tmp_rinfo = (RInfoType *)GKmalloc(sizeof(RInfoType)*nvtxs, "KWR: tmp_rinfo");
+  tmp_edegrees = (EdgeType *)GKmalloc(sizeof(EdgeType)*nedges, "KWR: tmp_edegrees");
+
+  idxcopy(nvtxs+graph->nrecv, where, tmp_where);
+  for (i=0; i<nvtxs; i++) {
+    tmp_rinfo[i].id = rinfo[i].id;
+    tmp_rinfo[i].ed = rinfo[i].ed;
+    tmp_rinfo[i].ndegrees = rinfo[i].ndegrees;
+    tmp_rinfo[i].degrees = tmp_edegrees+xadj[i];
+
+    for (j=0; j<rinfo[i].ndegrees; j++) {
+      tmp_rinfo[i].degrees[j].edge = rinfo[i].degrees[j].edge;
+      tmp_rinfo[i].degrees[j].ewgt = rinfo[i].degrees[j].ewgt;
+    }
+  }
+
+  nswaps = nzgswaps = 0;
+  /*********************************************************/
+  /* perform a small number of passes through the vertices */
+  /*********************************************************/
+  for (pass=0; pass<npasses; pass++) {
+    if (mype == 0)
+      RandomPermute(nparts, pperm, 1);
+    MPI_Bcast((void *)pperm, nparts, IDX_DATATYPE, 0, ctrl->comm);
+    FastRandomPermute(nvtxs, perm, 1);
+    oldcut = graph->mincut;
+
+    /* check to see if the partitioning is imbalanced */
+    Moc_ComputeParallelBalance(ctrl, graph, graph->where, lbvec);
+    ubavg = savg(ncon, ubvec);
+    lbavg = savg(ncon, lbvec);
+    imbalanced = (lbavg > ubavg) ? 1 : 0;
+
+    for (c=0; c<2; c++) {
+      scopy(ncon*nparts, gnpwgts, ognpwgts);
+      sset(ncon*nparts, 0.0, movewgts);
+      nmoved = 0;
+
+      /**********************************************/
+      /* PASS ONE -- record stats for desired moves */
+      /**********************************************/
+      for (iii=0; iii<nvtxs; iii++) {
+        i = perm[iii];
+        from = tmp_where[i];
+        nvwgt = graph->nvwgt+i*ncon;
+
+        for (h=0; h<ncon; h++)
+          if (fabs(nvwgt[h]-gnpwgts[from*ncon+h]) < SMALLFLOAT)
+            break;
+
+        if (h < ncon) {
+          continue;
+        }
+
+        /* check for a potential improvement */
+        if (tmp_rinfo[i].ed >= tmp_rinfo[i].id) {
+          my_edegrees = tmp_rinfo[i].degrees;
+
+          for (k=0; k<tmp_rinfo[i].ndegrees; k++) {
+            to = my_edegrees[k].edge;
+            if (ProperSide(c, pperm[from], pperm[to])) {
+              for (h=0; h<ncon; h++)
+                if (gnpwgts[to*ncon+h]+nvwgt[h] > badmaxpwgt[to*ncon+h] && nvwgt[h] > 0.0)
+                  break;
+
+              if (h == ncon)
+                break;
+            }
+          }
+          oldto = to;
+
+          /* check if a subdomain was found that fits */
+          if (k < tmp_rinfo[i].ndegrees) {
+            for (j=k+1; j<tmp_rinfo[i].ndegrees; j++) {
+              to = my_edegrees[j].edge;
+              if (ProperSide(c, pperm[from], pperm[to])) {
+                for (h=0; h<ncon; h++)
+                  if (gnpwgts[to*ncon+h]+nvwgt[h] > badmaxpwgt[to*ncon+h] && nvwgt[h] > 0.0)
+                    break;
+
+                if (h == ncon) {
+                  if (my_edegrees[j].ewgt > my_edegrees[k].ewgt ||
+                   (my_edegrees[j].ewgt == my_edegrees[k].ewgt &&
+                   IsHBalanceBetterTT(ncon,gnpwgts+oldto*ncon,gnpwgts+to*ncon,nvwgt,ubvec))){
+                    k = j;
+                    oldto = my_edegrees[k].edge;
+                  }
+                }
+              }
+            }
+            to = oldto;
+
+            if (my_edegrees[k].ewgt > tmp_rinfo[i].id ||
+            (my_edegrees[k].ewgt == tmp_rinfo[i].id &&
+            (imbalanced ||  graph->level > 3  || iii % 8 == 0) &&
+            IsHBalanceBetterFT(ncon,gnpwgts+from*ncon,gnpwgts+to*ncon,nvwgt,ubvec))){
+
+              /****************************************/
+              /* Update tmp arrays of the moved vertex */
+              /****************************************/
+              tmp_where[i] = to;
+              moved[nmoved++] = i;
+              for (h=0; h<ncon; h++) {
+                lnpwgts[to*ncon+h] += nvwgt[h];
+                lnpwgts[from*ncon+h] -= nvwgt[h];
+                gnpwgts[to*ncon+h] += nvwgt[h];
+                gnpwgts[from*ncon+h] -= nvwgt[h];
+                movewgts[to*ncon+h] += nvwgt[h];
+                movewgts[from*ncon+h] -= nvwgt[h];
+              }
+
+              tmp_rinfo[i].ed += tmp_rinfo[i].id-my_edegrees[k].ewgt;
+              SWAP(tmp_rinfo[i].id, my_edegrees[k].ewgt, j);
+              if (my_edegrees[k].ewgt == 0) {
+                tmp_rinfo[i].ndegrees--;
+                my_edegrees[k].edge = my_edegrees[tmp_rinfo[i].ndegrees].edge;
+                my_edegrees[k].ewgt = my_edegrees[tmp_rinfo[i].ndegrees].ewgt;
+              }
+              else {
+                my_edegrees[k].edge = from;
+              }
+
+              /* Update the degrees of adjacent vertices */
+              for (j=xadj[i]; j<xadj[i+1]; j++) {
+                /* no need to bother about vertices on different pe's */
+                if (ladjncy[j] >= nvtxs)
+                  continue;
+
+                me = ladjncy[j];
+                mydomain = tmp_where[me];
+
+                myrinfo = tmp_rinfo+me;
+                your_edegrees = myrinfo->degrees;
+
+                if (mydomain == from) {
+                  INC_DEC(myrinfo->ed, myrinfo->id, adjwgt[j]);
+                }
+                else {
+                  if (mydomain == to) {
+                    INC_DEC(myrinfo->id, myrinfo->ed, adjwgt[j]);
+                  }
+                }
+
+                /* Remove contribution from the .ed of 'from' */
+                if (mydomain != from) {
+                  for (k=0; k<myrinfo->ndegrees; k++) {
+                    if (your_edegrees[k].edge == from) {
+                      if (your_edegrees[k].ewgt == adjwgt[j]) {
+                        myrinfo->ndegrees--;
+                        your_edegrees[k].edge = your_edegrees[myrinfo->ndegrees].edge;
+                        your_edegrees[k].ewgt = your_edegrees[myrinfo->ndegrees].ewgt;
+                      }
+                      else {
+                        your_edegrees[k].ewgt -= adjwgt[j];
+                      }
+                      break;
+                    }
+                  }
+                }
+
+                /* Add contribution to the .ed of 'to' */
+                if (mydomain != to) {
+                  for (k=0; k<myrinfo->ndegrees; k++) {
+                    if (your_edegrees[k].edge == to) {
+                      your_edegrees[k].ewgt += adjwgt[j];
+                      break;
+                    }
+                  }
+                  if (k == myrinfo->ndegrees) {
+                    your_edegrees[myrinfo->ndegrees].edge = to;
+                    your_edegrees[myrinfo->ndegrees++].ewgt = adjwgt[j];
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+
+      /******************************************/
+      /* Let processors know the subdomain wgts */
+      /* if all proposed moves commit.          */
+      /******************************************/
+      MPI_Allreduce((void *)lnpwgts, (void *)pgnpwgts, nparts*ncon,
+      MPI_FLOAT, MPI_SUM, ctrl->comm);
+
+      /**************************/
+      /* compute overfill array */
+      /**************************/
+      overweight = 0;
+      for (j=0; j<nparts; j++) {
+        for (h=0; h<ncon; h++) {
+          if (pgnpwgts[j*ncon+h] > ognpwgts[j*ncon+h]) {
+            overfill[j*ncon+h] =
+            (pgnpwgts[j*ncon+h]-badmaxpwgt[j*ncon+h]) /
+            (pgnpwgts[j*ncon+h]-ognpwgts[j*ncon+h]);
+          }
+          else {
+            overfill[j*ncon+h] = 0.0;
+          }
+
+          overfill[j*ncon+h] = amax(overfill[j*ncon+h], 0.0);
+          overfill[j*ncon+h] *= movewgts[j*ncon+h];
+
+          if (overfill[j*ncon+h] > 0.0)
+            overweight = 1;
+
+          ASSERTP(ctrl, ognpwgts[j*ncon+h] <= badmaxpwgt[j*ncon+h] ||
+          pgnpwgts[j*ncon+h] <= ognpwgts[j*ncon+h],
+          (ctrl, "%.4f %.4f %.4f\n", ognpwgts[j*ncon+h],
+          badmaxpwgt[j*ncon+h], pgnpwgts[j*ncon+h]));
+        }
+      }
+
+      /****************************************************/
+      /* select moves to undo according to overfill array */
+      /****************************************************/
+      if (overweight == 1) {
+        for (iii=0; iii<nmoved; iii++) {
+          i = moved[iii];
+          oldto = tmp_where[i];
+          nvwgt = graph->nvwgt+i*ncon;
+          my_edegrees = tmp_rinfo[i].degrees;
+
+          for (k=0; k<tmp_rinfo[i].ndegrees; k++)
+            if (my_edegrees[k].edge == where[i])
+              break;
+
+          for (h=0; h<ncon; h++)
+            if (nvwgt[h] > 0.0 && overfill[oldto*ncon+h] > nvwgt[h]/4.0)
+              break;
+
+          /**********************************/
+          /* nullify this move if necessary */
+          /**********************************/
+          if (k != tmp_rinfo[i].ndegrees && h != ncon) {
+            moved[iii] = -1;
+            from = oldto;
+            to = where[i];
+
+            for (h=0; h<ncon; h++) {
+              overfill[oldto*ncon+h] = amax(overfill[oldto*ncon+h]-nvwgt[h], 0.0);
+            }
+
+            tmp_where[i] = to;
+            tmp_rinfo[i].ed += tmp_rinfo[i].id-my_edegrees[k].ewgt;
+            SWAP(tmp_rinfo[i].id, my_edegrees[k].ewgt, j);
+            if (my_edegrees[k].ewgt == 0) {
+              tmp_rinfo[i].ndegrees--;
+              my_edegrees[k].edge = my_edegrees[tmp_rinfo[i].ndegrees].edge;
+              my_edegrees[k].ewgt = my_edegrees[tmp_rinfo[i].ndegrees].ewgt;
+            }
+            else {
+              my_edegrees[k].edge = from;
+            }
+
+            for (h=0; h<ncon; h++) {
+              lnpwgts[to*ncon+h] += nvwgt[h];
+              lnpwgts[from*ncon+h] -= nvwgt[h];
+            }
+
+            /* Update the degrees of adjacent vertices */
+            for (j=xadj[i]; j<xadj[i+1]; j++) {
+              /* no need to bother about vertices on different pe's */
+              if (ladjncy[j] >= nvtxs)
+                continue;
+
+              me = ladjncy[j];
+              mydomain = tmp_where[me];
+
+              myrinfo = tmp_rinfo+me;
+              your_edegrees = myrinfo->degrees;
+
+              if (mydomain == from) {
+                INC_DEC(myrinfo->ed, myrinfo->id, adjwgt[j]);
+              }
+              else {
+                if (mydomain == to) {
+                  INC_DEC(myrinfo->id, myrinfo->ed, adjwgt[j]);
+                }
+              }
+
+              /* Remove contribution from the .ed of 'from' */
+              if (mydomain != from) {
+                for (k=0; k<myrinfo->ndegrees; k++) {
+                  if (your_edegrees[k].edge == from) {
+                    if (your_edegrees[k].ewgt == adjwgt[j]) {
+                      myrinfo->ndegrees--;
+                      your_edegrees[k].edge = your_edegrees[myrinfo->ndegrees].edge;
+                      your_edegrees[k].ewgt = your_edegrees[myrinfo->ndegrees].ewgt;
+                    }
+                    else {
+                      your_edegrees[k].ewgt -= adjwgt[j];
+                    }
+                    break;
+                  }
+                }
+              }
+
+              /* Add contribution to the .ed of 'to' */
+              if (mydomain != to) {
+                for (k=0; k<myrinfo->ndegrees; k++) {
+                  if (your_edegrees[k].edge == to) {
+                    your_edegrees[k].ewgt += adjwgt[j];
+                    break;
+                  }
+                }
+                if (k == myrinfo->ndegrees) {
+                  your_edegrees[myrinfo->ndegrees].edge = to;
+                  your_edegrees[myrinfo->ndegrees++].ewgt = adjwgt[j];
+                }
+              }
+            }
+          }
+        }
+      }
+
+      /*************************************************/
+      /* PASS TWO -- commit the remainder of the moves */
+      /*************************************************/
+      nlupd = nsupd = nmoves = nchanged = 0;
+      for (iii=0; iii<nmoved; iii++) {
+        i = moved[iii];
+        if (i == -1)
+          continue;
+
+        where[i] = tmp_where[i];
+
+        /* Make sure to update the vertex information */
+        if (htable[i] == 0) {
+          /* make sure you do the update */
+          htable[i] = 1;
+          update[nlupd++] = i;
+        }
+
+        /* Put the vertices adjacent to i into the update array */
+        for (j=xadj[i]; j<xadj[i+1]; j++) {
+          k = ladjncy[j];
+          if (htable[k] == 0) {
+            htable[k] = 1;
+            if (k<nvtxs)
+              update[nlupd++] = k;
+            else
+              supdate[nsupd++] = k;
+          }
+        }
+        nmoves++;
+        nswaps++;
+
+        /* check number of zero-gain moves */
+        for (k=0; k<rinfo[i].ndegrees; k++)
+          if (rinfo[i].degrees[k].edge == to)
+            break;
+        if (rinfo[i].id == rinfo[i].degrees[k].ewgt)
+          nzgswaps++;
+
+        if (graph->pexadj[i+1]-graph->pexadj[i] > 0)
+          changed[nchanged++] = i;
+      }
+
+      /* Tell interested pe's the new where[] info for the interface vertices */
+      CommChangedInterfaceData(ctrl, graph, nchanged, changed, where,
+      swchanges, rwchanges, wspace->pv4); 
+
+
+      IFSET(ctrl->dbglvl, DBG_RMOVEINFO,
+      rprintf(ctrl, "\t[%d %d], [%.4f],  [%d %d %d]\n",
+      pass, c, badmaxpwgt[0],
+      GlobalSESum(ctrl, nmoves),
+      GlobalSESum(ctrl, nsupd),
+      GlobalSESum(ctrl, nlupd)));
+
+      /*-------------------------------------------------------------
+      / Time to communicate with processors to send the vertices
+      / whose degrees need to be update.
+      /-------------------------------------------------------------*/
+      /* Issue the receives first */
+      for (i=0; i<nnbrs; i++) {
+        MPI_Irecv((void *)(rupdate+sendptr[i]), sendptr[i+1]-sendptr[i], IDX_DATATYPE,
+                  peind[i], 1, ctrl->comm, ctrl->rreq+i);
+      }
+
+      /* Issue the sends next. This needs some preporcessing */
+      for (i=0; i<nsupd; i++) {
+        htable[supdate[i]] = 0;
+        supdate[i] = graph->imap[supdate[i]];
+      }
+      iidxsort(nsupd, supdate);
+
+      for (j=i=0; i<nnbrs; i++) {
+        yourlastvtx = vtxdist[peind[i]+1];
+        for (k=j; k<nsupd && supdate[k] < yourlastvtx; k++); 
+        MPI_Isend((void *)(supdate+j), k-j, IDX_DATATYPE, peind[i], 1, ctrl->comm, ctrl->sreq+i);
+        j = k;
+      }
+
+      /* OK, now get into the loop waiting for the send/recv operations to finish */
+      MPI_Waitall(nnbrs, ctrl->rreq, ctrl->statuses);
+      for (i=0; i<nnbrs; i++) 
+        MPI_Get_count(ctrl->statuses+i, IDX_DATATYPE, nupds_pe+i);
+      MPI_Waitall(nnbrs, ctrl->sreq, ctrl->statuses);
+
+
+      /*-------------------------------------------------------------
+      / Place the recieved to-be updated vertices into update[] 
+      /-------------------------------------------------------------*/
+      for (i=0; i<nnbrs; i++) {
+        pe_updates = rupdate+sendptr[i];
+        for (j=0; j<nupds_pe[i]; j++) {
+          k = pe_updates[j];
+          if (htable[k-firstvtx] == 0) {
+            htable[k-firstvtx] = 1;
+            update[nlupd++] = k-firstvtx;
+          }
+        }
+      }
+
+
+      /*-------------------------------------------------------------
+      / Update the rinfo of the vertices in the update[] array
+      /-------------------------------------------------------------*/
+      for (ii=0; ii<nlupd; ii++) {
+        i = update[ii];
+        ASSERT(ctrl, htable[i] == 1);
+
+        htable[i] = 0;
+
+        mydomain = where[i];
+        myrinfo = rinfo+i;
+        tmp_myrinfo = tmp_rinfo+i;
+        my_edegrees = myrinfo->degrees;
+        your_edegrees = tmp_myrinfo->degrees;
+
+        graph->lmincut -= myrinfo->ed;
+        myrinfo->ndegrees = 0;
+        myrinfo->id = 0;
+        myrinfo->ed = 0;
+
+        for (j=xadj[i]; j<xadj[i+1]; j++) {
+          yourdomain = where[ladjncy[j]];
+          if (mydomain != yourdomain) {
+            myrinfo->ed += adjwgt[j];
+
+            for (k=0; k<myrinfo->ndegrees; k++) {
+              if (my_edegrees[k].edge == yourdomain) {
+                my_edegrees[k].ewgt += adjwgt[j];
+                your_edegrees[k].ewgt += adjwgt[j];
+                break;
+              }
+            }
+            if (k == myrinfo->ndegrees) {
+              my_edegrees[k].edge = yourdomain;
+              my_edegrees[k].ewgt = adjwgt[j];
+              your_edegrees[k].edge = yourdomain;
+              your_edegrees[k].ewgt = adjwgt[j];
+              myrinfo->ndegrees++;
+            }
+            ASSERT(ctrl, myrinfo->ndegrees <= xadj[i+1]-xadj[i]);
+            ASSERT(ctrl, tmp_myrinfo->ndegrees <= xadj[i+1]-xadj[i]);
+
+          }
+          else {
+            myrinfo->id += adjwgt[j];
+          }
+        }
+        graph->lmincut += myrinfo->ed;
+
+        tmp_myrinfo->id = myrinfo->id;
+        tmp_myrinfo->ed = myrinfo->ed;
+        tmp_myrinfo->ndegrees = myrinfo->ndegrees;
+      }
+
+      /* finally, sum-up the partition weights */
+      MPI_Allreduce((void *)lnpwgts, (void *)gnpwgts, nparts*ncon,
+      MPI_FLOAT, MPI_SUM, ctrl->comm);
+    }
+    graph->mincut = GlobalSESum(ctrl, graph->lmincut)/2;
+
+    if (graph->mincut == oldcut)
+      break;
+  }
+
+/*
+  gnswaps = GlobalSESum(ctrl, nswaps);
+  gnzgswaps = GlobalSESum(ctrl, nzgswaps);
+  if (mype == 0)
+    printf("niters: %d, nswaps: %d, nzgswaps: %d\n", pass+1, gnswaps, gnzgswaps);
+*/
+
+  GKfree((void **)&badmaxpwgt, (void **)&update, (void **)&nupds_pe, (void **)&htable, LTERM);
+  GKfree((void **)&changed, (void **)&pperm, (void **)&perm, (void **)&moved, LTERM);
+  GKfree((void **)&pgnpwgts, (void **)&ognpwgts, (void **)&overfill, (void **)&movewgts, LTERM);
+  GKfree((void **)&tmp_where, (void **)&tmp_rinfo, (void **)&tmp_edegrees, LTERM);
+
+  IFSET(ctrl->dbglvl, DBG_TIME, stoptimer(ctrl->KWayTmr));
+}
+
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/kwayrefine.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/kwayrefine.c
new file mode 100644
index 0000000..e4b776e
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/kwayrefine.c
@@ -0,0 +1,239 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * medge_refine.c
+ *
+ * This file contains code that performs the k-way refinement
+ *
+ * Started 3/1/96
+ * George
+ *
+ * $Id: kwayrefine.c,v 1.2 2003/07/21 17:18:49 karypis Exp $
+ */
+
+#include <parmetislib.h>
+
+#define ProperSide(c, from, other) \
+              (((c) == 0 && (from)-(other) < 0) || ((c) == 1 && (from)-(other) > 0))
+
+/*************************************************************************
+* This function projects a partition.
+**************************************************************************/
+void Moc_ProjectPartition(CtrlType *ctrl, GraphType *graph, WorkSpaceType *wspace)
+{
+  int i, nvtxs, nnbrs = -1, firstvtx, cfirstvtx;
+  idxtype *match, *cmap, *where, *cwhere;
+  idxtype *peind, *slens = NULL, *rlens = NULL;
+  KeyValueType *rcand, *scand = NULL;
+  GraphType *cgraph;
+
+
+  IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->ProjectTmr));
+
+  cgraph = graph->coarser;
+  cwhere = cgraph->where;
+  cfirstvtx = cgraph->vtxdist[ctrl->mype];
+
+  nvtxs = graph->nvtxs;
+  match = graph->match;
+  cmap = graph->cmap;
+  where = graph->where = idxmalloc(nvtxs+graph->nrecv, "ProjectPartition: graph->where");
+  firstvtx = graph->vtxdist[ctrl->mype];
+
+
+  if (graph->match_type == MATCH_GLOBAL) {  /* Only if global matching is on */
+    /*------------------------------------------------------------
+    / Start the transmission of the remote where information 
+    /------------------------------------------------------------*/
+    scand = wspace->pairs;
+    nnbrs = graph->nnbrs;
+    peind = graph->peind;
+    slens = graph->slens;
+    rlens = graph->rlens;
+    rcand = graph->rcand;
+
+    /* Issue the receives first */
+    for (i=0; i<nnbrs; i++) {
+      if (slens[i+1]-slens[i] > 0) /* Issue a receive only if you are getting something */
+        MPI_Irecv((void *)(scand+slens[i]), 2*(slens[i+1]-slens[i]), IDX_DATATYPE, peind[i], 1, ctrl->comm, ctrl->rreq+i);
+    }
+
+#ifdef DEBUG_PROJECT
+    PrintPairs(ctrl, rlens[nnbrs], rcand, "rcand"); 
+#endif
+
+    /* Put the where[rcand[].key] into the val field */
+    for (i=0; i<rlens[nnbrs]; i++) {
+      ASSERT(ctrl, rcand[i].val >= 0 && rcand[i].val < cgraph->nvtxs);
+      rcand[i].val = cwhere[rcand[i].val];
+    }
+
+#ifdef DEBUG_PROJECT
+    PrintPairs(ctrl, rlens[nnbrs], rcand, "rcand");
+    PrintVector(ctrl, nvtxs, firstvtx, cmap, "cmap");
+#endif
+
+    /* Issue the sends next */
+    for (i=0; i<nnbrs; i++) {
+      if (rlens[i+1]-rlens[i] > 0) /* Issue a send only if you are sending something */
+        MPI_Isend((void *)(rcand+rlens[i]), 2*(rlens[i+1]-rlens[i]), IDX_DATATYPE, peind[i], 1, ctrl->comm, ctrl->sreq+i);
+    }
+  }
+
+  /*------------------------------------------------------------
+  / Project local vertices first
+  /------------------------------------------------------------*/
+  for (i=0; i<nvtxs; i++) {
+    if (match[i] >= KEEP_BIT) {
+      ASSERT(ctrl, cmap[i]-cfirstvtx>=0 && cmap[i]-cfirstvtx<cgraph->nvtxs);
+      where[i] = cwhere[cmap[i]-cfirstvtx];
+    }
+  }
+
+  if (graph->match_type == MATCH_GLOBAL) {  /* Only if global matching is on */
+    /*------------------------------------------------------------
+    / Wait for the nonblocking operations to finish
+    /------------------------------------------------------------*/
+    for (i=0; i<nnbrs; i++) {
+      if (rlens[i+1]-rlens[i] > 0)  
+        MPI_Wait(ctrl->sreq+i, &ctrl->status);
+    }
+    for (i=0; i<nnbrs; i++) {
+      if (slens[i+1]-slens[i] > 0)  
+        MPI_Wait(ctrl->rreq+i, &ctrl->status);
+    }
+
+#ifdef DEBUG_PROJECT
+    PrintPairs(ctrl, slens[nnbrs], scand, "scand"); 
+#endif
+
+    /*------------------------------------------------------------
+    / Project received vertices now
+    /------------------------------------------------------------*/
+    for (i=0; i<slens[nnbrs]; i++) {
+      ASSERTP(ctrl, scand[i].key-firstvtx>=0 && scand[i].key-firstvtx<graph->nvtxs, (ctrl, "%d %d %d\n", scand[i].key, firstvtx, graph->nvtxs));
+      where[scand[i].key-firstvtx] = scand[i].val;
+    }
+  }
+
+
+  FreeGraph(graph->coarser);
+  graph->coarser = NULL;
+
+  IFSET(ctrl->dbglvl, DBG_TIME, stoptimer(ctrl->ProjectTmr));
+}
+
+
+
+/*************************************************************************
+* This function computes the initial id/ed 
+**************************************************************************/
+void Moc_ComputePartitionParams(CtrlType *ctrl, GraphType *graph, WorkSpaceType *wspace)
+{
+  int h, i, j, k;
+  int nvtxs, ncon;
+  int firstvtx, lastvtx;
+  idxtype *xadj, *ladjncy, *adjwgt, *vtxdist;
+  float *lnpwgts, *gnpwgts;
+  idxtype *where, *swhere, *rwhere;
+  RInfoType *rinfo, *myrinfo;
+  EdgeType *edegrees;
+  int me, other;
+
+  IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->KWayInitTmr));
+
+
+  nvtxs = graph->nvtxs;
+  ncon = graph->ncon;
+
+  vtxdist = graph->vtxdist;
+  xadj = graph->xadj;
+  ladjncy = graph->adjncy;
+  adjwgt = graph->adjwgt;
+
+  where = graph->where;
+  rinfo = graph->rinfo = (RInfoType *)GKmalloc(sizeof(RInfoType)*nvtxs, "CPP: rinfo");
+  lnpwgts = graph->lnpwgts = fmalloc(ctrl->nparts*ncon, "CPP: lnpwgts");
+  gnpwgts = graph->gnpwgts = fmalloc(ctrl->nparts*ncon, "CPP: gnpwgts");
+
+  sset(ctrl->nparts*ncon, 0, lnpwgts);
+
+  firstvtx = vtxdist[ctrl->mype];
+  lastvtx = vtxdist[ctrl->mype+1];
+
+  /*------------------------------------------------------------
+  / Send/Receive the where information of interface vertices
+  /------------------------------------------------------------*/
+  swhere = wspace->indices;
+  rwhere = where + nvtxs;
+
+  CommInterfaceData(ctrl, graph, where, swhere, rwhere); 
+
+#ifdef DEBUG_COMPUTEPPARAM
+  PrintVector(ctrl, nvtxs, firstvtx, where, "where");
+#endif
+
+  ASSERT(ctrl, wspace->nlarge >= xadj[nvtxs]);
+
+  /*------------------------------------------------------------
+  / Compute now the id/ed degrees
+  /------------------------------------------------------------*/
+  graph->lmincut = 0;
+  for (i=0; i<nvtxs; i++) {
+    me = where[i];
+    myrinfo = rinfo+i;
+
+    for (h=0; h<ncon; h++)
+      lnpwgts[me*ncon+h] += graph->nvwgt[i*ncon+h];
+
+    myrinfo->degrees = wspace->degrees + xadj[i];
+    myrinfo->ndegrees = myrinfo->id = myrinfo->ed = 0;
+
+    for (j=xadj[i]; j<xadj[i+1]; j++) {
+      if (me == where[ladjncy[j]])
+        myrinfo->id += adjwgt[j];
+      else
+        myrinfo->ed += adjwgt[j];
+    }
+
+
+    if (myrinfo->ed > 0) {  /* Time to do some serious work */
+      graph->lmincut += myrinfo->ed;
+      edegrees = myrinfo->degrees;
+
+      for (j=xadj[i]; j<xadj[i+1]; j++) {
+        other = where[ladjncy[j]];
+        if (me != other) {
+          for (k=0; k<myrinfo->ndegrees; k++) {
+            if (edegrees[k].edge == other) {
+              edegrees[k].ewgt += adjwgt[j];
+              break;
+            }
+          }
+          if (k == myrinfo->ndegrees) {
+            edegrees[k].edge = other;
+            edegrees[k].ewgt = adjwgt[j];
+            myrinfo->ndegrees++;
+          }
+          ASSERT(ctrl, myrinfo->ndegrees <= xadj[i+1]-xadj[i]);
+        }
+      }
+    }
+  }
+
+#ifdef DEBUG_COMPUTEPPARAM
+  PrintVector(ctrl, ctrl->nparts*ncon, 0, lnpwgts, "lnpwgts");
+#endif
+
+  /* Finally, sum-up the partition weights */
+  MPI_Allreduce((void *)lnpwgts, (void *)gnpwgts, ctrl->nparts*ncon, MPI_FLOAT, MPI_SUM, ctrl->comm);
+
+  graph->mincut = GlobalSESum(ctrl, graph->lmincut)/2;
+
+#ifdef DEBUG_COMPUTEPPARAM
+  PrintVector(ctrl, ctrl->nparts*ncon, 0, gnpwgts, "gnpwgts");
+#endif
+
+  IFSET(ctrl->dbglvl, DBG_TIME, stoptimer(ctrl->KWayInitTmr));
+}
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/lmatch.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/lmatch.c
new file mode 100644
index 0000000..d8601ef
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/lmatch.c
@@ -0,0 +1,364 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * coarsen.c
+ *
+ * This file contains code that finds a matching and performs the coarsening
+ *
+ * Started 2/22/96
+ * George
+ *
+ * $Id: lmatch.c,v 1.2 2003/07/21 17:18:50 karypis Exp $
+ *
+ */
+
+#include <parmetislib.h>
+
+
+/*************************************************************************
+* This function finds a HEM matching between local vertices only
+**************************************************************************/
+void Mc_LocalMatch_HEM(CtrlType *ctrl, GraphType *graph, WorkSpaceType *wspace)
+{
+  int h, i, ii, j, k;
+  int nvtxs, ncon, cnvtxs, firstvtx, maxi, maxidx, edge; 
+  idxtype *xadj, *ladjncy, *adjwgt, *vtxdist, *home, *myhome, *shome, *rhome;
+  idxtype *perm, *match;
+  float maxnvwgt, *nvwgt;
+
+  graph->match_type = MATCH_LOCAL;
+  maxnvwgt = 1.0/((float)(ctrl->nparts)*MAXVWGT_FACTOR);
+
+  IFSET(ctrl->dbglvl, DBG_TIME, MPI_Barrier(ctrl->comm));
+  IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->MatchTmr));
+
+  nvtxs = graph->nvtxs;
+  ncon = graph->ncon;
+  xadj = graph->xadj;
+  nvwgt = graph->nvwgt;
+  ladjncy = graph->adjncy;
+  adjwgt = graph->adjwgt;
+  home = graph->home;
+
+  vtxdist = graph->vtxdist;
+  firstvtx = vtxdist[ctrl->mype];
+
+  match = graph->match = idxmalloc(nvtxs+graph->nrecv, "HEM_Match: match");
+  myhome = idxsmalloc(nvtxs+graph->nrecv, UNMATCHED, "HEM_Match: myhome");
+
+  idxset(nvtxs, UNMATCHED, match);
+  idxset(graph->nrecv, 0, match+nvtxs);  /* Easy way to handle remote vertices */
+
+  /*------------------------------------------------------------
+  / Send/Receive the home information of interface vertices
+  /------------------------------------------------------------*/
+  if (ctrl->partType == ADAPTIVE_PARTITION || ctrl->partType == REFINE_PARTITION) {
+    idxcopy(nvtxs, home, myhome);
+    shome = wspace->indices;
+    rhome = myhome + nvtxs;
+    CommInterfaceData(ctrl, graph, myhome, shome, rhome);
+  }
+
+  /*************************************************************
+   * Go now and find a local matching 
+   *************************************************************/
+  perm = wspace->indices;
+  FastRandomPermute(nvtxs, perm, 1);
+  cnvtxs = 0;
+  for (ii=0; ii<nvtxs; ii++) {
+    i = perm[ii];
+    if (match[i] == UNMATCHED) {
+      maxidx = maxi = -1;
+
+      /* Find a heavy-edge matching, if the weight of the vertex is OK */
+      for (h=0; h<ncon; h++)
+        if (nvwgt[i*ncon+h] > maxnvwgt)
+          break;
+
+      if (h == ncon) {
+        for (j=xadj[i]; j<xadj[i+1]; j++) {
+          edge = ladjncy[j];
+
+          /* match only with local vertices */
+          if (myhome[edge] != myhome[i] || edge >= nvtxs)
+            continue;
+
+          for (h=0; h<ncon; h++)
+            if (nvwgt[edge*ncon+h] > maxnvwgt) 
+              break;
+
+          if (h == ncon) {
+            if (match[edge] == UNMATCHED &&
+              (maxi == -1 ||
+               adjwgt[maxi] < adjwgt[j] ||
+              (adjwgt[maxi] == adjwgt[j] &&
+               BetterVBalance(ncon,nvwgt+i*ncon,nvwgt+maxidx*ncon,nvwgt+edge*ncon) >= 0))) {
+              maxi = j;
+              maxidx = edge;
+            }
+          }
+        }
+      }
+
+      if (maxi != -1) {
+        k = ladjncy[maxi];
+        if (i <= k) {
+          match[i] = firstvtx+k + KEEP_BIT;
+          match[k] = firstvtx+i;
+        }
+        else {
+          match[i] = firstvtx+k;
+          match[k] = firstvtx+i + KEEP_BIT;
+        }
+      }
+      else {
+        match[i] = (firstvtx+i) + KEEP_BIT;
+      }
+      cnvtxs++;
+    }
+  }
+
+  CommInterfaceData(ctrl, graph, match, wspace->indices, match+nvtxs);
+  GKfree((void **)(&myhome), LTERM);
+
+#ifdef DEBUG_MATCH
+  PrintVector2(ctrl, nvtxs, firstvtx, match, "Match1");
+#endif
+
+
+  if (ctrl->dbglvl&DBG_MATCHINFO) {
+    PrintVector2(ctrl, nvtxs, firstvtx, match, "Match");
+    myprintf(ctrl, "Cnvtxs: %d\n", cnvtxs);
+    rprintf(ctrl, "Done with matching...\n");
+  }
+
+  IFSET(ctrl->dbglvl, DBG_TIME, MPI_Barrier(ctrl->comm));
+  IFSET(ctrl->dbglvl, DBG_TIME, stoptimer(ctrl->MatchTmr));
+
+  IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->ContractTmr));
+  Mc_Local_CreateCoarseGraph(ctrl, graph, wspace, cnvtxs);
+  IFSET(ctrl->dbglvl, DBG_TIME, stoptimer(ctrl->ContractTmr));
+
+}
+
+
+
+
+
+/*************************************************************************
+* This function creates the coarser graph
+**************************************************************************/
+void Mc_Local_CreateCoarseGraph(CtrlType *ctrl, GraphType *graph, WorkSpaceType *wspace, int cnvtxs)
+{
+  int h, i, j, k, l;
+  int nvtxs, ncon, nedges, firstvtx, cfirstvtx;
+  int npes=ctrl->npes, mype=ctrl->mype;
+  int cnedges, v, u;
+  idxtype *xadj, *vwgt, *vsize, *ladjncy, *adjwgt, *vtxdist, *where, *home;
+  idxtype *match, *cmap;
+  idxtype *cxadj, *cvwgt, *cvsize = NULL, *cadjncy, *cadjwgt, *cvtxdist, *chome = NULL, *cwhere = NULL;
+  float *cnvwgt;
+  GraphType *cgraph;
+  int mask=(1<<13)-1, htable[8192], htableidx[8192];
+
+  nvtxs = graph->nvtxs;
+  ncon = graph->ncon;
+
+  vtxdist = graph->vtxdist;
+  xadj = graph->xadj;
+  vwgt = graph->vwgt;
+  home = graph->home;
+  vsize = graph->vsize;
+  ladjncy = graph->adjncy;
+  adjwgt = graph->adjwgt;
+  where = graph->where;
+  match = graph->match;
+
+  firstvtx = vtxdist[mype];
+
+  cmap = graph->cmap = idxmalloc(nvtxs+graph->nrecv, "CreateCoarseGraph: cmap");
+
+  /* Initialize the coarser graph */
+  cgraph = CreateGraph();
+  cgraph->nvtxs = cnvtxs;
+  cgraph->level = graph->level+1;
+  cgraph->ncon = ncon;
+
+  cgraph->finer = graph;
+  graph->coarser = cgraph;
+
+
+  /*************************************************************
+  * Obtain the vtxdist of the coarser graph 
+  **************************************************************/
+  cvtxdist = cgraph->vtxdist = idxmalloc(npes+1, "CreateCoarseGraph: cvtxdist");
+  cvtxdist[npes] = cnvtxs;  /* Use last position in the cvtxdist as a temp buffer */
+
+  MPI_Allgather((void *)(cvtxdist+npes), 1, IDX_DATATYPE, (void *)cvtxdist, 1, IDX_DATATYPE, ctrl->comm);
+
+  MAKECSR(i, npes, cvtxdist);
+
+  cgraph->gnvtxs = cvtxdist[npes];
+
+#ifdef DEBUG_CONTRACT
+  PrintVector(ctrl, npes+1, 0, cvtxdist, "cvtxdist");
+#endif
+
+
+  /*************************************************************
+  * Construct the cmap vector 
+  **************************************************************/
+  cfirstvtx = cvtxdist[mype];
+
+  /* Create the cmap of what you know so far locally */
+  cnvtxs = 0;
+  for (i=0; i<nvtxs; i++) {
+    if (match[i] >= KEEP_BIT) {
+      k = match[i] - KEEP_BIT;
+      if (k<firstvtx+i)
+        continue;  /* i has been matched via the (k,i) side */
+
+      cmap[i] = cfirstvtx + cnvtxs++;
+      if (k != firstvtx+i) {
+        cmap[k-firstvtx] = cmap[i];
+        match[k-firstvtx] += KEEP_BIT;  /* Add the KEEP_BIT to simplify coding */
+      }
+    }
+  }
+
+  CommInterfaceData(ctrl, graph, cmap, wspace->indices, cmap+nvtxs);
+
+
+#ifdef DEBUG_CONTRACT
+  PrintVector(ctrl, nvtxs, firstvtx, cmap, "Cmap");
+#endif
+
+
+
+  /*************************************************************
+  * Finally, create the coarser graph
+  **************************************************************/
+  /* Allocate memory for the coarser graph, and fire up coarsening */
+  cxadj = cgraph->xadj = idxmalloc(cnvtxs+1, "CreateCoarserGraph: cxadj");
+  cvwgt = cgraph->vwgt = idxmalloc(cnvtxs*ncon, "CreateCoarserGraph: cvwgt");
+  cnvwgt = cgraph->nvwgt = fmalloc(cnvtxs*ncon, "CreateCoarserGraph: cnvwgt");
+  if (ctrl->partType == ADAPTIVE_PARTITION || ctrl->partType == REFINE_PARTITION)
+    chome = cgraph->home = idxmalloc(cnvtxs, "CreateCoarserGraph: chome");
+  if (vsize != NULL)
+    cvsize = cgraph->vsize = idxmalloc(cnvtxs, "CreateCoarserGraph: cvsize");
+  if (where != NULL)
+    cwhere = cgraph->where = idxmalloc(cnvtxs, "CreateCoarserGraph: cwhere");
+  cadjncy = idxmalloc(2*graph->nedges, "CreateCoarserGraph: cadjncy");
+  cadjwgt = cadjncy+graph->nedges;
+
+  iset(8192, -1, htable);
+
+  cxadj[0] = cnvtxs = cnedges = 0;
+  for (i=0; i<nvtxs; i++) {
+    v = firstvtx+i; 
+    u = match[i]-KEEP_BIT;
+
+    if (v > u) 
+      continue;  /* I have already collapsed it as (u,v) */
+
+    /* Collapse the v vertex first, which you know that is local */
+    for (h=0; h<ncon; h++)
+      cvwgt[cnvtxs*ncon+h] = vwgt[i*ncon+h];
+    if (ctrl->partType == ADAPTIVE_PARTITION || ctrl->partType == REFINE_PARTITION)
+      chome[cnvtxs] = home[i];
+    if (vsize != NULL)
+      cvsize[cnvtxs] = vsize[i];
+    if (where != NULL)
+      cwhere[cnvtxs] = where[i];
+    nedges = 0;
+
+    for (j=xadj[i]; j<xadj[i+1]; j++) {
+      k = cmap[ladjncy[j]];
+      if (k != cfirstvtx+cnvtxs) {  /* If this is not an internal edge */
+        l = k&mask;
+        if (htable[l] == -1) { /* Seeing this for first time */
+          htable[l] = k;
+          htableidx[l] = cnedges+nedges;
+          cadjncy[cnedges+nedges] = k; 
+          cadjwgt[cnedges+nedges++] = adjwgt[j];
+        }
+        else if (htable[l] == k) {
+          cadjwgt[htableidx[l]] += adjwgt[j];
+        }
+        else { /* Now you have to go and do a search. Expensive case */
+          for (l=0; l<nedges; l++) {
+            if (cadjncy[cnedges+l] == k)
+              break;
+          }
+          if (l < nedges) {
+            cadjwgt[cnedges+l] += adjwgt[j];
+          }
+          else {
+            cadjncy[cnedges+nedges] = k; 
+            cadjwgt[cnedges+nedges++] = adjwgt[j];
+          }
+        }
+      }
+    }
+
+    /* Collapse the u vertex next */
+    if (v != u) { 
+      u -= firstvtx;
+      for (h=0; h<ncon; h++)
+        cvwgt[cnvtxs*ncon+h] += vwgt[u*ncon+h];
+      if (vsize != NULL)
+        cvsize[cnvtxs] += vsize[u];
+      if (where != NULL && cwhere[cnvtxs] != where[u])
+        myprintf(ctrl, "Something went wrong with the where local matching! %d %d\n", cwhere[cnvtxs], where[u]);
+
+      for (j=xadj[u]; j<xadj[u+1]; j++) {
+        k = cmap[ladjncy[j]];
+        if (k != cfirstvtx+cnvtxs) {  /* If this is not an internal edge */
+          l = k&mask;
+          if (htable[l] == -1) { /* Seeing this for first time */
+            htable[l] = k;
+            htableidx[l] = cnedges+nedges;
+            cadjncy[cnedges+nedges] = k; 
+            cadjwgt[cnedges+nedges++] = adjwgt[j];
+          }
+          else if (htable[l] == k) {
+            cadjwgt[htableidx[l]] += adjwgt[j];
+          }
+          else { /* Now you have to go and do a search. Expensive case */
+            for (l=0; l<nedges; l++) {
+              if (cadjncy[cnedges+l] == k)
+                break;
+            }
+            if (l < nedges) {
+              cadjwgt[cnedges+l] += adjwgt[j];
+            }
+            else {
+              cadjncy[cnedges+nedges] = k; 
+              cadjwgt[cnedges+nedges++] = adjwgt[j];
+            }
+          }
+        }
+      }
+    }
+
+    cnedges += nedges;
+    for (j=cxadj[cnvtxs]; j<cnedges; j++)
+      htable[cadjncy[j]&mask] = -1;  /* reset the htable */
+    cxadj[++cnvtxs] = cnedges;
+  }
+
+  cgraph->nedges = cnedges;
+
+  for (j=0; j<cnvtxs; j++)
+    for (h=0; h<ncon; h++)
+      cgraph->nvwgt[j*ncon+h] = (float)(cvwgt[j*ncon+h])/(float)(ctrl->tvwgts[h]);
+
+  cgraph->adjncy = idxmalloc(cnedges, "CreateCoarserGraph: cadjncy");
+  cgraph->adjwgt = idxmalloc(cnedges, "CreateCoarserGraph: cadjwgt");
+  idxcopy(cnedges, cadjncy, cgraph->adjncy);
+  idxcopy(cnedges, cadjwgt, cgraph->adjwgt);
+  GKfree((void **)&cadjncy, (void **)&graph->where, LTERM); /* Note that graph->where works fine even if it is NULL */
+
+}
+
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/macros.h b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/macros.h
new file mode 100644
index 0000000..31861f7
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/macros.h
@@ -0,0 +1,163 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * macros.h
+ *
+ * This file contains macros used in multilevel
+ *
+ * Started 9/25/94
+ * George
+ *
+ * $Id: macros.h,v 1.8 2003/07/21 19:11:46 karypis Exp $
+ *
+ */
+
+
+/*************************************************************************
+* The following macro returns a random number in the specified range
+**************************************************************************/
+#define RandomInRange(u) ((int)(1.0*(u)*rand()/(RAND_MAX+1.0)))
+
+#define amax(a, b) ((a) >= (b) ? (a) : (b))
+#define amin(a, b) ((a) >= (b) ? (b) : (a))
+
+#define AND(a, b) ((a) < 0 ? ((-(a))&(b)) : ((a)&(b)))
+#define OR(a, b) ((a) < 0 ? -((-(a))|(b)) : ((a)|(b)))
+#define XOR(a, b) ((a) < 0 ? -((-(a))^(b)) : ((a)^(b)))
+
+#define SWAP(a, b, tmp)  \
+                 do {(tmp) = (a); (a) = (b); (b) = (tmp);} while(0) 
+
+#define INC_DEC(a, b, val) \
+                 do {(a) += (val); (b) -= (val);} while(0)
+
+
+#define icopy(n, a, b) memcpy((b), (a), sizeof(int)*(n))
+#define scopy(n, a, b) memcpy((b), (a), sizeof(float)*(n))
+#define idxcopy(n, a, b) memcpy((b), (a), sizeof(idxtype)*(n))
+
+#define HASHFCT(key, size) ((key)%(size))
+
+
+/*************************************************************************
+* Timer macros
+**************************************************************************/
+#define cleartimer(tmr) (tmr = 0.0)
+#define starttimer(tmr) (tmr -= MPI_Wtime())
+#define stoptimer(tmr) (tmr += MPI_Wtime())
+#define gettimer(tmr) (tmr)
+
+
+/*************************************************************************
+* This macro is used to handle dbglvl
+**************************************************************************/
+#define IFSET(a, flag, cmd) if ((a)&(flag)) (cmd);
+
+/*************************************************************************
+* These macros are used for debuging memory leaks
+**************************************************************************/
+#ifdef DMALLOC
+#define imalloc(n, msg) (malloc(sizeof(int)*(n)))
+#define fmalloc(n, msg) (malloc(sizeof(float)*(n)))
+#define idxmalloc(n, msg) (malloc(sizeof(idxtype)*(n)))
+#define ismalloc(n, val, msg) (iset((n), (val), malloc(sizeof(int)*(n))))
+#define idxsmalloc(n, val, msg) (idxset((n), (val), malloc(sizeof(idxtype)*(n))))
+#define GKmalloc(a, b) (malloc(a))
+#endif
+
+#ifdef DMALLOC
+#   define MALLOC_CHECK(ptr);
+/*
+#   define MALLOC_CHECK(ptr)                                          \
+    if (malloc_verify((ptr)) == DMALLOC_VERIFY_ERROR) {  \
+        printf("***MALLOC_CHECK failed on line %d of file %s: " #ptr "\n", \
+              __LINE__, __FILE__);                               \
+        abort();                                                \
+    }
+*/
+#else
+#   define MALLOC_CHECK(ptr) ;
+#endif 
+
+/*************************************************************************
+* This macro converts a length array in a CSR one
+**************************************************************************/
+#define MAKECSR(i, n, a) \
+   do { \
+     for (i=1; i<n; i++) a[i] += a[i-1]; \
+     for (i=n; i>0; i--) a[i] = a[i-1]; \
+     a[0] = 0; \
+   } while(0) 
+
+
+#define SHIFTCSR(i, n, a) \
+   do { \
+     for (i=n; i>0; i--) a[i] = a[i-1]; \
+     a[0] = 0; \
+   } while(0)
+
+
+
+#ifdef DEBUG
+#   define ASSERT(ctrl, expr)                                          \
+    if (!(expr)) {                                               \
+        myprintf(ctrl, "***ASSERTION failed on line %d of file %s: " #expr "\n", \
+              __LINE__, __FILE__);                               \
+        abort();                                                \
+    }
+#else
+#   define ASSERT(ctrl, expr) ;
+#endif 
+
+#ifdef DEBUG
+#   define ASSERTP(ctrl, expr, msg)                                          \
+    if (!(expr)) {                                               \
+        myprintf(ctrl, "***ASSERTION failed on line %d of file %s:" #expr "\n", \
+              __LINE__, __FILE__);                               \
+        myprintf msg ; \
+        abort();                                                \
+    }
+#else
+#   define ASSERTP(ctrl, expr,msg) ;
+#endif 
+
+#ifdef DEBUGS
+#   define ASSERTS(expr)                                          \
+    if (!(expr)) {                                               \
+        printf("***ASSERTION failed on line %d of file %s: " #expr "\n", \
+              __LINE__, __FILE__);                               \
+        abort();                                                \
+    }
+#else
+#   define ASSERTS(expr) ;
+#endif 
+
+#ifdef DEBUGS
+#   define ASSERTSP(expr, msg)                                          \
+    if (!(expr)) {                                               \
+        printf("***ASSERTION failed on line %d of file %s: " #expr "\n", \
+              __LINE__, __FILE__);                               \
+        printf msg ; \
+        abort();                                                \
+    }
+#else
+#   define ASSERTSP(expr, msg) ;
+#endif 
+
+/*************************************************************************
+ * * These macros insert and remove nodes from the boundary list
+ * **************************************************************************/
+#define BNDInsert(nbnd, bndind, bndptr, vtx) \
+   do { \
+	        bndind[nbnd] = vtx; \
+			     bndptr[vtx] = nbnd++;\
+			        } while(0)
+
+#define BNDDelete(nbnd, bndind, bndptr, vtx) \
+   do { \
+	        bndind[bndptr[vtx]] = bndind[--nbnd]; \
+			     bndptr[bndind[nbnd]] = bndptr[vtx]; \
+			          bndptr[vtx] = -1; \
+				     } while(0)
+
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/match.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/match.c
new file mode 100644
index 0000000..89bfa62
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/match.c
@@ -0,0 +1,320 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * mmatch.c
+ *
+ * This file contains code that finds a matching
+ *
+ * Started 2/22/96
+ * George
+ *
+ * $Id: match.c,v 1.2 2003/07/21 17:18:50 karypis Exp $
+ *
+ */
+
+#include <parmetislib.h>
+
+
+/*************************************************************************
+* This function finds a matching
+**************************************************************************/
+void Moc_GlobalMatch_Balance(CtrlType *ctrl, GraphType *graph, WorkSpaceType *wspace)
+{
+  int h, i, ii, j, k;
+  int nnbrs, nvtxs, ncon, cnvtxs, firstvtx, lastvtx, maxi, maxidx, nkept;
+  int otherlastvtx, nrequests, nchanged, pass, nmatched, wside;
+  idxtype *xadj, *ladjncy, *adjwgt, *vtxdist, *home, *myhome, *shome, *rhome;
+  idxtype *match, *rmatch, *smatch;
+  idxtype *peind, *sendptr, *recvptr;
+  idxtype *perm, *iperm, *nperm, *changed;
+  float *nvwgt, maxnvwgt;
+  int *nreqs_pe;
+  KeyValueType *match_requests, *match_granted, *pe_requests;
+
+  maxnvwgt = 1.0/((float)(ctrl->nparts)*MAXNVWGT_FACTOR);
+
+  graph->match_type = MATCH_GLOBAL;
+
+  IFSET(ctrl->dbglvl, DBG_TIME, MPI_Barrier(ctrl->comm));
+  IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->MatchTmr));
+
+  nvtxs = graph->nvtxs;
+  ncon = graph->ncon;
+  xadj = graph->xadj;
+  ladjncy = graph->adjncy;
+  adjwgt = graph->adjwgt;
+  home = graph->home;
+  nvwgt = graph->nvwgt;
+
+  vtxdist = graph->vtxdist;
+  firstvtx = vtxdist[ctrl->mype];
+  lastvtx = vtxdist[ctrl->mype+1];
+
+  match = graph->match = idxsmalloc(nvtxs+graph->nrecv, UNMATCHED, "HEM_Match: match");
+  myhome = idxsmalloc(nvtxs+graph->nrecv, UNMATCHED, "HEM_Match: myhome");
+
+  /*------------------------------------------------------------
+  / Send/Receive the home information of interface vertices
+  /------------------------------------------------------------*/
+  if (ctrl->partType == ADAPTIVE_PARTITION || ctrl->partType == REFINE_PARTITION) {
+    idxcopy(nvtxs, home, myhome);
+    shome = wspace->indices;
+    rhome = myhome + nvtxs;
+    CommInterfaceData(ctrl, graph, myhome, shome, rhome);
+  }
+
+  nnbrs = graph->nnbrs;
+  peind = graph->peind;
+  sendptr = graph->sendptr;
+  recvptr = graph->recvptr;
+
+  /* Use wspace->indices as the tmp space for matching info of the boundary
+   * vertices that are sent and received */
+  rmatch = match + nvtxs;
+  smatch = wspace->indices;
+  changed = smatch+graph->nsend;
+
+  /* Use wspace->indices as the tmp space for match requests of the boundary
+   * vertices that are sent and received */
+  match_requests = wspace->pairs;
+  match_granted = match_requests + graph->nsend;
+
+  nreqs_pe = ismalloc(nnbrs, 0, "Match_HEM: nreqs_pe");
+
+  nkept = graph->gnvtxs/ctrl->npes - nvtxs;
+
+  perm = (idxtype *)wspace->degrees;
+  iperm = perm + nvtxs;
+  FastRandomPermute(nvtxs, perm, 1);
+  for (i=0; i<nvtxs; i++)
+    iperm[perm[i]] = i;
+
+  nperm = iperm + nvtxs;
+  for (i=0; i<nnbrs; i++)
+    nperm[i] = i;
+
+  /*************************************************************
+   * Go now and find a matching by doing multiple iterations
+   *************************************************************/
+  /* First nullify the heavy vertices */
+  for (nchanged=i=0; i<nvtxs; i++) {
+    for (h=0; h<ncon; h++)
+      if (nvwgt[i*ncon+h] > maxnvwgt) {
+        break;
+      }
+
+    if (h != ncon) {
+      match[i] = TOO_HEAVY;
+      nchanged++;
+    }
+  }
+  if (GlobalSESum(ctrl, nchanged) > 0) {
+    IFSET(ctrl->dbglvl, DBG_PROGRESS,
+    rprintf(ctrl, "We found %d heavy vertices!\n", GlobalSESum(ctrl, nchanged)));
+    CommInterfaceData(ctrl, graph, match, smatch, rmatch);
+  }
+
+
+  for (nmatched=pass=0; pass<NMATCH_PASSES; pass++) {
+    wside = (graph->level+pass)%2;
+    nchanged = nrequests = 0;
+    for (ii=nmatched; ii<nvtxs; ii++) {
+      i = perm[ii];
+      if (match[i] == UNMATCHED) {  /* Unmatched */
+        maxidx = i;
+        maxi = -1;
+
+        /* Find a heavy-edge matching */
+        for (j=xadj[i]; j<xadj[i+1]; j++) {
+          k = ladjncy[j];
+          if (match[k] == UNMATCHED &&
+               myhome[k] == myhome[i] &&
+               (maxi == -1 ||
+               adjwgt[maxi] < adjwgt[j] ||
+               (maxidx < nvtxs &&
+               k < nvtxs &&
+               adjwgt[maxi] == adjwgt[j] &&
+               BetterVBalance(ncon,nvwgt+i*ncon,nvwgt+maxidx*ncon,nvwgt+k*ncon) >= 0))) {
+            maxi = j;
+            maxidx = k;
+          }
+        }
+
+        if (maxi != -1) {
+          k = ladjncy[maxi];
+          if (k < nvtxs) { /* Take care the local vertices first */
+            /* Here we give preference the local matching by granting it right away */
+            if (i <= k) {
+              match[i] = firstvtx+k + KEEP_BIT;
+              match[k] = firstvtx+i;
+            }
+            else {
+              match[i] = firstvtx+k;
+              match[k] = firstvtx+i + KEEP_BIT;
+            }
+            changed[nchanged++] = i;
+            changed[nchanged++] = k;
+          }
+          else { /* Take care any remote boundary vertices */
+            match[k] = MAYBE_MATCHED;
+            /* Alternate among which vertices will issue the requests */
+            if ((wside ==0 && firstvtx+i < graph->imap[k]) || (wside == 1 && firstvtx+i > graph->imap[k])) { 
+              match[i] = MAYBE_MATCHED;
+              match_requests[nrequests].key = graph->imap[k];
+              match_requests[nrequests].val = firstvtx+i;
+              nrequests++;
+            }
+          }
+        }
+      }
+    }
+
+
+#ifdef DEBUG_MATCH
+    PrintVector2(ctrl, nvtxs, firstvtx, match, "Match1");
+    myprintf(ctrl, "[c: %2d] Nlocal: %d, Nrequests: %d\n", c, nlocal, nrequests);
+#endif
+
+
+    /***********************************************************
+    * Exchange the match_requests, requests for me are stored in
+    * match_granted 
+    ************************************************************/
+    /* Issue the receives first. Note that from each PE can receive a maximum
+       of the interface node that it needs to send it in the case of a mat-vec */
+    for (i=0; i<nnbrs; i++) {
+      MPI_Irecv((void *)(match_granted+recvptr[i]), 2*(recvptr[i+1]-recvptr[i]), IDX_DATATYPE,
+                peind[i], 1, ctrl->comm, ctrl->rreq+i);
+    }
+
+    /* Issue the sends next. This needs some work */
+    ikeysort(nrequests, match_requests);
+    for (j=i=0; i<nnbrs; i++) {
+      otherlastvtx = vtxdist[peind[i]+1];
+      for (k=j; k<nrequests && match_requests[k].key < otherlastvtx; k++);
+      MPI_Isend((void *)(match_requests+j), 2*(k-j), IDX_DATATYPE, peind[i], 1, ctrl->comm, ctrl->sreq+i);
+      j = k;
+    }
+
+    /* OK, now get into the loop waiting for the operations to finish */
+    MPI_Waitall(nnbrs, ctrl->rreq, ctrl->statuses);
+    for (i=0; i<nnbrs; i++) {
+      MPI_Get_count(ctrl->statuses+i, IDX_DATATYPE, nreqs_pe+i);
+      nreqs_pe[i] = nreqs_pe[i]/2;  /* Adjust for pairs of IDX_DATATYPE */
+    }
+    MPI_Waitall(nnbrs, ctrl->sreq, ctrl->statuses);
+
+
+    /***********************************************************
+    * Now, go and service the requests that you received in 
+    * match_granted 
+    ************************************************************/
+    RandomPermute(nnbrs, nperm, 0);
+    for (ii=0; ii<nnbrs; ii++) {
+      i = nperm[ii];
+      pe_requests = match_granted+recvptr[i];
+      for (j=0; j<nreqs_pe[i]; j++) {
+        k = pe_requests[j].key;
+        ASSERTP(ctrl, k >= firstvtx && k < lastvtx, (ctrl, "%d %d %d %d %d\n", firstvtx, lastvtx, k, j, peind[i]));
+        /* myprintf(ctrl, "Requesting a match %d %d\n", pe_requests[j].key, pe_requests[j].val); */
+        if (match[k-firstvtx] == UNMATCHED) { /* Bingo, lets grant this request */
+          changed[nchanged++] = k-firstvtx;
+          if (nkept >= 0) { /* Flip a coin for who gets it */
+            match[k-firstvtx] = pe_requests[j].val + KEEP_BIT;
+            nkept--;
+          }
+          else {
+            match[k-firstvtx] = pe_requests[j].val;
+            pe_requests[j].key += KEEP_BIT;
+            nkept++;
+          }
+          /* myprintf(ctrl, "Request from pe:%d (%d %d) granted!\n", peind[i], pe_requests[j].val, pe_requests[j].key); */ 
+        }
+        else { /* We are not granting the request */
+          /* myprintf(ctrl, "Request from pe:%d (%d %d) not granted!\n", peind[i], pe_requests[j].val, pe_requests[j].key); */ 
+          pe_requests[j].key = UNMATCHED;
+        }
+      }
+    }
+
+
+    /***********************************************************
+    * Exchange the match_granted information. It is stored in
+    * match_requests 
+    ************************************************************/
+    /* Issue the receives first. Note that from each PE can receive a maximum
+       of the interface node that it needs to send during the case of a mat-vec */
+    for (i=0; i<nnbrs; i++) {
+      MPI_Irecv((void *)(match_requests+sendptr[i]), 2*(sendptr[i+1]-sendptr[i]), IDX_DATATYPE,
+                peind[i], 1, ctrl->comm, ctrl->rreq+i);
+    }
+
+    /* Issue the sends next. */
+    for (i=0; i<nnbrs; i++) {
+      MPI_Isend((void *)(match_granted+recvptr[i]), 2*nreqs_pe[i], IDX_DATATYPE, 
+                peind[i], 1, ctrl->comm, ctrl->sreq+i);
+    }
+
+    /* OK, now get into the loop waiting for the operations to finish */
+    MPI_Waitall(nnbrs, ctrl->rreq, ctrl->statuses);
+    for (i=0; i<nnbrs; i++) {
+      MPI_Get_count(ctrl->statuses+i, IDX_DATATYPE, nreqs_pe+i);
+      nreqs_pe[i] = nreqs_pe[i]/2;  /* Adjust for pairs of IDX_DATATYPE */
+    }
+    MPI_Waitall(nnbrs, ctrl->sreq, ctrl->statuses);
+
+
+    /***********************************************************
+    * Now, go and through the match_requests and update local
+    * match information for the matchings that were granted.
+    ************************************************************/
+    for (i=0; i<nnbrs; i++) {
+      pe_requests = match_requests+sendptr[i];
+      for (j=0; j<nreqs_pe[i]; j++) {
+        match[pe_requests[j].val-firstvtx] = pe_requests[j].key;
+        if (pe_requests[j].key != UNMATCHED)
+          changed[nchanged++] = pe_requests[j].val-firstvtx;
+      }
+    }
+
+    for (i=0; i<nchanged; i++) {
+      ii = iperm[changed[i]];
+      perm[ii] = perm[nmatched];
+      iperm[perm[nmatched]] = ii;
+      nmatched++;
+    }
+
+    CommChangedInterfaceData(ctrl, graph, nchanged, changed, match, match_requests, match_granted, wspace->pv4);
+  }
+
+  /* Traverse the vertices and those that were unmatched, match them with themselves */
+  cnvtxs = 0;
+  for (i=0; i<nvtxs; i++) {
+    if (match[i] == UNMATCHED || match[i] == TOO_HEAVY) {
+      match[i] = (firstvtx+i) + KEEP_BIT;
+      cnvtxs++;
+    }
+    else if (match[i] >= KEEP_BIT) {  /* A matched vertex which I get to keep */
+      cnvtxs++;
+    }
+  }
+
+  if (ctrl->dbglvl&DBG_MATCHINFO) {
+    PrintVector2(ctrl, nvtxs, firstvtx, match, "Match");
+    myprintf(ctrl, "Cnvtxs: %d\n", cnvtxs);
+    rprintf(ctrl, "Done with matching...\n");
+  }
+
+  GKfree((void **)(&myhome), (void **)(&nreqs_pe), LTERM);
+
+  IFSET(ctrl->dbglvl, DBG_TIME, MPI_Barrier(ctrl->comm));
+  IFSET(ctrl->dbglvl, DBG_TIME, stoptimer(ctrl->MatchTmr));
+  IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->ContractTmr));
+
+  Moc_Global_CreateCoarseGraph(ctrl, graph, wspace, cnvtxs);
+
+  IFSET(ctrl->dbglvl, DBG_TIME, MPI_Barrier(ctrl->comm));
+  IFSET(ctrl->dbglvl, DBG_TIME, stoptimer(ctrl->ContractTmr));
+
+}
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/mdiffusion.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/mdiffusion.c
new file mode 100644
index 0000000..6c02138
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/mdiffusion.c
@@ -0,0 +1,455 @@
+/* * Copyright 1997, Regents of the University of Minnesota
+ *
+ * mdiffusion.c
+ *
+ * This file contains code that performs mc-diffusion
+ *
+ * Started 9/16/99
+ * George
+ *
+ * $Id: mdiffusion.c,v 1.2 2003/07/21 17:18:50 karypis Exp $
+ */
+
+#include <parmetislib.h>
+
+#define PE	-1
+
+/*************************************************************************
+* This function is the entry point of the initial partitioning algorithm.
+* This algorithm assembles the graph to all the processors and preceed
+* serially.
+**************************************************************************/
+int Moc_Diffusion(CtrlType *ctrl, GraphType *graph, idxtype *vtxdist,
+  idxtype *where, idxtype *home, WorkSpaceType *wspace, int npasses)
+{
+  int h, i, j;
+  int nvtxs, nedges, ncon, pass, iter, domain, processor;
+  int nparts, mype, npes, nlinks, me, you, wsize;
+  int nvisited, nswaps = -1, tnswaps, done, alldone = -1;
+  idxtype *rowptr, *colind, *diff_where, *sr_where, *ehome, *map, *rmap;
+  idxtype *pack, *unpack, *match, *proc2sub, *sub2proc;
+  idxtype *visited, *gvisited;
+  float *transfer, *npwgts, maxdiff, minflow, maxflow;
+  float lbavg, oldlbavg, ubavg, lbvec[MAXNCON];
+  float diff_flows[MAXNCON], sr_flows[MAXNCON];
+  float diff_lbavg, sr_lbavg, diff_cost, sr_cost;
+  idxtype *rbuffer, *sbuffer; 
+  int *rcount, *rdispl;
+  float *solution, *load, *workspace;
+  EdgeType *degrees;
+  MatrixType matrix;
+  GraphType *egraph;
+  RInfoType *rinfo;
+
+  if (graph->ncon > 3)
+    return 0;
+
+  nvtxs = graph->nvtxs;
+  nedges = graph->nedges;
+  ncon = graph->ncon;
+
+  nparts = ctrl->nparts;
+  mype = ctrl->mype;
+  npes = ctrl->npes;
+  ubavg = savg(ncon, ctrl->ubvec);
+
+  /********************************************/
+  /* initialize variables and allocate memory */
+  /********************************************/
+  load = fmalloc(nparts*(2+ncon)+nedges*(1+ncon), "load");
+  solution =                   load + nparts;
+  npwgts = graph->gnpwgts =    load + 2*nparts;
+  matrix.values =              load + (2+ncon)*nparts;
+  transfer = matrix.transfer = load + (2+ncon)*nparts + nedges;
+
+  proc2sub = idxmalloc(amax(nparts, npes*2), "Mc_Diffusion: proc2sub");
+  sub2proc = idxmalloc(nparts*3+nedges+1, "Mc_Diffusion: match");
+  match =                  sub2proc + nparts;
+  rowptr = matrix.rowptr = sub2proc + 2*nparts;
+  colind = matrix.colind = sub2proc + 3*nparts + 1;
+
+  rcount = imalloc(2*npes+1, "Mc_Diffusion: rcount");
+  rdispl = rcount + npes;
+
+  pack = idxmalloc(nvtxs*8, "Mc_Diffusion: pack");
+  unpack =     pack + nvtxs;
+  rbuffer =    pack + 2*nvtxs;
+  sbuffer =    pack + 3*nvtxs;
+  map =        pack + 4*nvtxs;
+  rmap =       pack + 5*nvtxs;
+  diff_where = pack + 6*nvtxs;
+  ehome =      pack + 7*nvtxs;
+
+  wsize = amax(sizeof(float)*nparts*6, sizeof(idxtype)*(nvtxs+nparts*2+1));
+  workspace = (float *)GKmalloc(wsize, "Moc_Diffusion: workspace");
+  degrees = GKmalloc(nedges*sizeof(EdgeType), "Mc_Diffusion: degrees");
+  rinfo = graph->rinfo = GKmalloc(nvtxs*sizeof(RInfoType), "Mc_Diffusion: rinfo");
+
+  /******************************************/
+  /* construct subdomain connectivity graph */
+  /******************************************/
+  matrix.nrows = nparts;
+  SetUpConnectGraph(graph, &matrix, (idxtype *)workspace);
+  nlinks = (matrix.nnzs-nparts) / 2;
+
+  visited = idxmalloc(matrix.nnzs*2, "visited");
+  gvisited = visited + matrix.nnzs;
+
+  for (pass=0; pass<npasses; pass++) {
+    sset(matrix.nnzs*ncon, 0.0, transfer);
+    idxset(matrix.nnzs, 0, gvisited);
+    idxset(matrix.nnzs, 0, visited);
+    iter = nvisited = 0;
+
+    /*******************************/
+    /* compute ncon flow solutions */
+    /*******************************/
+    for (h=0; h<ncon; h++) {
+      sset(nparts, 0.0, solution);
+      ComputeLoad(graph, nparts, load, ctrl->tpwgts, h);
+
+      lbvec[h] = (load[samax(nparts, load)]+1.0/(float)nparts) * (float)nparts;
+
+      ConjGrad2(&matrix, load, solution, 0.001, workspace);
+      ComputeTransferVector(ncon, &matrix, solution, transfer, h);
+    }
+
+    oldlbavg = savg(ncon, lbvec);
+    tnswaps = 0;
+    maxdiff = 0.0;
+    for (i=0; i<nparts; i++) {
+      for (j=rowptr[i]; j<rowptr[i+1]; j++) {
+        minflow = transfer[j*ncon+samin(ncon, transfer+j*ncon)];
+        maxflow = transfer[j*ncon+samax(ncon, transfer+j*ncon)];
+        maxdiff = (maxflow - minflow > maxdiff) ? maxflow - minflow : maxdiff;
+      }
+    }
+
+    while (nvisited < nlinks) {
+
+      /******************************************/
+      /* compute independent sets of subdomains */
+      /******************************************/
+      idxset(amax(nparts, npes*2), UNMATCHED, proc2sub);
+      CSR_Match_SHEM(&matrix, match, proc2sub, gvisited, ncon);
+
+      /*****************************/
+      /* Set up the packing arrays */
+      /*****************************/
+      idxset(nparts, UNMATCHED, sub2proc);
+      for (i=0; i<npes*2; i++) {
+        if (proc2sub[i] == UNMATCHED)
+          break;
+
+        sub2proc[proc2sub[i]] = i/2;
+      }
+
+      iset(npes, 0, rcount);
+      for (i=0; i<nvtxs; i++) {
+        domain = where[i];
+        processor = sub2proc[domain];
+        if (processor != UNMATCHED) {
+          rcount[processor]++;
+        }
+      }
+
+      rdispl[0] = 0;
+      for (i=1; i<npes+1; i++)
+        rdispl[i] = rdispl[i-1] + rcount[i-1];
+
+      idxset(nvtxs, UNMATCHED, unpack);
+      for (i=0; i<nvtxs; i++) {
+        domain = where[i];
+        processor = sub2proc[domain];
+        if (processor != UNMATCHED) {
+          unpack[rdispl[processor]++] = i;
+        }
+      }
+
+      for (i=npes; i>0; i--)
+        rdispl[i] = rdispl[i-1];
+      rdispl[0] = 0;
+
+      idxset(nvtxs, UNMATCHED, pack);
+      for (i=0; i<rdispl[npes]; i++) {
+        ASSERTS(unpack[i] != UNMATCHED);
+        domain = where[unpack[i]];
+        processor = sub2proc[domain];
+        if (processor != UNMATCHED) {
+          pack[unpack[i]] = i;
+        }
+      }
+
+      /*********************/
+      /* Compute the flows */
+      /*********************/
+      if (proc2sub[mype*2] != UNMATCHED) {
+        me = proc2sub[mype*2];
+        you = proc2sub[mype*2+1];
+        ASSERTS(me != you);
+
+        for (j=rowptr[me]; j<rowptr[me+1]; j++) {
+          if (colind[j] == you) {
+            visited[j] = 1;
+            scopy(ncon, transfer+j*ncon, diff_flows);
+            break;
+          }
+        }
+
+        for (j=rowptr[you]; j<rowptr[you+1]; j++) {
+          if (colind[j] == me) {
+            visited[j] = 1;
+            for (h=0; h<ncon; h++)
+              if (transfer[j*ncon+h] > 0.0)
+                diff_flows[h] = -1.0 * transfer[j*ncon+h];
+            break;
+          }
+        } 
+
+        nswaps = 1;
+        scopy(ncon, diff_flows, sr_flows);
+
+        idxset(nvtxs, 0, sbuffer);
+        for (i=0; i<nvtxs; i++)
+          if (where[i] == me || where[i] == you)
+            sbuffer[i] = 1;
+
+        egraph = ExtractGraph(ctrl, graph, sbuffer, map, rmap);
+
+        if (egraph != NULL) {
+          idxcopy(egraph->nvtxs, egraph->where, diff_where);
+          for (j=0; j<egraph->nvtxs; j++)
+            ehome[j] = home[map[j]];
+ 
+          RedoMyLink(ctrl, egraph, ehome, me, you, sr_flows, &sr_cost, &sr_lbavg);
+
+          if (ncon <= 4) {
+            sr_where = egraph->where;
+            egraph->where = diff_where;
+
+            nswaps = BalanceMyLink(ctrl, egraph, ehome, me, you, diff_flows, maxdiff, &diff_cost, &diff_lbavg, 1.0/(float)nvtxs);
+
+            if ((sr_lbavg < diff_lbavg &&
+            (diff_lbavg >= ubavg-1.0 || sr_cost == diff_cost)) ||
+            (sr_lbavg < ubavg-1.0 && sr_cost < diff_cost)) {
+              for (i=0; i<egraph->nvtxs; i++)
+                where[map[i]] = sr_where[i];
+            }
+            else {
+              for (i=0; i<egraph->nvtxs; i++)
+                where[map[i]] = diff_where[i];
+            }
+          }
+          else {
+              for (i=0; i<egraph->nvtxs; i++)
+                where[map[i]] = egraph->where[i];
+          }
+
+          GKfree((void **)&egraph->xadj, (void **)&egraph->nvwgt, (void **)&egraph->adjncy, LTERM);
+          GKfree((void **)&egraph, LTERM);
+        }
+
+        /**********************/
+        /* Pack the flow data */
+        /**********************/
+        idxset(nvtxs, UNMATCHED, sbuffer);
+        for (i=0; i<nvtxs; i++) {
+          domain = where[i];
+          if (domain == you || domain == me) {
+            sbuffer[pack[i]] = where[i];
+          }
+        }
+      }
+
+      /***************************/
+      /* Broadcast the flow data */
+      /***************************/
+      MPI_Allgatherv((void *)&sbuffer[rdispl[mype]], rcount[mype], IDX_DATATYPE, (void *)rbuffer, rcount, rdispl, IDX_DATATYPE, ctrl->comm);
+
+
+      /************************/
+      /* Unpack the flow data */
+      /************************/
+      for (i=0; i<rdispl[npes]; i++) {
+        if (rbuffer[i] != UNMATCHED) {
+          where[unpack[i]] = rbuffer[i];
+        }
+      }
+
+
+      /******************/
+      /* Do other stuff */
+      /******************/
+      MPI_Allreduce((void *)visited, (void *)gvisited, matrix.nnzs,
+      IDX_DATATYPE, MPI_MAX, ctrl->comm);
+      nvisited = idxsum(matrix.nnzs, gvisited)/2;
+      tnswaps += GlobalSESum(ctrl, nswaps);
+
+      if (iter++ == NGD_PASSES)
+        break;
+    }
+
+    /*****************************/
+    /* perform serial refinement */
+    /*****************************/
+    Moc_ComputeSerialPartitionParams(graph, nparts, degrees);
+    Moc_SerialKWayAdaptRefine(graph, nparts, home, ctrl->ubvec, 10);
+
+
+    /****************************/
+    /* check for early breakout */
+    /****************************/
+    for (h=0; h<ncon; h++) {
+      lbvec[h] = (float)(nparts) *
+        npwgts[samax_strd(nparts,npwgts+h,ncon)*ncon+h];
+    }
+    lbavg = savg(ncon, lbvec);
+
+    done = 0;
+    if (
+      tnswaps == 0 ||
+      lbavg >= oldlbavg ||
+      lbavg <= ubavg + 0.035
+    )
+      done = 1;
+
+    alldone = GlobalSEMax(ctrl, done);
+    if (alldone == 1)
+      break;
+  }
+
+  /*******************************************************/
+  /* ensure that all subdomains have at least one vertex */
+  /*******************************************************/
+/*
+  idxset(nparts, 0, match);
+  for (i=0; i<nvtxs; i++)
+    match[where[i]]++;
+
+  done = 0;
+  while (done == 0) {
+    done = 1;
+
+    me = idxamin(nparts, match);  
+    if (match[me] == 0) {
+if (ctrl->mype == PE) printf("WARNING: empty subdomain %d in Moc_Diffusion\n", me);
+      you = idxamax(nparts, match);  
+      for (i=0; i<nvtxs; i++) {
+        if (where[i] == you) {
+          where[i] = me;
+          match[you]--;
+          match[me]++;
+          done = 0;
+          break;
+        }
+      }
+    }
+  }
+*/
+ 
+  /******************************/
+  /* now free memory and return */
+  /******************************/
+  GKfree((void **)&load, (void **)&proc2sub, (void **)&sub2proc, (void **)&rcount, LTERM);
+  GKfree((void **)&pack, (void **)&workspace, (void **)&degrees, (void **)&rinfo, LTERM);
+  GKfree((void **)&visited, LTERM);
+  graph->gnpwgts = NULL;
+  graph->rinfo = NULL;
+
+  return 0;
+}
+
+
+/*************************************************************************
+* This function extracts a subgraph from a graph given an indicator array.
+**************************************************************************/
+GraphType *ExtractGraph(CtrlType *ctrl, GraphType *graph, idxtype *indicator,
+  idxtype *map, idxtype *rmap)
+{
+  int h, i, j;
+  int nvtxs, envtxs, enedges, ncon;
+  int vtx, count;
+  idxtype *xadj, *vsize, *adjncy, *adjwgt, *where;
+  idxtype *exadj, *evsize, *eadjncy, *eadjwgt, *ewhere;
+  float *nvwgt, *envwgt;
+  GraphType *egraph;
+
+  nvtxs = graph->nvtxs;
+  ncon = graph->ncon;
+  xadj = graph->xadj;
+  nvwgt = graph->nvwgt;
+  vsize = graph->vsize;
+  adjncy = graph->adjncy;
+  adjwgt = graph->adjwgt;
+  where = graph->where;
+
+  count = 0;
+  for (i=0; i<nvtxs; i++) {
+    if (indicator[i] == 1) {
+      map[count] = i;
+      rmap[i] = count;
+      count++;
+    }
+  }
+
+  if (count == 0) {
+    return NULL;
+  }
+
+  /*******************/
+  /* allocate memory */
+  /*******************/
+  egraph = CreateGraph();
+  envtxs = egraph->nvtxs = count;
+  egraph->ncon = graph->ncon;
+
+  exadj = egraph->xadj = idxmalloc(envtxs*3+1, "exadj");
+  ewhere = egraph->where = exadj + envtxs + 1;
+  evsize = egraph->vsize = exadj + 2*envtxs + 1;
+
+  envwgt = egraph->nvwgt = fmalloc(envtxs*ncon, "envwgt");
+
+  /************************************************/
+  /* compute xadj, where, nvwgt, and vsize arrays */
+  /************************************************/
+  idxset(envtxs+1, 0, exadj);
+  for (i=0; i<envtxs; i++) {
+    vtx = map[i];
+
+    ewhere[i] = where[vtx];
+    for (h=0; h<ncon; h++)
+      envwgt[i*ncon+h] = nvwgt[vtx*ncon+h];
+
+    if (ctrl->partType == ADAPTIVE_PARTITION || ctrl->partType == REFINE_PARTITION) 
+      evsize[i] = vsize[vtx];
+
+    for (j=xadj[vtx]; j<xadj[vtx+1]; j++)
+      if (indicator[adjncy[j]] == 1)
+        exadj[i]++;
+
+  }
+  MAKECSR(i, envtxs, exadj);
+
+  /************************************/
+  /* compute adjncy and adjwgt arrays */
+  /************************************/
+  enedges = egraph->nedges = exadj[envtxs];
+  eadjncy = egraph->adjncy = idxmalloc(enedges*2, "eadjncy");
+  eadjwgt = egraph->adjwgt = eadjncy + enedges;
+
+  for (i=0; i<envtxs; i++) {
+    vtx = map[i];
+    for (j=xadj[vtx]; j<xadj[vtx+1]; j++) {
+      if (indicator[adjncy[j]] == 1) {
+        eadjncy[exadj[i]] = rmap[adjncy[j]];
+        eadjwgt[exadj[i]++] = adjwgt[j];
+      }
+    }
+  }
+
+  for (i=envtxs; i>0; i--)
+    exadj[i] = exadj[i-1];
+  exadj[0] = 0;
+
+  return egraph;
+}
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/mdrivers.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/mdrivers.c
new file mode 100644
index 0000000..467df79
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/mdrivers.c
@@ -0,0 +1,116 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * mdrivers.c
+ *
+ * This file contains the driving routines for the various parallel
+ * multilevel partitioning and repartitioning algorithms
+ *
+ * Started 11/19/96
+ * George
+ *
+ * $Id: mdrivers.c,v 1.3 2003/07/22 20:29:06 karypis Exp $
+ *
+ */
+
+#include <parmetislib.h>
+
+
+
+/*************************************************************************
+* This function is the driver to the multi-constraint partitioning algorithm.
+**************************************************************************/
+void Moc_Global_Partition(CtrlType *ctrl, GraphType *graph, WorkSpaceType *wspace)
+{
+  int i, ncon, nparts;
+  float ftmp, ubavg, lbavg, lbvec[MAXNCON];
+ 
+  ncon = graph->ncon;
+  nparts = ctrl->nparts;
+  ubavg = savg(graph->ncon, ctrl->ubvec);
+
+  SetUp(ctrl, graph, wspace);
+
+  if (ctrl->dbglvl&DBG_PROGRESS) {
+    rprintf(ctrl, "[%6d %8d %5d %5d] [%d] [", graph->gnvtxs, GlobalSESum(ctrl, graph->nedges),
+	    GlobalSEMin(ctrl, graph->nvtxs), GlobalSEMax(ctrl, graph->nvtxs), ctrl->CoarsenTo);
+    for (i=0; i<ncon; i++)
+      rprintf(ctrl, " %.3f", GlobalSEMinFloat(ctrl,graph->nvwgt[samin_strd(graph->nvtxs, graph->nvwgt+i, ncon)*ncon+i]));  
+    rprintf(ctrl, "] [");
+    for (i=0; i<ncon; i++)
+      rprintf(ctrl, " %.3f", GlobalSEMaxFloat(ctrl, graph->nvwgt[samax_strd(graph->nvtxs, graph->nvwgt+i, ncon)*ncon+i]));  
+    rprintf(ctrl, "]\n");
+  }
+
+  if (graph->gnvtxs < 1.3*ctrl->CoarsenTo ||
+	(graph->finer != NULL &&
+	graph->gnvtxs > graph->finer->gnvtxs*COARSEN_FRACTION)) {
+
+    /* Done with coarsening. Find a partition */
+    graph->where = idxmalloc(graph->nvtxs+graph->nrecv, "graph->where");
+    Moc_InitPartition_RB(ctrl, graph, wspace);
+
+    if (ctrl->dbglvl&DBG_PROGRESS) {
+      Moc_ComputeParallelBalance(ctrl, graph, graph->where, lbvec);
+      rprintf(ctrl, "nvtxs: %10d, balance: ", graph->gnvtxs);
+      for (i=0; i<graph->ncon; i++) 
+        rprintf(ctrl, "%.3f ", lbvec[i]);
+      rprintf(ctrl, "\n");
+    }
+
+    /* In case no coarsening took place */
+    if (graph->finer == NULL) {
+      Moc_ComputePartitionParams(ctrl, graph, wspace);
+      Moc_KWayFM(ctrl, graph, wspace, NGR_PASSES);
+    }
+  }
+  else {
+    Moc_GlobalMatch_Balance(ctrl, graph, wspace);
+
+    Moc_Global_Partition(ctrl, graph->coarser, wspace);
+
+    Moc_ProjectPartition(ctrl, graph, wspace);
+    Moc_ComputePartitionParams(ctrl, graph, wspace);
+
+    if (graph->ncon > 1 && graph->level < 3) {
+      for (i=0; i<ncon; i++) {
+        ftmp = ssum_strd(nparts, graph->gnpwgts+i, ncon);
+        if (ftmp != 0.0)
+          lbvec[i] = (float)(nparts) *
+          graph->gnpwgts[samax_strd(nparts, graph->gnpwgts+i, ncon)*ncon+i]/ftmp;
+        else
+          lbvec[i] = 1.0;
+      }
+      lbavg = savg(graph->ncon, lbvec);
+
+      if (lbavg > ubavg + 0.035) {
+        if (ctrl->dbglvl&DBG_PROGRESS) {
+          Moc_ComputeParallelBalance(ctrl, graph, graph->where, lbvec);
+          rprintf(ctrl, "nvtxs: %10d, cut: %8d, balance: ", graph->gnvtxs, graph->mincut);
+          for (i=0; i<graph->ncon; i++) 
+            rprintf(ctrl, "%.3f ", lbvec[i]);
+          rprintf(ctrl, "\n");
+	}
+
+        Moc_KWayBalance(ctrl, graph, wspace, graph->ncon);
+      }
+    }
+
+    Moc_KWayFM(ctrl, graph, wspace, NGR_PASSES);
+
+    if (ctrl->dbglvl&DBG_PROGRESS) {
+      Moc_ComputeParallelBalance(ctrl, graph, graph->where, lbvec);
+      rprintf(ctrl, "nvtxs: %10d, cut: %8d, balance: ", graph->gnvtxs, graph->mincut);
+      for (i=0; i<graph->ncon; i++) 
+        rprintf(ctrl, "%.3f ", lbvec[i]);
+      rprintf(ctrl, "\n");
+    }
+
+    if (graph->level != 0)
+      GKfree((void **)&graph->lnpwgts, (void **)&graph->gnpwgts, LTERM);
+  }
+
+  return;
+}
+
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/memory.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/memory.c
new file mode 100644
index 0000000..a8eeaff
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/memory.c
@@ -0,0 +1,216 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * memory.c
+ *
+ * This file contains routines that deal with memory allocation
+ *
+ * Started 2/24/96
+ * George
+ *
+ * $Id: memory.c,v 1.3 2003/07/30 18:37:59 karypis Exp $
+ *
+ */
+
+#include <parmetislib.h>
+
+
+/*************************************************************************
+* This function allocate various pools of memory
+**************************************************************************/
+void PreAllocateMemory(CtrlType *ctrl, GraphType *graph, WorkSpaceType *wspace)
+{
+  wspace->nlarge  = 2*graph->nedges;
+
+  wspace->maxcore = 8*graph->nedges+1;
+  wspace->core    = idxmalloc(wspace->maxcore, "PreAllocateMemory: wspace->core");
+
+  wspace->pairs   = (KeyValueType *)wspace->core;
+  wspace->indices = (idxtype *)(wspace->pairs + wspace->nlarge);
+  wspace->degrees = (EdgeType *)(wspace->indices + wspace->nlarge);
+
+
+  wspace->pv1 = idxmalloc(ctrl->nparts+ctrl->npes+1, "PreAllocateMemory: wspace->pv?");
+  wspace->pv2 = idxmalloc(ctrl->nparts+ctrl->npes+1, "PreAllocateMemory: wspace->pv?");
+  wspace->pv3 = idxmalloc(ctrl->nparts+ctrl->npes+1, "PreAllocateMemory: wspace->pv?");
+  wspace->pv4 = idxmalloc(ctrl->nparts+ctrl->npes+1, "PreAllocateMemory: wspace->pv?");
+
+  wspace->pepairs1 = (KeyValueType *)GKmalloc(sizeof(KeyValueType)*(ctrl->nparts+ctrl->npes+1), "PreAllocateMemory: wspace->pepairs?");
+  wspace->pepairs2 = (KeyValueType *)GKmalloc(sizeof(KeyValueType)*(ctrl->nparts+ctrl->npes+1), "PreAllocateMemory: wspace->pepairs?");
+
+}
+
+
+/*************************************************************************
+* This function de-allocate various pools of memory
+**************************************************************************/
+void FreeWSpace(WorkSpaceType *wspace)
+{
+
+  GKfree((void **)&wspace->core, 
+         (void **)&wspace->pv1, 
+         (void **)&wspace->pv2, 
+         (void **)&wspace->pv3,
+         (void **)&wspace->pv4, 
+         (void **)&wspace->pepairs1, 
+         (void **)&wspace->pepairs2, 
+         LTERM);
+}
+
+
+/*************************************************************************
+* This function de-allocates memory allocated for the control structures
+**************************************************************************/
+void FreeCtrl(CtrlType *ctrl)
+{
+  MPI_Comm_free(&(ctrl->gcomm));
+}
+
+
+/*************************************************************************
+* This function creates a CoarseGraphType data structure and initializes
+* the various fields
+**************************************************************************/
+GraphType *CreateGraph(void)
+{
+  GraphType *graph;
+
+  graph = (GraphType *)GKmalloc(sizeof(GraphType), "CreateCoarseGraph: graph");
+
+  InitGraph(graph);
+
+  return graph;
+}
+
+
+/*************************************************************************
+* This function creates a CoarseGraphType data structure and initializes
+* the various fields
+**************************************************************************/
+void InitGraph(GraphType *graph) 
+{
+  graph->gnvtxs = graph->nvtxs = graph->nedges = graph->nsep = -1;
+  graph->nnbrs = graph->nrecv = graph->nsend = graph->nlocal = -1;
+  graph->xadj = graph->vwgt = graph->vsize = graph->adjncy = graph->adjwgt = NULL;
+  graph->nvwgt = NULL;
+  graph->vtxdist = NULL;
+  graph->match = graph->cmap = NULL;
+  graph->label = NULL;
+
+  graph->peind = NULL;
+  graph->sendptr = graph->sendind = graph->recvptr = graph->recvind = NULL;
+  graph->imap = NULL;
+  graph->pexadj = graph->peadjncy = graph->peadjloc = NULL;
+  graph->lperm = NULL;
+
+  graph->slens = graph->rlens = NULL;
+  graph->rcand = NULL;
+
+  graph->where = graph->home = graph->lpwgts = graph->gpwgts = NULL;
+  graph->lnpwgts = graph->gnpwgts = NULL;
+  graph->rinfo = NULL;
+
+  graph->nrinfo = NULL;
+  graph->sepind = NULL;
+
+  graph->coarser = graph->finer = NULL;
+
+}
+
+/*************************************************************************
+* This function deallocates any memory stored in a graph
+**************************************************************************/
+void FreeGraph(GraphType *graph) 
+{
+
+  GKfree((void **)&graph->xadj, 
+         (void **)&graph->vwgt,
+         (void **)&graph->nvwgt,
+         (void **)&graph->vsize,
+         (void **)&graph->adjncy,
+         (void **)&graph->adjwgt,
+         (void **)&graph->vtxdist, 
+         (void **)&graph->match, 
+         (void **)&graph->cmap, 
+         (void **)&graph->lperm, 
+         (void **)&graph->label, 
+         (void **)&graph->where, 
+         (void **)&graph->home, 
+         (void **)&graph->rinfo, 
+         (void **)&graph->nrinfo, 
+         (void **)&graph->sepind,
+         (void **)&graph->lpwgts, 
+         (void **)&graph->gpwgts, 
+         (void **)&graph->lnpwgts, 
+         (void **)&graph->gnpwgts, 
+         (void **)&graph->peind, 
+         (void **)&graph->sendptr, 
+         (void **)&graph->sendind, 
+         (void **)&graph->recvptr, 
+         (void **)&graph->recvind, 
+         (void **)&graph->imap,
+         (void **)&graph->rlens,
+         (void **)&graph->slens,
+         (void **)&graph->rcand,
+         (void **)&graph->pexadj,
+         (void **)&graph->peadjncy,
+         (void **)&graph->peadjloc,
+         LTERM);
+
+  free(graph);
+}
+
+
+
+/*************************************************************************
+* This function deallocates any memory stored in a graph
+**************************************************************************/
+void FreeInitialGraphAndRemap(GraphType *graph, int wgtflag) 
+{
+  int i, nedges;
+  idxtype *adjncy, *imap;
+
+  nedges = graph->nedges;
+  adjncy = graph->adjncy;
+  imap = graph->imap;
+
+  if (imap != NULL) {
+    for (i=0; i<nedges; i++)
+      adjncy[i] = imap[adjncy[i]];  /* Apply local to global transformation */
+  }
+
+  /* Free Metis's things */
+  GKfree((void **)&graph->match, 
+         (void **)&graph->cmap, 
+         (void **)&graph->lperm, 
+         (void **)&graph->where, 
+         (void **)&graph->label, 
+         (void **)&graph->rinfo, 
+         (void **)&graph->nrinfo, 
+         (void **)&graph->nvwgt, 
+         (void **)&graph->lpwgts, 
+         (void **)&graph->gpwgts, 
+         (void **)&graph->lnpwgts, 
+         (void **)&graph->gnpwgts, 
+         (void **)&graph->sepind,
+         (void **)&graph->peind, 
+         (void **)&graph->sendptr, 
+         (void **)&graph->sendind, 
+         (void **)&graph->recvptr, 
+         (void **)&graph->recvind, 
+         (void **)&graph->imap,
+         (void **)&graph->rlens,
+         (void **)&graph->slens,
+         (void **)&graph->rcand,
+         (void **)&graph->pexadj,
+         (void **)&graph->peadjncy,
+         (void **)&graph->peadjloc,
+         LTERM);
+
+  if ((wgtflag&2) == 0) 
+    GKfree((void **)&graph->vwgt, (void **)&graph->vsize, LTERM);
+  if ((wgtflag&1) == 0) 
+    GKfree((void **)&graph->adjwgt, LTERM);
+
+  free(graph);
+}
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/mesh.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/mesh.c
new file mode 100644
index 0000000..57a80c0
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/mesh.c
@@ -0,0 +1,335 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * mesh.c
+ *
+ * This file contains routines for constructing the dual graph of a mesh.
+ * Assumes that each processor has at least one mesh element.
+ *
+ * Started 10/19/94
+ * George
+ *
+ * $Id: mesh.c,v 1.11 2003/07/25 04:01:04 karypis Exp $
+ *
+ */
+
+#include <parmetislib.h>
+
+
+/*************************************************************************
+* This function converts a mesh into a dual graph
+**************************************************************************/
+void ParMETIS_V3_Mesh2Dual(idxtype *elmdist, idxtype *eptr, idxtype *eind, 
+                 int *numflag, int *ncommonnodes, idxtype **xadj, 
+		 idxtype **adjncy, MPI_Comm *comm)
+{
+  int i, j, jj, k, kk, m;
+  int npes, mype, pe, count, mask, pass;
+  int nelms, lnns, my_nns, node;
+  int firstelm, firstnode, lnode, nrecv, nsend;
+  int *scounts, *rcounts, *sdispl, *rdispl;
+  idxtype *nodedist, *nmap, *auxarray;
+  idxtype *gnptr, *gnind, *nptr, *nind, *myxadj, *myadjncy = NULL;
+  idxtype *sbuffer, *rbuffer, *htable;
+  KeyValueType *nodelist, *recvbuffer;
+  idxtype ind[200], wgt[200];
+  int gmaxnode, gminnode;
+  CtrlType ctrl;
+
+
+  SetUpCtrl(&ctrl, -1, 0, *comm);
+
+  npes = ctrl.npes;
+  mype = ctrl.mype;
+
+  nelms = elmdist[mype+1]-elmdist[mype];
+
+  if (*numflag == 1) 
+    ChangeNumberingMesh2(elmdist, eptr, eind, NULL, NULL, NULL, npes, mype, 1);
+
+  mask = (1<<11)-1;
+
+  /*****************************/
+  /* Determine number of nodes */
+  /*****************************/
+  gminnode = GlobalSEMin(&ctrl, eind[idxamin(eptr[nelms], eind)]);
+  for (i=0; i<eptr[nelms]; i++)
+    eind[i] -= gminnode;
+
+  gmaxnode = GlobalSEMax(&ctrl, eind[idxamax(eptr[nelms], eind)]);
+
+
+  /**************************/
+  /* Check for input errors */
+  /**************************/
+  ASSERTS(nelms > 0);
+
+  /* construct node distribution array */
+  nodedist = idxsmalloc(npes+1, 0, "nodedist");
+  for (nodedist[0]=0, i=0,j=gmaxnode+1; i<npes; i++) {
+    k = j/(npes-i);
+    nodedist[i+1] = nodedist[i]+k;
+    j -= k;
+  }
+  my_nns = nodedist[mype+1]-nodedist[mype];
+  firstnode = nodedist[mype];
+
+  nodelist = (KeyValueType *)GKmalloc(eptr[nelms]*sizeof(KeyValueType), "nodelist");
+  auxarray = idxmalloc(eptr[nelms], "auxarray");
+  htable   = idxsmalloc(amax(my_nns, mask+1), -1, "htable");
+  scounts  = imalloc(4*npes+2, "scounts");
+  rcounts  = scounts+npes;
+  sdispl   = scounts+2*npes;
+  rdispl   = scounts+3*npes+1;
+
+
+  /*********************************************/
+  /* first find a local numbering of the nodes */
+  /*********************************************/
+  for (i=0; i<nelms; i++) {
+    for (j=eptr[i]; j<eptr[i+1]; j++) {
+      nodelist[j].key = eind[j];
+      nodelist[j].val = j;
+      auxarray[j]     = i; /* remember the local element ID that uses this node */
+    }
+  }
+  ikeysort(eptr[nelms], nodelist);
+
+  for (count=1, i=1; i<eptr[nelms]; i++) {
+    if (nodelist[i].key > nodelist[i-1].key)
+      count++;
+  }
+
+  lnns = count;
+  nmap = idxmalloc(lnns, "nmap");
+
+  /* renumber the nodes of the elements array */
+  count = 1;
+  nmap[0] = nodelist[0].key;
+  eind[nodelist[0].val] = 0;
+  nodelist[0].val = auxarray[nodelist[0].val];  /* Store the local element ID */
+  for (i=1; i<eptr[nelms]; i++) {
+    if (nodelist[i].key > nodelist[i-1].key) {
+      nmap[count] = nodelist[i].key;
+      count++;
+    }
+    eind[nodelist[i].val] = count-1;
+    nodelist[i].val = auxarray[nodelist[i].val];  /* Store the local element ID */
+  }
+  MPI_Barrier(*comm);
+
+  /**********************************************************/
+  /* perform comms necessary to construct node-element list */
+  /**********************************************************/
+  iset(npes, 0, scounts);
+  for (pe=i=0; i<eptr[nelms]; i++) {
+    while (nodelist[i].key >= nodedist[pe+1])
+      pe++;
+    scounts[pe] += 2;
+  }
+  ASSERTS(pe < npes);
+
+  MPI_Alltoall((void *)scounts, 1, MPI_INT, (void *)rcounts, 1, MPI_INT, *comm);
+
+  icopy(npes, scounts, sdispl);
+  MAKECSR(i, npes, sdispl);
+
+  icopy(npes, rcounts, rdispl);
+  MAKECSR(i, npes, rdispl);
+
+  ASSERTS(sdispl[npes] == eptr[nelms]*2);
+
+  nrecv = rdispl[npes]/2;
+  recvbuffer = (KeyValueType *)GKmalloc(amax(1, nrecv)*sizeof(KeyValueType), "recvbuffer");
+
+  MPI_Alltoallv((void *)nodelist, scounts, sdispl, IDX_DATATYPE, (void *)recvbuffer, 
+                rcounts, rdispl, IDX_DATATYPE, *comm);
+
+  /**************************************/
+  /* construct global node-element list */
+  /**************************************/
+  gnptr = idxsmalloc(my_nns+1, 0, "gnptr");
+
+  for (i=0; i<npes; i++) {
+    for (j=rdispl[i]/2; j<rdispl[i+1]/2; j++) {
+      lnode = recvbuffer[j].key-firstnode;
+      ASSERTS(lnode >= 0 && lnode < my_nns)
+
+      gnptr[lnode]++;
+    }
+  }
+  MAKECSR(i, my_nns, gnptr);
+
+  gnind = idxmalloc(amax(1, gnptr[my_nns]), "gnind");
+  for (pe=0; pe<npes; pe++) {
+    firstelm = elmdist[pe];
+    for (j=rdispl[pe]/2; j<rdispl[pe+1]/2; j++) {
+      lnode = recvbuffer[j].key-firstnode;
+      gnind[gnptr[lnode]++] = recvbuffer[j].val+firstelm;
+    }
+  }
+  SHIFTCSR(i, my_nns, gnptr);
+
+
+  /*********************************************************/
+  /* send the node-element info to the relevant processors */
+  /*********************************************************/
+  iset(npes, 0, scounts);
+
+  /* use a hash table to ensure that each node is sent to a proc only once */
+  for (pe=0; pe<npes; pe++) {
+    for (j=rdispl[pe]/2; j<rdispl[pe+1]/2; j++) {
+      lnode = recvbuffer[j].key-firstnode;
+      if (htable[lnode] == -1) {
+        scounts[pe] += gnptr[lnode+1]-gnptr[lnode];
+        htable[lnode] = 1;
+      }
+    }
+
+    /* now reset the hash table */
+    for (j=rdispl[pe]/2; j<rdispl[pe+1]/2; j++) {
+      lnode = recvbuffer[j].key-firstnode;
+      htable[lnode] = -1;
+    }
+  }
+
+
+  MPI_Alltoall((void *)scounts, 1, MPI_INT, (void *)rcounts, 1, MPI_INT, *comm);
+
+  icopy(npes, scounts, sdispl);
+  MAKECSR(i, npes, sdispl);
+
+  /* create the send buffer */
+  nsend = sdispl[npes];
+  sbuffer = (idxtype *)realloc(nodelist, sizeof(idxtype)*amax(1, nsend));
+
+  count = 0;
+  for (pe=0; pe<npes; pe++) {
+    for (j=rdispl[pe]/2; j<rdispl[pe+1]/2; j++) {
+      lnode = recvbuffer[j].key-firstnode;
+      if (htable[lnode] == -1) {
+        for (k=gnptr[lnode]; k<gnptr[lnode+1]; k++) {
+          if (k == gnptr[lnode])
+            sbuffer[count++] = -1*(gnind[k]+1);
+          else
+            sbuffer[count++] = gnind[k];
+        }
+        htable[lnode] = 1;
+      }
+    }
+    ASSERTS(count == sdispl[pe+1]);
+
+    /* now reset the hash table */
+    for (j=rdispl[pe]/2; j<rdispl[pe+1]/2; j++) {
+      lnode = recvbuffer[j].key-firstnode;
+      htable[lnode] = -1;
+    }
+  }
+
+  icopy(npes, rcounts, rdispl);
+  MAKECSR(i, npes, rdispl);
+
+  nrecv = rdispl[npes];
+  rbuffer = (idxtype *)realloc(recvbuffer, sizeof(idxtype)*amax(1, nrecv));
+
+  MPI_Alltoallv((void *)sbuffer, scounts, sdispl, IDX_DATATYPE, (void *)rbuffer, 
+                rcounts, rdispl, IDX_DATATYPE, *comm);
+
+  k = -1;
+  nptr = idxsmalloc(lnns+1, 0, "nptr");
+  nind = rbuffer;
+  for (pe=0; pe<npes; pe++) {
+    for (j=rdispl[pe]; j<rdispl[pe+1]; j++) {
+      if (nind[j] < 0) {
+        k++;
+        nind[j] = (-1*nind[j])-1;
+      }
+      nptr[k]++;
+    }
+  }
+  MAKECSR(i, lnns, nptr);
+
+  ASSERTS(k+1 == lnns);
+  ASSERTS(nptr[lnns] == nrecv)
+
+  myxadj = *xadj = idxsmalloc(nelms+1, 0, "xadj");
+  idxset(mask+1, -1, htable);
+
+  firstelm = elmdist[mype];
+
+  /* Two passes -- in first pass, simply find out the memory requirements */
+  for (pass=0; pass<2; pass++) {
+    for (i=0; i<nelms; i++) {
+      for (count=0, j=eptr[i]; j<eptr[i+1]; j++) {
+        node = eind[j];
+
+        for (k=nptr[node]; k<nptr[node+1]; k++) {
+          if ((kk=nind[k]) == firstelm+i) 
+	    continue;
+	    
+          m = htable[(kk&mask)];
+
+          if (m == -1) {
+            ind[count] = kk;
+            wgt[count] = 1;
+            htable[(kk&mask)] = count++;
+          }
+          else {
+            if (ind[m] == kk) { 
+              wgt[m]++;
+            }
+            else {
+              for (jj=0; jj<count; jj++) {
+                if (ind[jj] == kk) {
+                  wgt[jj]++;
+                  break;
+	        }
+              }
+              if (jj == count) {
+                ind[count]   = kk;
+                wgt[count++] = 1;
+              }
+	    }
+          }
+        }
+      }
+
+      for (j=0; j<count; j++) {
+        htable[(ind[j]&mask)] = -1;
+        if (wgt[j] >= *ncommonnodes) {
+          if (pass == 0) 
+            myxadj[i]++;
+          else 
+            myadjncy[myxadj[i]++] = ind[j];
+	}
+      }
+    }
+
+    if (pass == 0) {
+      MAKECSR(i, nelms, myxadj);
+      myadjncy = *adjncy = idxmalloc(myxadj[nelms], "adjncy");
+    }
+    else {
+      SHIFTCSR(i, nelms, myxadj);
+    }
+  }
+
+  /*****************************************/
+  /* correctly renumber the elements array */
+  /*****************************************/
+  for (i=0; i<eptr[nelms]; i++)
+    eind[i] = nmap[eind[i]] + gminnode;
+
+  if (*numflag == 1) 
+    ChangeNumberingMesh2(elmdist, eptr, eind, myxadj, myadjncy, NULL, npes, mype, 0);
+
+  /* do not free nodelist, recvbuffer, rbuffer */
+  GKfree((void **)&scounts, (void **)&nodedist, (void **)&nmap, (void **)&sbuffer, 
+         (void **)&htable, (void **)&nptr, (void **)&nind, (void **)&gnptr, 
+	 (void **)&gnind, (void **)&auxarray, LTERM);
+
+  FreeCtrl(&ctrl);
+
+  return;
+}
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/mmetis.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/mmetis.c
new file mode 100644
index 0000000..b262ed1
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/mmetis.c
@@ -0,0 +1,95 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * mmetis.c
+ *
+ * This is the entry point of ParMETIS_V3_PartMeshKway
+ *
+ * Started 10/19/96
+ * George
+ *
+ * $Id: mmetis.c,v 1.8 2003/07/25 04:01:04 karypis Exp $
+ *
+ */
+
+#include <parmetislib.h>
+
+
+/***********************************************************************************
+* This function is the entry point of the parallel k-way multilevel mesh partitionioner. 
+* This function assumes nothing about the mesh distribution.
+* It is the general case.
+************************************************************************************/
+void ParMETIS_V3_PartMeshKway(idxtype *elmdist, idxtype *eptr, idxtype *eind, idxtype *elmwgt, 
+                 int *wgtflag, int *numflag, int *ncon, int *ncommonnodes, int *nparts, 
+		 float *tpwgts, float *ubvec, int *options, int *edgecut, idxtype *part, 
+		 MPI_Comm *comm)
+{
+  int i, nvtxs, nedges, gnedges, npes, mype;
+  idxtype *xadj, *adjncy;
+  timer TotalTmr, Mesh2DualTmr, ParMETISTmr;
+  CtrlType ctrl;
+
+  /********************************/
+  /* Try and take care bad inputs */
+  /********************************/
+  if (elmdist == NULL || eptr == NULL || eind == NULL || wgtflag == NULL || 
+      numflag == NULL || ncon == NULL || ncommonnodes == NULL || nparts == NULL ||
+      tpwgts == NULL || ubvec == NULL || options == NULL || edgecut == NULL || 
+      part == NULL || comm == NULL) {
+    printf("ERROR: One or more required parameters is NULL. Aborting.\n");
+    abort();
+  }
+  if (((*wgtflag)&2) && elmwgt == NULL) {
+    printf("ERROR: elmwgt == NULL when vertex weights were specified. Aborting.\n");
+    abort();
+  }
+
+  
+  SetUpCtrl(&ctrl, *nparts, (options[0] == 1 ? options[PMV3_OPTION_DBGLVL] : 0), *comm);
+  npes = ctrl.npes;
+  mype = ctrl.mype;
+
+  cleartimer(TotalTmr);
+  cleartimer(Mesh2DualTmr);
+  cleartimer(ParMETISTmr);
+
+  MPI_Barrier(ctrl.comm);
+  starttimer(TotalTmr);
+  starttimer(Mesh2DualTmr);
+
+  ParMETIS_V3_Mesh2Dual(elmdist, eptr, eind, numflag, ncommonnodes, &xadj, &adjncy, &(ctrl.comm));
+
+  if (ctrl.dbglvl&DBG_INFO) {
+    nvtxs = elmdist[mype+1]-elmdist[mype];
+    nedges = xadj[nvtxs] + (*numflag == 0 ? 0 : -1);
+    rprintf(&ctrl, "Completed Dual Graph -- Nvtxs: %d, Nedges: %d \n", 
+            elmdist[npes], GlobalSESum(&ctrl, nedges));
+  }
+
+  MPI_Barrier(ctrl.comm);
+  stoptimer(Mesh2DualTmr);
+
+
+  /***********************/
+  /* Partition the graph */
+  /***********************/
+  starttimer(ParMETISTmr);
+
+  ParMETIS_V3_PartKway(elmdist, xadj, adjncy, elmwgt, NULL, wgtflag, numflag, ncon, 
+                       nparts, tpwgts, ubvec, options, edgecut, part, &(ctrl.comm));
+
+  MPI_Barrier(ctrl.comm);
+  stoptimer(ParMETISTmr);
+  stoptimer(TotalTmr);
+
+  IFSET(ctrl.dbglvl, DBG_TIME, PrintTimer(&ctrl, Mesh2DualTmr,	"   Mesh2Dual"));
+  IFSET(ctrl.dbglvl, DBG_TIME, PrintTimer(&ctrl, ParMETISTmr,	"    ParMETIS"));
+  IFSET(ctrl.dbglvl, DBG_TIME, PrintTimer(&ctrl, TotalTmr,	"       Total"));
+
+  GKfree((void **)&xadj, (void **)&adjncy, LTERM);
+
+  FreeCtrl(&ctrl);
+
+  return;
+}
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/move.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/move.c
new file mode 100644
index 0000000..06a8e5c
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/move.c
@@ -0,0 +1,338 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * mmove.c
+ *
+ * This file contains functions that move the graph given a partition
+ *
+ * Started 11/22/96
+ * George
+ *
+ * $Id: move.c,v 1.3 2003/07/31 16:23:30 karypis Exp $
+ *
+ */
+
+#include <parmetislib.h>
+
+/*************************************************************************
+* This function moves the graph, and returns a new graph.
+* This routine can be called with or without performing refinement.
+* In the latter case it allocates and computes lpwgts itself.
+**************************************************************************/
+GraphType *Moc_MoveGraph(CtrlType *ctrl, GraphType *graph, WorkSpaceType *wspace)
+{
+  int h, i, ii, j, jj, nvtxs, ncon, nparts;
+  idxtype *xadj, *vwgt, *adjncy, *adjwgt, *mvtxdist;
+  idxtype *where, *newlabel, *lpwgts, *gpwgts;
+  idxtype *sgraph, *rgraph;
+  KeyValueType *sinfo, *rinfo;
+  GraphType *mgraph;
+
+  nparts = ctrl->nparts;
+  ASSERT(ctrl, nparts == ctrl->npes);
+
+  nvtxs = graph->nvtxs;
+  ncon = graph->ncon;
+  xadj = graph->xadj;
+  vwgt = graph->vwgt;
+  adjncy = graph->adjncy;
+  adjwgt = graph->adjwgt;
+  where = graph->where;
+
+  mvtxdist = idxmalloc(nparts+1, "MoveGraph: mvtxdist");
+
+  /* Let's do a prefix scan to determine the labeling of the nodes given */
+  lpwgts = wspace->pv1;
+  gpwgts = wspace->pv2;
+  sinfo = wspace->pepairs1;
+  rinfo = wspace->pepairs2;
+  for (i=0; i<nparts; i++)
+    sinfo[i].key = sinfo[i].val = 0;
+
+  for (i=0; i<nvtxs; i++) {
+    sinfo[where[i]].key++;
+    sinfo[where[i]].val += xadj[i+1]-xadj[i];
+  }
+  for (i=0; i<nparts; i++)
+    lpwgts[i] = sinfo[i].key;
+
+  MPI_Scan((void *)lpwgts, (void *)gpwgts, nparts, IDX_DATATYPE, MPI_SUM, ctrl->comm);
+  MPI_Allreduce((void *)lpwgts, (void *)mvtxdist, nparts, IDX_DATATYPE, MPI_SUM, ctrl->comm);
+
+  MAKECSR(i, nparts, mvtxdist);
+
+  /* gpwgts[i] will store the label of the first vertex for each domain in each processor */
+  for (i=0; i<nparts; i++)
+    /* We were interested in an exclusive Scan */
+    gpwgts[i] = mvtxdist[i] + gpwgts[i] - lpwgts[i];
+
+  newlabel = idxmalloc(nvtxs+graph->nrecv, "MoveGraph: newlabel");
+
+  for (i=0; i<nvtxs; i++) 
+    newlabel[i] = gpwgts[where[i]]++;
+
+  /* OK, now send the newlabel info to processors storing adjacent interface nodes */
+  CommInterfaceData(ctrl, graph, newlabel, wspace->indices, newlabel+nvtxs);
+
+  /* Now lets tell everybody what and from where he will get it. Assume nparts == npes */
+  MPI_Alltoall((void *)sinfo, 2, IDX_DATATYPE, (void *)rinfo, 2, IDX_DATATYPE, ctrl->comm);
+
+  /* Use lpwgts and gpwgts as pointers to where data will be received and send */
+  lpwgts[0] = 0;  /* Send part */
+  gpwgts[0] = 0;  /* Received part */
+  for (i=0; i<nparts; i++) {
+    lpwgts[i+1] = lpwgts[i] + (1+ncon)*sinfo[i].key + 2*sinfo[i].val;
+    gpwgts[i+1] = gpwgts[i] + (1+ncon)*rinfo[i].key + 2*rinfo[i].val;
+  }
+
+
+  if (lpwgts[nparts]+gpwgts[nparts] > wspace->maxcore) {
+    /* Adjust core memory, incase the graph was originally very memory unbalanced */
+    free(wspace->core);
+    wspace->maxcore = lpwgts[nparts]+4*gpwgts[nparts]; /* In spirit of the 8*nedges */
+    wspace->core = idxmalloc(wspace->maxcore, "Moc_MoveGraph: wspace->core");
+  }
+
+  sgraph = wspace->core;
+  rgraph = wspace->core + lpwgts[nparts];
+
+  /* Issue the receives first */
+  for (i=0; i<nparts; i++) {
+    if (rinfo[i].key > 0) 
+      MPI_Irecv((void *)(rgraph+gpwgts[i]), gpwgts[i+1]-gpwgts[i], IDX_DATATYPE, i, 1, ctrl->comm, ctrl->rreq+i);
+    else
+      ASSERT(ctrl, gpwgts[i+1]-gpwgts[i] == 0);
+  }
+
+  /* Assemble the graph to be sent and send it */
+  for (i=0; i<nvtxs; i++) {
+    ii = lpwgts[where[i]];
+    sgraph[ii++] = xadj[i+1]-xadj[i];
+    for (h=0; h<ncon; h++)
+      sgraph[ii++] = vwgt[i*ncon+h];
+    for (j=xadj[i]; j<xadj[i+1]; j++) {
+      sgraph[ii++] = newlabel[adjncy[j]];
+      sgraph[ii++] = adjwgt[j];
+    }
+    lpwgts[where[i]] = ii;
+  }
+
+  for (i=nparts; i>0; i--)
+    lpwgts[i] = lpwgts[i-1];
+  lpwgts[0] = 0;
+
+  for (i=0; i<nparts; i++) {
+    if (sinfo[i].key > 0)
+      MPI_Isend((void *)(sgraph+lpwgts[i]), lpwgts[i+1]-lpwgts[i], IDX_DATATYPE, i, 1, ctrl->comm, ctrl->sreq+i);
+    else
+      ASSERT(ctrl, lpwgts[i+1]-lpwgts[i] == 0);
+  }
+
+/*
+#ifdef DMALLOC
+  ASSERT(ctrl, dmalloc_verify(NULL) == DMALLOC_VERIFY_NOERROR);
+#endif
+*/
+
+  /* Wait for the send/recv to finish */
+  for (i=0; i<nparts; i++) {
+    if (sinfo[i].key > 0) 
+      MPI_Wait(ctrl->sreq+i, &ctrl->status);
+  }
+  for (i=0; i<nparts; i++) {
+    if (rinfo[i].key > 0) 
+      MPI_Wait(ctrl->rreq+i, &ctrl->status); 
+  }
+
+  /* OK, now go and put the graph into GraphType Format */
+  mgraph = CreateGraph();
+  mgraph->gnvtxs = graph->gnvtxs;
+  mgraph->ncon = ncon;
+  mgraph->level = 0;
+  mgraph->nvtxs = mgraph->nedges = 0;
+  for (i=0; i<nparts; i++) {
+    mgraph->nvtxs += rinfo[i].key;
+    mgraph->nedges += rinfo[i].val;
+  }
+  nvtxs = mgraph->nvtxs;
+  xadj = mgraph->xadj = idxmalloc(nvtxs+1, "MMG: mgraph->xadj");
+  vwgt = mgraph->vwgt = idxmalloc(nvtxs*ncon, "MMG: mgraph->vwgt");
+  adjncy = mgraph->adjncy = idxmalloc(mgraph->nedges, "MMG: mgraph->adjncy");
+  adjwgt = mgraph->adjwgt = idxmalloc(mgraph->nedges, "MMG: mgraph->adjwgt");
+  mgraph->vtxdist = mvtxdist;
+
+  for (jj=ii=i=0; i<nvtxs; i++) {
+    xadj[i] = rgraph[ii++];
+    for (h=0; h<ncon; h++)
+      vwgt[i*ncon+h] = rgraph[ii++]; 
+    for (j=0; j<xadj[i]; j++) {
+      adjncy[jj] = rgraph[ii++];
+      adjwgt[jj++] = rgraph[ii++];
+    }
+  }
+  MAKECSR(i, nvtxs, xadj);
+
+  ASSERTP(ctrl, jj == mgraph->nedges, (ctrl, "%d %d\n", jj, mgraph->nedges));
+  ASSERTP(ctrl, ii == gpwgts[nparts], (ctrl, "%d %d %d %d %d\n", ii, gpwgts[nparts], jj, mgraph->nedges, nvtxs));
+
+  free(newlabel);
+
+#ifdef DEBUG
+  IFSET(ctrl->dbglvl, DBG_INFO, rprintf(ctrl, "Checking moved graph...\n"));
+  CheckMGraph(ctrl, mgraph);
+  IFSET(ctrl->dbglvl, DBG_INFO, rprintf(ctrl, "Moved graph is consistent.\n"));
+#endif
+
+  return mgraph;
+}
+
+
+/*************************************************************************
+* This function is used to transfer information from the moved graph
+* back to the original graph. The information is transfered from array
+* minfo to array info. The routine assumes that graph->where is left intact
+* and it is used to get the inverse mapping information.
+* The routine assumes that graph->where corresponds to a npes-way partition.
+**************************************************************************/
+void ProjectInfoBack(CtrlType *ctrl, GraphType *graph, idxtype *info, idxtype *minfo, 
+                     WorkSpaceType *wspace)
+{
+  int i, nvtxs, nparts;
+  idxtype *where, *auxinfo, *sinfo, *rinfo;
+
+  nparts = ctrl->npes;
+
+  nvtxs = graph->nvtxs;
+  where = graph->where;
+
+  sinfo   = wspace->pv1;
+  rinfo   = wspace->pv2;
+
+  /* Find out in rinfo how many entries are received per partition */
+  idxset(nparts, 0, rinfo);
+  for (i=0; i<nvtxs; i++)
+    rinfo[where[i]]++;
+
+  /* The rinfo are transposed and become the sinfo for the back-projection */
+  MPI_Alltoall((void *)rinfo, 1, IDX_DATATYPE, (void *)sinfo, 1, IDX_DATATYPE, ctrl->comm);
+
+  MAKECSR(i, nparts, sinfo);
+  MAKECSR(i, nparts, rinfo);
+
+  /* allocate memory for auxinfo */
+  auxinfo = idxmalloc(rinfo[nparts], "ProjectInfoBack: auxinfo");
+
+  /*-----------------------------------------------------------------
+   * Now, go and send back the minfo
+   -----------------------------------------------------------------*/
+  for (i=0; i<nparts; i++) {
+    if (rinfo[i+1]-rinfo[i] > 0)
+      MPI_Irecv((void *)(auxinfo+rinfo[i]), rinfo[i+1]-rinfo[i], IDX_DATATYPE, i, 1, ctrl->comm, ctrl->rreq+i);
+  }
+
+  for (i=0; i<nparts; i++) {
+    if (sinfo[i+1]-sinfo[i] > 0)
+      MPI_Isend((void *)(minfo+sinfo[i]), sinfo[i+1]-sinfo[i], IDX_DATATYPE, i, 1, ctrl->comm, ctrl->sreq+i);
+  }
+
+  /* Wait for the send/recv to finish */
+  for (i=0; i<nparts; i++) {
+    if (rinfo[i+1]-rinfo[i] > 0)
+      MPI_Wait(ctrl->rreq+i, &ctrl->status);
+  }
+  for (i=0; i<nparts; i++) {
+    if (sinfo[i+1]-sinfo[i] > 0)
+      MPI_Wait(ctrl->sreq+i, &ctrl->status);
+  }
+
+  /* Scatter the info received in auxinfo back to info. */
+  for (i=0; i<nvtxs; i++)
+    info[i] = auxinfo[rinfo[where[i]]++];
+
+  free(auxinfo);
+}
+
+
+
+/*************************************************************************
+* This function is used to convert a partition vector to a permutation
+* vector.
+**************************************************************************/
+void FindVtxPerm(CtrlType *ctrl, GraphType *graph, idxtype *perm, WorkSpaceType *wspace)
+{
+  int i, nvtxs, nparts;
+  idxtype *xadj, *adjncy, *adjwgt, *mvtxdist;
+  idxtype *where, *lpwgts, *gpwgts;
+
+  nparts = ctrl->nparts;
+
+  nvtxs = graph->nvtxs;
+  xadj = graph->xadj;
+  adjncy = graph->adjncy;
+  adjwgt = graph->adjwgt;
+  where = graph->where;
+
+  mvtxdist = idxmalloc(nparts+1, "MoveGraph: mvtxdist");
+
+  /* Let's do a prefix scan to determine the labeling of the nodes given */
+  lpwgts = wspace->pv1;
+  gpwgts = wspace->pv2;
+
+  /* Here we care about the count and not total weight (diff since graph may be weighted */
+  idxset(nparts, 0, lpwgts);
+  for (i=0; i<nvtxs; i++)
+    lpwgts[where[i]]++;
+
+  MPI_Scan((void *)lpwgts, (void *)gpwgts, nparts, IDX_DATATYPE, MPI_SUM, ctrl->comm);
+  MPI_Allreduce((void *)lpwgts, (void *)mvtxdist, nparts, IDX_DATATYPE, MPI_SUM, ctrl->comm);
+
+  MAKECSR(i, nparts, mvtxdist);
+
+  for (i=0; i<nparts; i++)
+    gpwgts[i] = mvtxdist[i] + gpwgts[i] - lpwgts[i];  /* We were interested in an exclusive Scan */
+
+  for (i=0; i<nvtxs; i++)
+    perm[i] = gpwgts[where[i]]++;
+
+  free(mvtxdist);
+
+}
+
+
+
+
+/*************************************************************************
+* This function quickly performs a check on the consistency of moved graph.
+**************************************************************************/
+void CheckMGraph(CtrlType *ctrl, GraphType *graph)
+{
+  int i, j, jj, k, nvtxs, firstvtx, lastvtx;
+  idxtype *xadj, *adjncy, *vtxdist;
+
+
+  nvtxs = graph->nvtxs;
+  xadj = graph->xadj;
+  adjncy = graph->adjncy;
+  vtxdist = graph->vtxdist;
+
+  firstvtx = vtxdist[ctrl->mype];
+  lastvtx = vtxdist[ctrl->mype+1];
+
+  for (i=0; i<nvtxs; i++) {
+    for (j=xadj[i]; j<xadj[i+1]; j++) {
+      ASSERT(ctrl, firstvtx+i != adjncy[j]);
+      if (adjncy[j] >= firstvtx && adjncy[j] < lastvtx) {
+        k = adjncy[j]-firstvtx;
+        for (jj=xadj[k]; jj<xadj[k+1]; jj++) {
+          if (adjncy[jj] == firstvtx+i)
+            break;
+        }
+        if (jj == xadj[k+1])
+          myprintf(ctrl, "(%d %d) but not (%d %d)\n", firstvtx+i, k, k, firstvtx+i);
+      }
+    }
+  }
+}
+
+
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/msetup.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/msetup.c
new file mode 100644
index 0000000..decfc65
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/msetup.c
@@ -0,0 +1,95 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * msetup.c
+ *
+ * This file contain various routines for setting up a mesh
+ *
+ * Started 10/19/96
+ * George
+ *
+ * $Id: msetup.c,v 1.3 2003/07/31 06:14:01 karypis Exp $
+ *
+ */
+
+#include <parmetis.h>
+
+
+
+/*************************************************************************
+* This function setsup the CtrlType structure
+**************************************************************************/
+MeshType *SetUpMesh(int *etype, int *ncon, idxtype *elmdist, idxtype *elements,
+  idxtype *elmwgt, int *wgtflag, MPI_Comm *comm)
+{
+  MeshType *mesh;
+  int i, npes, mype;
+  int esizes[5] = {-1, 3, 4, 8, 4};
+  int maxnode, gmaxnode, minnode, gminnode;
+
+  MPI_Comm_size(*comm, &npes);
+  MPI_Comm_rank(*comm, &mype);
+
+  mesh = CreateMesh();
+  mesh->elmdist = elmdist;
+  mesh->gnelms = elmdist[npes];
+  mesh->nelms = elmdist[mype+1]-elmdist[mype];
+  mesh->elements = elements;
+  mesh->elmwgt = elmwgt;
+  mesh->etype = *etype;
+  mesh->ncon = *ncon;
+  mesh->esize = esizes[*etype];
+
+  if (((*wgtflag)&1) == 0) {
+    mesh->elmwgt = idxsmalloc(mesh->nelms*mesh->ncon, 1, "SetUpMesh: elmwgt");
+  }
+
+  minnode = elements[idxamin(mesh->nelms*mesh->esize, elements)];
+  MPI_Allreduce((void *)&minnode, (void *)&gminnode, 1, MPI_INT, MPI_MIN, *comm);
+  for (i=0; i<mesh->nelms*mesh->esize; i++)
+    elements[i] -= gminnode;
+  mesh->gminnode = gminnode;
+
+  maxnode = elements[idxamax(mesh->nelms*mesh->esize, elements)];
+  MPI_Allreduce((void *)&maxnode, (void *)&gmaxnode, 1, MPI_INT, MPI_MAX, *comm);
+  mesh->gnns = gmaxnode+1;
+
+  return mesh;
+}
+
+/*************************************************************************
+* This function creates a MeshType data structure and initializes
+* the various fields
+**************************************************************************/
+MeshType *CreateMesh(void)
+{
+  MeshType *mesh;
+
+  mesh = (MeshType *)GKmalloc(sizeof(MeshType), "CreateMesh: mesh");
+
+  InitMesh(mesh);
+
+  return mesh;
+}
+
+/*************************************************************************
+* This function initializes the various fields of a MeshType.
+**************************************************************************/
+void InitMesh(MeshType *mesh)
+{
+
+  mesh->etype = -1;
+  mesh->gnelms = -1;
+  mesh->gnns = -1;
+  mesh->nelms = -1;
+  mesh->nns = -1;
+  mesh->ncon = -1;
+  mesh->esize = -1;
+  mesh->gminnode = 0;
+  mesh->elmdist = NULL;
+  mesh->elements = NULL;
+  mesh->elmwgt = NULL;
+
+  return;
+}
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/node_refine.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/node_refine.c
new file mode 100644
index 0000000..3f57aa7
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/node_refine.c
@@ -0,0 +1,383 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * node_refine.c
+ *
+ * This file contains code that performs the k-way refinement
+ *
+ * Started 3/1/96
+ * George
+ *
+ * $Id: node_refine.c,v 1.2 2003/07/21 17:18:50 karypis Exp $
+ */
+
+#include <parmetislib.h>
+
+#define PackWeightWhereInfo(a, b) (((a)<<10) + (b))
+#define SelectWhere(a) ((a)%1024)
+#define SelectWeight(a) (((a)>>10))
+
+
+
+/*************************************************************************
+* This function computes the initial id/ed 
+**************************************************************************/
+void ComputeNodePartitionParams(CtrlType *ctrl, GraphType *graph, WorkSpaceType *wspace)
+{
+  int i, j, nparts, nvtxs, nsep, firstvtx, lastvtx;
+  idxtype *xadj, *ladjncy, *adjwgt, *vtxdist, *vwgt, *lpwgts, *gpwgts, *sepind;
+  idxtype *where, *swhere, *rwhere;
+  NRInfoType *rinfo, *myrinfo;
+  int me, other, otherwgt;
+
+  IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->KWayInitTmr));
+
+  nvtxs = graph->nvtxs;
+  nparts = ctrl->nparts;
+
+  vtxdist = graph->vtxdist;
+  xadj = graph->xadj;
+  ladjncy = graph->adjncy;
+  adjwgt = graph->adjwgt;
+  vwgt = graph->vwgt;
+
+  where = graph->where;
+  rinfo = graph->nrinfo = (NRInfoType *)GKmalloc(sizeof(NRInfoType)*nvtxs, "ComputeNodePartitionParams: rinfo");
+  lpwgts = graph->lpwgts = idxsmalloc(2*nparts, 0, "ComputePartitionParams: lpwgts");
+  gpwgts = graph->gpwgts = idxmalloc(2*nparts, "ComputePartitionParams: gpwgts");
+  sepind = graph->sepind = idxmalloc(nvtxs, "ComputePartitionParams: sepind");
+
+  firstvtx = vtxdist[ctrl->mype];
+  lastvtx = vtxdist[ctrl->mype+1];
+
+  /*------------------------------------------------------------
+  / Send/Receive the where information of interface vertices.
+  / Also use this to also encode the vwgt information of this
+  / vertex. This is a hack, but it should work for now!
+  /------------------------------------------------------------*/
+  swhere = wspace->indices;
+  rwhere = where + nvtxs;
+
+  for (i=0; i<nvtxs; i++) {
+    ASSERTP(ctrl, where[i] >= 0 && where[i] < 2*nparts, (ctrl, "%d\n", where[i]) );
+    where[i] = PackWeightWhereInfo(vwgt[i], where[i]);
+  }
+
+  CommInterfaceData(ctrl, graph, where, swhere, rwhere); 
+
+  /*------------------------------------------------------------
+  / Compute now the degrees
+  /------------------------------------------------------------*/
+  for (nsep=i=0; i<nvtxs; i++) {
+    me = SelectWhere(where[i]);
+    ASSERT(ctrl, me >= 0 && me < 2*nparts);
+    lpwgts[me] += vwgt[i];
+
+    if (me >= nparts) {  /* If it is a separator vertex */
+      sepind[nsep++] = i;
+      lpwgts[2*nparts-1] += vwgt[i];
+
+      myrinfo = rinfo+i;
+      myrinfo->edegrees[0] = myrinfo->edegrees[1] = 0;
+
+      for (j=xadj[i]; j<xadj[i+1]; j++) {
+        other = SelectWhere(where[ladjncy[j]]);
+        otherwgt = SelectWeight(where[ladjncy[j]]);
+        if (me != other)
+          myrinfo->edegrees[other%2] += otherwgt;
+      }
+    }
+  }
+  graph->nsep = nsep;
+
+  /* Finally, sum-up the partition weights */
+  MPI_Allreduce((void *)lpwgts, (void *)gpwgts, 2*nparts, IDX_DATATYPE, MPI_SUM, ctrl->comm);
+  graph->mincut = gpwgts[2*nparts-1];
+
+#ifdef XX
+  /* Print Weight information */
+  if (ctrl->mype == 0) {
+    for (i=0; i<nparts; i+=2) 
+      printf("[%5d %5d %5d] ", gpwgts[i], gpwgts[i+1], gpwgts[nparts+i]); 
+    printf("\n");
+  }
+#endif
+
+  IFSET(ctrl->dbglvl, DBG_TIME, stoptimer(ctrl->KWayInitTmr));
+}
+
+
+
+/*************************************************************************
+* This function performs k-way refinement
+**************************************************************************/
+void KWayNodeRefine(CtrlType *ctrl, GraphType *graph, WorkSpaceType *wspace, int npasses, float ubfraction)
+{
+  int i, ii, j, k, pass, nvtxs, firstvtx, lastvtx, otherlastvtx, c, nmoves, 
+      nlupd, nsupd, nnbrs, nchanged, nsep;
+  int npes = ctrl->npes, mype = ctrl->mype, nparts = ctrl->nparts;
+  idxtype *xadj, *ladjncy, *adjwgt, *vtxdist, *vwgt;
+  idxtype *where, *lpwgts, *gpwgts, *sepind;
+  idxtype *peind, *recvptr, *sendptr;
+  idxtype *update, *supdate, *rupdate, *pe_updates, *htable, *changed;
+  idxtype *badminpwgt, *badmaxpwgt;
+  KeyValueType *swchanges, *rwchanges;
+  int *nupds_pe;
+  NRInfoType *rinfo, *myrinfo;
+  int from, me, other, otherwgt, oldcut;
+
+  IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->KWayTmr));
+
+  nvtxs = graph->nvtxs;
+
+  vtxdist = graph->vtxdist;
+  xadj = graph->xadj;
+  ladjncy = graph->adjncy;
+  adjwgt = graph->adjwgt;
+  vwgt = graph->vwgt;
+
+  firstvtx = vtxdist[mype];
+  lastvtx = vtxdist[mype+1];
+
+  where = graph->where;
+  rinfo = graph->nrinfo;
+  lpwgts = graph->lpwgts;
+  gpwgts = graph->gpwgts;
+
+  nsep = graph->nsep;
+  sepind = graph->sepind;
+
+  nnbrs = graph->nnbrs;
+  peind = graph->peind;
+  recvptr = graph->recvptr;
+  sendptr = graph->sendptr;
+
+  changed = idxmalloc(nvtxs, "KWayRefine: changed");
+  rwchanges = wspace->pairs;
+  swchanges = rwchanges + recvptr[nnbrs];
+
+  update = idxmalloc(nvtxs, "KWayRefine: update");
+  supdate = wspace->indices;
+  rupdate = supdate + recvptr[nnbrs];
+  nupds_pe = imalloc(npes, "KWayRefine: nupds_pe");
+
+  htable = idxsmalloc(nvtxs+graph->nrecv, 0, "KWayRefine: lhtable");
+
+  badminpwgt = wspace->pv1;
+  badmaxpwgt = wspace->pv2;
+
+  for (i=0; i<nparts; i+=2) {
+    badminpwgt[i] = badminpwgt[i+1] = (1.0/ubfraction)*(gpwgts[i]+gpwgts[i+1])/2;
+    badmaxpwgt[i] = badmaxpwgt[i+1] = ubfraction*(gpwgts[i]+gpwgts[i+1])/2;
+  }
+
+  IFSET(ctrl->dbglvl, DBG_REFINEINFO, PrintNodeBalanceInfo(ctrl, nparts, gpwgts, badminpwgt, badmaxpwgt, 1));
+
+  for (pass=0; pass<npasses; pass++) {
+    oldcut = graph->mincut;
+
+    for (c=0; c<2; c++) {
+      for (i=0; i<nparts; i+=2) {
+        badminpwgt[i] = badminpwgt[i+1] = (1.0/ubfraction)*(gpwgts[i]+gpwgts[i+1])/2;
+        badmaxpwgt[i] = badmaxpwgt[i+1] = ubfraction*(gpwgts[i]+gpwgts[i+1])/2;
+      }
+
+      nlupd = nsupd = nmoves = nchanged = 0;
+      for (ii=0; ii<nsep; ii++) {
+        i = sepind[ii];
+        from = SelectWhere(where[i]);
+
+        ASSERT(ctrl, from >= nparts);
+
+        /* Go through the loop if gain is possible for the separator vertex */
+        if (rinfo[i].edegrees[(c+1)%2] <= vwgt[i]) {
+          other = from%nparts+c;  /* It is one-sided move so we know where it goes */
+
+          if (gpwgts[other]+vwgt[i] > badmaxpwgt[other]) {
+            /* printf("Skip because of weight! %d\n", vwgt[i]-rinfo[i].edegrees[(c+1)%2]); */
+            continue;   /* We cannot move it there because it gets too heavy */
+          }
+
+          /* Update where, weight, and ID/ED information of the vertex you moved */
+          where[i] = PackWeightWhereInfo(vwgt[i], other);
+
+          /* Remove this vertex from the sepind. Note the trick for looking at the sepind[ii] again */
+          sepind[ii--] = sepind[--nsep]; 
+
+          /* myprintf(ctrl, "Vertex %d [%d %d] is moving to %d from %d [%d]\n", i+firstvtx, vwgt[i], rinfo[i].edegrees[(c+1)%2], other, from, SelectWhere(where[i])); */
+
+          lpwgts[from] -= vwgt[i];
+          lpwgts[2*nparts-1] -= vwgt[i];
+          lpwgts[other] += vwgt[i];
+          gpwgts[other] += vwgt[i];
+
+          /* 
+           * Put the vertices adjacent to i that belong to either the separator or
+           * the (c+1)%2 partition into the update array 
+           */
+          for (j=xadj[i]; j<xadj[i+1]; j++) {
+            k = ladjncy[j];
+            if (htable[k] == 0 && SelectWhere(where[k]) != other) {
+              htable[k] = 1;
+              if (k<nvtxs)
+                update[nlupd++] = k;
+              else
+                supdate[nsupd++] = k;
+            }
+          }
+          nmoves++;
+          if (graph->pexadj[i+1]-graph->pexadj[i] > 0)
+            changed[nchanged++] = i;
+        }
+      }
+
+      /* myprintf(ctrl, "nmoves: %d, nlupd: %d, nsupd: %d\n", nmoves, nlupd, nsupd); */
+
+      /* Tell everybody interested what the new where[] info is for the interface vertices */
+      CommChangedInterfaceData(ctrl, graph, nchanged, changed, where, swchanges, rwchanges, wspace->pv4); 
+
+
+      IFSET(ctrl->dbglvl, DBG_RMOVEINFO, rprintf(ctrl, "\t[%d %d], [%d %d %d]\n", 
+                pass, c, GlobalSESum(ctrl, nmoves), GlobalSESum(ctrl, nsupd), GlobalSESum(ctrl, nlupd)));
+
+
+      /*-------------------------------------------------------------
+      / Time to communicate with processors to send the vertices
+      / whose degrees need to be update.
+      /-------------------------------------------------------------*/
+      /* Issue the receives first */
+      for (i=0; i<nnbrs; i++) {
+        MPI_Irecv((void *)(rupdate+sendptr[i]), sendptr[i+1]-sendptr[i], IDX_DATATYPE,
+                  peind[i], 1, ctrl->comm, ctrl->rreq+i);
+      }
+
+      /* Issue the sends next. This needs some preporcessing */
+      for (i=0; i<nsupd; i++) {
+        htable[supdate[i]] = 0;
+        supdate[i] = graph->imap[supdate[i]];
+      }
+      iidxsort(nsupd, supdate);
+
+      for (j=i=0; i<nnbrs; i++) {
+        otherlastvtx = vtxdist[peind[i]+1];
+        for (k=j; k<nsupd && supdate[k] < otherlastvtx; k++); 
+        MPI_Isend((void *)(supdate+j), k-j, IDX_DATATYPE, peind[i], 1, ctrl->comm, ctrl->sreq+i);
+        j = k;
+      }
+
+      /* OK, now get into the loop waiting for the send/recv operations to finish */
+      MPI_Waitall(nnbrs, ctrl->rreq, ctrl->statuses);
+      for (i=0; i<nnbrs; i++) 
+        MPI_Get_count(ctrl->statuses+i, IDX_DATATYPE, nupds_pe+i);
+      MPI_Waitall(nnbrs, ctrl->sreq, ctrl->statuses);
+
+
+      /*-------------------------------------------------------------
+      / Place the received to-be updated vertices into update[] 
+      /-------------------------------------------------------------*/
+      for (i=0; i<nnbrs; i++) {
+        pe_updates = rupdate+sendptr[i];
+        for (j=0; j<nupds_pe[i]; j++) {
+          k = pe_updates[j];
+          if (htable[k-firstvtx] == 0) {
+            htable[k-firstvtx] = 1;
+            update[nlupd++] = k-firstvtx;
+          }
+        }
+      }
+
+
+      /*-------------------------------------------------------------
+      / Update the where information of the vertices that are pulled
+      / into the separator.
+      /-------------------------------------------------------------*/
+      nchanged = 0;
+      for (ii=0; ii<nlupd; ii++) {
+        i = update[ii];
+        me = SelectWhere(where[i]);
+        if (me < nparts && me%2 == (c+1)%2) { /* This vertex is pulled into the separator */
+          lpwgts[me] -= vwgt[i];
+          where[i] = PackWeightWhereInfo(vwgt[i], nparts+me-(me%2)); 
+          sepind[nsep++] = i;  /* Put the vertex into the sepind array */
+          if (graph->pexadj[i+1]-graph->pexadj[i] > 0)
+            changed[nchanged++] = i;
+
+          lpwgts[SelectWhere(where[i])] += vwgt[i];
+          lpwgts[2*nparts-1] += vwgt[i];
+          /* myprintf(ctrl, "Vertex %d moves into the separator from %d to %d\n", i+firstvtx, me, SelectWhere(where[i])); */
+        }
+      }
+
+      /* Tell everybody interested what the new where[] info is for the interface vertices */
+      CommChangedInterfaceData(ctrl, graph, nchanged, changed, where, swchanges, rwchanges, wspace->pv4); 
+
+
+      /*-------------------------------------------------------------
+      / Update the rinfo of the vertices in the update[] array
+      /-------------------------------------------------------------*/
+      for (ii=0; ii<nlupd; ii++) {
+        i = update[ii];
+        ASSERT(ctrl, htable[i] == 1);
+
+        htable[i] = 0;
+
+        me = SelectWhere(where[i]);
+        if (me >= nparts) {  /* If it is a separator vertex */
+          /* myprintf(ctrl, "Updating %d %d\n", i+firstvtx, me); */
+
+          myrinfo = rinfo+i;
+          myrinfo->edegrees[0] = myrinfo->edegrees[1] = 0;
+
+          for (j=xadj[i]; j<xadj[i+1]; j++) {
+            other = SelectWhere(where[ladjncy[j]]);
+            otherwgt = SelectWeight(where[ladjncy[j]]);
+            if (me != other)
+              myrinfo->edegrees[other%2] += otherwgt;
+          }
+        }
+      }
+
+      /* Finally, sum-up the partition weights */
+      MPI_Allreduce((void *)lpwgts, (void *)gpwgts, 2*nparts, IDX_DATATYPE, MPI_SUM, ctrl->comm);
+      graph->mincut = gpwgts[2*nparts-1];
+
+      IFSET(ctrl->dbglvl, DBG_REFINEINFO, PrintNodeBalanceInfo(ctrl, nparts, gpwgts, badminpwgt, badmaxpwgt, 0));
+    }
+
+    if (graph->mincut == oldcut)
+      break;
+  }
+
+  /* Go and clear-up the where array */
+  for (i=0; i<nvtxs+graph->nrecv; i++)
+    where[i] = SelectWhere(where[i]);
+
+  GKfree((void **)&update, (void **)&nupds_pe, (void **)&htable, (void **)&changed, LTERM);
+
+  IFSET(ctrl->dbglvl, DBG_TIME, stoptimer(ctrl->KWayTmr));
+}
+
+
+
+
+/*************************************************************************
+* This function prints balance information for the parallel k-section 
+* refinement algorithm
+**************************************************************************/
+void PrintNodeBalanceInfo(CtrlType *ctrl, int nparts, idxtype *gpwgts, idxtype *badminpwgt, idxtype *badmaxpwgt, int title)
+{
+  int i;
+
+  if (ctrl->mype == 0) {
+    if (title)
+      printf("K-way sep-refinement: TotalSep: %d, ", gpwgts[2*nparts-1]);
+    else
+      printf("\tTotalSep: %d, ", gpwgts[2*nparts-1]);
+
+    for (i=0; i<nparts; i+=2) 
+      printf(" [%5d %5d %5d %5d %5d]", gpwgts[i], gpwgts[i+1], gpwgts[nparts+i], badminpwgt[i], badmaxpwgt[i]); 
+    printf("\n");
+  }
+  MPI_Barrier(ctrl->comm);
+}
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/ometis.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/ometis.c
new file mode 100644
index 0000000..1a461f1
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/ometis.c
@@ -0,0 +1,188 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * ometis.c
+ *
+ * This is the entry point of parallel ordering
+ *
+ * Started 10/19/96
+ * George
+ *
+ * $Id: ometis.c,v 1.4 2003/07/25 04:01:04 karypis Exp $
+ *
+ */
+
+#include <parmetislib.h>
+
+
+
+
+/***********************************************************************************
+* This function is the entry point of the parallel ordering algorithm.
+* This function assumes that the graph is already nice partitioned among the 
+* processors and then proceeds to perform recursive bisection.
+************************************************************************************/
+void ParMETIS_V3_NodeND(idxtype *vtxdist, idxtype *xadj, idxtype *adjncy, int *numflag,
+              int *options, idxtype *order, idxtype *sizes, MPI_Comm *comm)
+{
+  int i, j;
+  int ltvwgts[MAXNCON];
+  int nparts, npes, mype, wgtflag = 0, seed = GLOBAL_SEED;
+  CtrlType ctrl;
+  WorkSpaceType wspace;
+  GraphType *graph, *mgraph;
+  idxtype *morder;
+  int minnvtxs;
+
+  MPI_Comm_size(*comm, &npes);
+  MPI_Comm_rank(*comm, &mype);
+  nparts = npes;
+
+  if (!ispow2(npes)) {
+    if (mype == 0)
+      printf("Error: The number of processors must be a power of 2!\n");
+    return;
+  }
+
+  if (vtxdist[npes] < (int)((float)(npes*npes)*1.2)) {
+    if (mype == 0)
+      printf("Error: Too many processors for this many vertices.\n");
+    return;
+  }
+
+  minnvtxs = vtxdist[1]-vtxdist[0];
+  for (i=0; i<npes; i++)
+    minnvtxs = (minnvtxs < vtxdist[i+1]-vtxdist[i]) ? minnvtxs : vtxdist[i+1]-vtxdist[i];
+
+  if (minnvtxs < (int)((float)npes*1.1)) {
+    if (mype == 0)
+      printf("Error: vertices are not distributed equally.\n");
+    return;
+  }
+ 
+
+  if (*numflag == 1) 
+    ChangeNumbering(vtxdist, xadj, adjncy, order, npes, mype, 1);
+
+  SetUpCtrl(&ctrl, nparts, options[PMV3_OPTION_DBGLVL], *comm);
+  ctrl.CoarsenTo = amin(vtxdist[npes]+1, 25*npes);
+
+  ctrl.CoarsenTo = amin(vtxdist[npes]+1, 25*amax(npes, nparts));
+  ctrl.seed = mype;
+  ctrl.sync = seed;
+  ctrl.partType = STATIC_PARTITION;
+  ctrl.ps_relation = -1;
+  ctrl.tpwgts = fsmalloc(nparts, 1.0/(float)(nparts), "tpwgts");
+  ctrl.ubvec[0] = 1.03;
+
+  graph = Moc_SetUpGraph(&ctrl, 1, vtxdist, xadj, NULL, adjncy, NULL, &wgtflag);
+
+  PreAllocateMemory(&ctrl, graph, &wspace);
+
+  /*=======================================================
+   * Compute the initial k-way partitioning 
+   =======================================================*/
+  IFSET(ctrl.dbglvl, DBG_TIME, InitTimers(&ctrl));
+  IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
+  IFSET(ctrl.dbglvl, DBG_TIME, starttimer(ctrl.TotalTmr));
+
+  Moc_Global_Partition(&ctrl, graph, &wspace);
+
+  IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
+  IFSET(ctrl.dbglvl, DBG_TIME, stoptimer(ctrl.TotalTmr));
+  IFSET(ctrl.dbglvl, DBG_TIME, PrintTimingInfo(&ctrl));
+
+  /*=======================================================
+   * Move the graph according to the partitioning
+   =======================================================*/
+  IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
+  IFSET(ctrl.dbglvl, DBG_TIME, starttimer(ctrl.MoveTmr));
+
+  MALLOC_CHECK(NULL);
+  graph->ncon = 1;
+  mgraph = Moc_MoveGraph(&ctrl, graph, &wspace);
+  MALLOC_CHECK(NULL);
+
+  IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
+  IFSET(ctrl.dbglvl, DBG_TIME, stoptimer(ctrl.MoveTmr));
+
+  /*=======================================================
+   * Now compute an ordering of the moved graph
+   =======================================================*/
+  IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
+  IFSET(ctrl.dbglvl, DBG_TIME, starttimer(ctrl.TotalTmr));
+
+  FreeWSpace(&wspace);
+  PreAllocateMemory(&ctrl, mgraph, &wspace);
+
+  ctrl.ipart = ISEP_NODE;
+  ctrl.CoarsenTo = amin(vtxdist[npes]+1, amax(20*npes, 1000));
+
+  /* compute tvwgts */
+  for (j=0; j<mgraph->ncon; j++)
+    ltvwgts[j] = 0;
+
+  for (i=0; i<mgraph->nvtxs; i++)
+    for (j=0; j<mgraph->ncon; j++)
+      ltvwgts[j] += mgraph->vwgt[i*mgraph->ncon+j];
+
+  for (j=0; j<mgraph->ncon; j++)
+    ctrl.tvwgts[j] = GlobalSESum(&ctrl, ltvwgts[j]);
+
+  mgraph->nvwgt = fmalloc(mgraph->nvtxs*mgraph->ncon, "mgraph->nvwgt");
+  for (i=0; i<mgraph->nvtxs; i++)
+    for (j=0; j<mgraph->ncon; j++)
+      mgraph->nvwgt[i*mgraph->ncon+j] = (float)(mgraph->vwgt[i*mgraph->ncon+j]) / (float)(ctrl.tvwgts[j]);
+
+
+  morder = idxmalloc(mgraph->nvtxs, "PAROMETIS: morder");
+  MultilevelOrder(&ctrl, mgraph, morder, sizes, &wspace);
+
+  MALLOC_CHECK(NULL);
+
+  /* Invert the ordering back to the original graph */
+  ProjectInfoBack(&ctrl, graph, order, morder, &wspace);
+
+  MALLOC_CHECK(NULL);
+
+  IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
+  IFSET(ctrl.dbglvl, DBG_TIME, stoptimer(ctrl.TotalTmr));
+  IFSET(ctrl.dbglvl, DBG_TIME, PrintTimingInfo(&ctrl));
+  IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
+
+  free(ctrl.tpwgts);
+  free(morder);
+  FreeGraph(mgraph);
+  FreeInitialGraphAndRemap(graph, 0);
+  FreeWSpace(&wspace);
+  FreeCtrl(&ctrl);
+
+  if (*numflag == 1) 
+    ChangeNumbering(vtxdist, xadj, adjncy, order, npes, mype, 0);
+
+  MALLOC_CHECK(NULL);
+}
+
+
+/***********************************************************************************
+* This function is the entry point of the parallel ordering algorithm.
+* This function assumes that the graph is already nice partitioned among the 
+* processors and then proceeds to perform recursive bisection.
+************************************************************************************/
+void PAROMETIS(idxtype *vtxdist, idxtype *xadj, idxtype *vwgt, idxtype *adjncy, idxtype *adjwgt, 
+                idxtype *order, idxtype *sizes, int *options, MPI_Comm comm)
+{
+  int numflag, newoptions[5];
+
+  newoptions[0] = 1;
+  newoptions[PMV3_OPTION_DBGLVL] = options[4];
+  newoptions[PMV3_OPTION_SEED] = GLOBAL_SEED;
+
+  numflag = options[3];
+
+  ParMETIS_V3_NodeND(vtxdist, xadj, adjncy, &numflag, newoptions, order, sizes, &comm);
+
+  options[0] = -1;
+
+}
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/order.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/order.c
new file mode 100644
index 0000000..a73c87b
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/order.c
@@ -0,0 +1,348 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * order.c
+ *
+ * This file contains the driving routines for the multilevel ordering algorithm
+ *
+ * Started 5/3/97
+ * George
+ *
+ * $Id: order.c,v 1.2 2003/07/21 17:18:50 karypis Exp $
+ *
+ */
+
+#define DEBUG_ORDER_
+
+#include <parmetislib.h>
+
+/*************************************************************************
+* This is the top level ordering routine
+**************************************************************************/
+void MultilevelOrder(CtrlType *ctrl, GraphType *graph, idxtype *order, idxtype *sizes, WorkSpaceType *wspace)
+{
+  int i, nparts, nvtxs, npes;
+  idxtype *perm, *lastnode, *morder, *porder;
+  GraphType *mgraph;
+
+  npes = ctrl->npes;
+  nvtxs = graph->nvtxs;
+
+  perm = idxmalloc(nvtxs, "MultilevelOrder: perm");
+  lastnode = idxsmalloc(4*npes, -1, "MultilevelOrder: lastnode");
+
+  for (i=0; i<nvtxs; i++) 
+    perm[i] = i;
+  lastnode[2] = graph->gnvtxs;
+
+  idxset(nvtxs, -1, order);
+
+  sizes[0] = 2*npes-1;
+
+  graph->where = idxsmalloc(nvtxs, 0, "MultilevelOrder: graph->where");
+
+  for (nparts=2; nparts<=ctrl->npes; nparts*=2) {
+    ctrl->nparts = nparts;
+
+    Order_Partition(ctrl, graph, wspace);
+
+    LabelSeparators(ctrl, graph, lastnode, perm, order, sizes, wspace);
+
+    CompactGraph(ctrl, graph, perm, wspace);
+
+    if (ctrl->CoarsenTo < 100*nparts) {
+      ctrl->CoarsenTo = 1.5*ctrl->CoarsenTo;
+    }
+    ctrl->CoarsenTo = amin(ctrl->CoarsenTo, graph->gnvtxs-1);
+  }
+
+
+  /*-----------------------------------------------------------------
+   / Move the graph so that each processor gets its partition 
+   -----------------------------------------------------------------*/
+  IFSET(ctrl->dbglvl, DBG_TIME, MPI_Barrier(ctrl->comm));
+  IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->MoveTmr));
+
+  SetUp(ctrl, graph, wspace);
+  graph->ncon = 1; /*needed for Moc_MoveGraph */
+  mgraph = Moc_MoveGraph(ctrl, graph, wspace);
+
+  /* Fill in the sizes[] array for the local part. Just the vtxdist of the mgraph */
+  for (i=0; i<npes; i++)
+    sizes[i] = mgraph->vtxdist[i+1]-mgraph->vtxdist[i];
+
+  porder = idxmalloc(graph->nvtxs, "MultilevelOrder: porder");
+  morder = idxmalloc(mgraph->nvtxs, "MultilevelOrder: morder");
+
+  IFSET(ctrl->dbglvl, DBG_TIME, MPI_Barrier(ctrl->comm));
+  IFSET(ctrl->dbglvl, DBG_TIME, stoptimer(ctrl->MoveTmr));
+
+  /* Find the local ordering */
+  LocalNDOrder(ctrl, mgraph, morder, lastnode[2*(ctrl->npes+ctrl->mype)]-mgraph->nvtxs, wspace);
+
+  /* Project the ordering back to the before-move graph */
+  ProjectInfoBack(ctrl, graph, porder, morder, wspace);
+
+  /* Copy the ordering from porder to order using perm */
+  for (i=0; i<graph->nvtxs; i++) {
+    ASSERT(ctrl, order[perm[i]] == -1);
+    order[perm[i]] = porder[i];
+  }
+
+  FreeGraph(mgraph);
+  GKfree((void **)&perm, (void **)&lastnode, (void **)&porder, (void **)&morder, LTERM);
+
+  /* PrintVector(ctrl, 2*npes-1, 0, sizes, "SIZES"); */
+}
+
+
+/*************************************************************************
+* This function is used to assign labels to the nodes in the separators
+* It uses the appropriate entry in the lastnode array to select label
+* boundaries and adjusts it for the next level
+**************************************************************************/
+void LabelSeparators(CtrlType *ctrl, GraphType *graph, idxtype *lastnode, idxtype *perm, idxtype *order, idxtype *sizes, WorkSpaceType *wspace)
+{
+  int i, nvtxs, nparts, sid;
+  idxtype *where, *lpwgts, *gpwgts, *sizescan;
+
+  nparts = ctrl->nparts;
+
+  nvtxs = graph->nvtxs;
+  where = graph->where;
+  lpwgts = graph->lpwgts;
+  gpwgts = graph->gpwgts;
+
+  /* Compute the local size of the separator. This is required in case the 
+   * graph has vertex weights */
+  idxset(2*nparts, 0, lpwgts);
+  for (i=0; i<nvtxs; i++) 
+    lpwgts[where[i]]++;
+
+  sizescan = idxmalloc(2*nparts, "LabelSeparators: sizescan");
+
+  /* Perform a Prefix scan of the separator sizes to determine the boundaries */
+  MPI_Scan((void *)lpwgts, (void *)sizescan, 2*nparts, IDX_DATATYPE, MPI_SUM, ctrl->comm);
+  MPI_Allreduce((void *)lpwgts, (void *)gpwgts, 2*nparts, IDX_DATATYPE, MPI_SUM, ctrl->comm);
+
+#ifdef DEBUG_ORDER
+  PrintVector(ctrl, 2*nparts, 0, lpwgts, "Lpwgts");
+  PrintVector(ctrl, 2*nparts, 0, sizescan, "SizeScan");
+  PrintVector(ctrl, 2*nparts, 0, lastnode, "LastNode");
+#endif
+
+  /* Fillin the sizes[] array */
+  for (i=nparts-2; i>=0; i-=2) 
+    sizes[--sizes[0]] = gpwgts[nparts+i];
+
+  if (ctrl->dbglvl&DBG_INFO) {
+    if (ctrl->mype == 0) {
+      printf("SepSizes: ");
+      for (i=0; i<nparts; i+=2)
+        printf(" %d [%d %d]", gpwgts[nparts+i], gpwgts[i], gpwgts[i+1]);
+      printf("\n");
+    }
+    MPI_Barrier(ctrl->comm);
+  }
+
+  for (i=0; i<2*nparts; i++)
+    sizescan[i] -= lpwgts[i];
+
+  for (i=0; i<nvtxs; i++) {
+    if (where[i] >= nparts) {
+      sid = where[i];
+      sizescan[sid]++;
+      ASSERT(ctrl, order[perm[i]] == -1);
+      order[perm[i]] = lastnode[sid] - sizescan[sid];
+      /* myprintf(ctrl, "order[%d] = %d, %d\n", perm[i], order[perm[i]], sid); */
+    }
+  }
+
+  /* Update lastnode array */
+  idxcopy(2*nparts, lastnode, sizescan);
+  for (i=0; i<nparts; i+=2) {
+    lastnode[2*nparts+2*i] = sizescan[nparts+i]-gpwgts[nparts+i]-gpwgts[i+1];
+    lastnode[2*nparts+2*(i+1)] = sizescan[nparts+i]-gpwgts[nparts+i];
+  }
+
+  free(sizescan);
+
+}
+
+
+
+
+/*************************************************************************
+* This function compacts a graph by removing the vertex separator
+**************************************************************************/
+void CompactGraph(CtrlType *ctrl, GraphType *graph, idxtype *perm, WorkSpaceType *wspace)
+{
+  int i, j, l, nvtxs, cnvtxs, cfirstvtx, nparts, npes; 
+  idxtype *xadj, *ladjncy, *adjwgt, *vtxdist, *where;
+  idxtype *cmap, *cvtxdist, *newwhere;
+
+  nparts = ctrl->nparts;
+  npes = ctrl->npes;
+
+  nvtxs = graph->nvtxs;
+  xadj = graph->xadj;
+  ladjncy = graph->adjncy;
+  adjwgt = graph->adjwgt;
+  where = graph->where;
+
+  if (graph->cmap == NULL)
+    graph->cmap = idxmalloc(nvtxs+graph->nrecv, "CompactGraph: cmap");
+  cmap = graph->cmap;
+
+  vtxdist = graph->vtxdist;
+
+  /*************************************************************
+  * Construct the cvtxdist of the contracted graph. Uses the fact
+  * that lpwgts stores the local non separator vertices.
+  **************************************************************/
+  cvtxdist = wspace->pv1;
+  cnvtxs = cvtxdist[npes] = idxsum(nparts, graph->lpwgts);
+
+  MPI_Allgather((void *)(cvtxdist+npes), 1, IDX_DATATYPE, (void *)cvtxdist, 1, IDX_DATATYPE, ctrl->comm);
+  MAKECSR(i, npes, cvtxdist);
+
+#ifdef DEBUG_ORDER
+  PrintVector(ctrl, npes+1, 0, cvtxdist, "cvtxdist");
+#endif
+
+
+  /*************************************************************
+  * Construct the cmap vector 
+  **************************************************************/
+  cfirstvtx = cvtxdist[ctrl->mype];
+
+  /* Create the cmap of what you know so far locally */
+  for (cnvtxs=0, i=0; i<nvtxs; i++) {
+    if (where[i] < nparts) {
+      perm[cnvtxs] = perm[i];
+      cmap[i] = cfirstvtx + cnvtxs++;
+    }
+  }
+
+  CommInterfaceData(ctrl, graph, cmap, wspace->indices, cmap+nvtxs);
+
+
+  /*************************************************************
+  * Finally, compact the graph
+  **************************************************************/
+  newwhere = idxmalloc(cnvtxs, "CompactGraph: newwhere");
+  cnvtxs = l = 0;
+  for (i=0; i<nvtxs; i++) {
+    if (where[i] < nparts) {
+      for (j=xadj[i]; j<xadj[i+1]; j++) {
+        if (where[i] == where[ladjncy[j]]) {
+          ladjncy[l] = cmap[ladjncy[j]];
+          adjwgt[l++] = adjwgt[j];
+        }
+#ifdef DEBUG_ORDER
+        else if (where[ladjncy[j]] < nparts)
+          printf("It seems that the separation has failed: %d %d\n", where[i], where[ladjncy[j]]);
+#endif
+      }
+
+      xadj[cnvtxs] = l;
+      graph->vwgt[cnvtxs] = graph->vwgt[i];
+      newwhere[cnvtxs] = where[i];
+      cnvtxs++;
+    }
+  }
+  for (i=cnvtxs; i>0; i--)
+    xadj[i] = xadj[i-1];
+  xadj[0] = 0;
+
+  GKfree((void **)&graph->match, (void **)&graph->cmap, (void **)&graph->lperm, (void **)&graph->where, (void **)&graph->label, (void **)&graph->rinfo,
+         (void **)&graph->nrinfo, (void **)&graph->lpwgts, (void **)&graph->gpwgts, (void **)&graph->sepind, (void **)&graph->peind,
+         (void **)&graph->sendptr, (void **)&graph->sendind, (void **)&graph->recvptr, (void **)&graph->recvind, 
+         (void **)&graph->imap, (void **)&graph->rlens, (void **)&graph->slens, (void **)&graph->rcand, (void **)&graph->pexadj, 
+         (void **)&graph->peadjncy, (void **)&graph->peadjloc, LTERM);
+ 
+  graph->nvtxs = cnvtxs;
+  graph->nedges = l;
+  graph->gnvtxs = cvtxdist[npes];
+  idxcopy(npes+1, cvtxdist, graph->vtxdist);
+  graph->where = newwhere;
+
+}
+
+
+/*************************************************************************
+* This function orders the locally stored graph using MMD. 
+* The vertices will be ordered from firstnode onwards.
+**************************************************************************/
+void LocalNDOrder(CtrlType *ctrl, GraphType *graph, idxtype *order, int firstnode, WorkSpaceType *wspace)
+{
+  int i, j, nvtxs, firstvtx, lastvtx;
+  idxtype *xadj, *adjncy;
+  idxtype *perm, *iperm;
+  int numflag=0, options[10];
+
+  nvtxs = graph->nvtxs;
+  xadj = graph->xadj;
+  adjncy = graph->adjncy;
+
+  firstvtx = graph->vtxdist[ctrl->mype];
+  lastvtx = graph->vtxdist[ctrl->mype+1];
+
+  /* Relabel the vertices so that they are in local index space */
+  for (i=0; i<nvtxs; i++) {
+    for (j=xadj[i]; j<xadj[i+1]; j++) {
+      ASSERT(ctrl, adjncy[j]>=firstvtx && adjncy[j]<lastvtx);
+      adjncy[j] -= firstvtx;
+    }
+  }
+
+  ASSERT(ctrl, 2*(nvtxs+5) < wspace->maxcore);
+
+  perm = wspace->core;
+  iperm = perm + nvtxs + 5;
+
+  options[0] = 0;
+  METIS_NodeND(&nvtxs, xadj, adjncy, &numflag, options, perm, iperm);
+
+  for (i=0; i<nvtxs; i++) {
+    ASSERT(ctrl, iperm[i]>=0 && iperm[i]<nvtxs);
+    order[i] = firstnode+iperm[i];
+  }
+
+}
+
+/*************************************************************************
+* This function is the driver for the partition refinement mode of ParMETIS
+**************************************************************************/
+void Order_Partition(CtrlType *ctrl, GraphType *graph, WorkSpaceType *wspace)
+{
+
+  SetUp(ctrl, graph, wspace);
+  graph->ncon = 1;
+
+  IFSET(ctrl->dbglvl, DBG_PROGRESS, rprintf(ctrl, "[%6d %8d %5d %5d][%d][%d]\n",
+        graph->gnvtxs, GlobalSESum(ctrl, graph->nedges), GlobalSEMin(ctrl, graph->nvtxs),
+        GlobalSEMax(ctrl, graph->nvtxs), ctrl->CoarsenTo,
+        GlobalSEMax(ctrl, graph->vwgt[idxamax(graph->nvtxs, graph->vwgt)])));
+
+  if (graph->gnvtxs < 1.3*ctrl->CoarsenTo || (graph->finer != NULL && graph->gnvtxs > graph->finer->gnvtxs*COARSEN_FRACTION)) {
+    /* Compute the initial npart-way multisection */
+    InitMultisection(ctrl, graph, wspace);
+
+    if (graph->finer == NULL) { /* Do that only of no-coarsening took place */
+      ComputeNodePartitionParams(ctrl, graph, wspace);
+      KWayNodeRefine(ctrl, graph, wspace, 2*NGR_PASSES, ORDER_UNBALANCE_FRACTION);
+    }
+  }
+  else { /* Coarsen it and the partition it */
+    Mc_LocalMatch_HEM(ctrl, graph, wspace);
+
+    Order_Partition(ctrl, graph->coarser, wspace);
+
+    Moc_ProjectPartition(ctrl, graph, wspace);
+    ComputeNodePartitionParams(ctrl, graph, wspace);
+    KWayNodeRefine(ctrl, graph, wspace, 2*NGR_PASSES, ORDER_UNBALANCE_FRACTION);
+  }
+}
+
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/parmetislib.h b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/parmetislib.h
new file mode 100644
index 0000000..36c1041
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/parmetislib.h
@@ -0,0 +1,31 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * par_metis.h
+ *
+ * This file includes all necessary header files
+ *
+ * Started 8/27/94
+ * George
+ *
+ * $Id: parmetislib.h,v 1.2 2003/07/21 17:50:22 karypis Exp $
+ */
+
+/*
+#define DEBUG			1
+#define DMALLOC			1
+*/
+
+#include <stdheaders.h>
+#include "../parmetis.h"
+
+#ifdef DMALLOC
+#include <dmalloc.h>
+#endif
+
+#include <rename.h>
+#include <defs.h>
+#include <struct.h>
+#include <macros.h>
+#include <proto.h>
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/proto.h b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/proto.h
new file mode 100644
index 0000000..bbab2e5
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/proto.h
@@ -0,0 +1,352 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * proto.h
+ *
+ * This file contains header files
+ *
+ * Started 10/19/95
+ * George
+ *
+ * $Id: proto.h,v 1.11 2003/07/25 13:52:03 karypis Exp $
+ *
+ */
+
+/* kmetis.c */
+void Moc_Global_Partition(CtrlType *, GraphType *, WorkSpaceType *);
+
+/* mmetis.c */
+
+/* gkmetis.c */
+
+/* match.c */
+void Moc_GlobalMatch_Balance(CtrlType *, GraphType *, WorkSpaceType *);
+
+/* coarsen.c */
+void Moc_Global_CreateCoarseGraph(CtrlType *, GraphType *, WorkSpaceType *, int);
+
+/* initpart.c */
+void Moc_InitPartition_RB(CtrlType *, GraphType *, WorkSpaceType *);
+void Moc_KeepPart(GraphType *, WorkSpaceType *, idxtype *, int);
+
+/* kwayrefine.c */
+void Moc_ProjectPartition(CtrlType *, GraphType *, WorkSpaceType *);
+void Moc_ComputePartitionParams(CtrlType *, GraphType *, WorkSpaceType *);
+
+/* kwayfm.c */
+void Moc_KWayFM(CtrlType *, GraphType *, WorkSpaceType *, int);
+
+/* kwaybalance.c */
+void Moc_KWayBalance(CtrlType *, GraphType *, WorkSpaceType *, int);
+
+/* remap.c */
+void ParallelReMapGraph(CtrlType *, GraphType *, WorkSpaceType *);
+void ParallelTotalVReMap(CtrlType *, idxtype *, idxtype *, WorkSpaceType *, int, int);
+int SimilarTpwgts(float *, int, int, int);
+
+/* move.c */
+GraphType *Moc_MoveGraph(CtrlType *, GraphType *, WorkSpaceType *);
+/* move.c */
+void CheckMGraph(CtrlType *, GraphType *); 
+void ProjectInfoBack(CtrlType *, GraphType *, idxtype *, idxtype *, WorkSpaceType *);
+void FindVtxPerm(CtrlType *, GraphType *, idxtype *, WorkSpaceType *);
+
+/* memory.c */
+void PreAllocateMemory(CtrlType *, GraphType *, WorkSpaceType *);
+void FreeWSpace(WorkSpaceType *);
+void FreeCtrl(CtrlType *);
+GraphType *CreateGraph(void);
+void InitGraph(GraphType *);
+void FreeGraph(GraphType *);
+void FreeInitialGraphAndRemap(GraphType *, int);
+
+
+/* ametis.c */
+void Adaptive_Partition(CtrlType *, GraphType *, WorkSpaceType *);
+
+/* rmetis.c */
+
+
+/* lmatch.c */
+void Mc_LocalMatch_HEM(CtrlType *, GraphType *, WorkSpaceType *);
+void Mc_Local_CreateCoarseGraph(CtrlType *, GraphType *, WorkSpaceType *, int);
+
+/* wave.c */
+float WavefrontDiffusion(CtrlType *, GraphType *, idxtype *);
+
+/* balancemylink.c */
+int BalanceMyLink(CtrlType *, GraphType *, idxtype *, int, int, float *, float, float *, float *, float);
+
+/* redomylink.c */
+void RedoMyLink(CtrlType *, GraphType *, idxtype *, int, int, float *, float *, float *);
+
+/* initbalance.c */
+void Balance_Partition(CtrlType *, GraphType *, WorkSpaceType *);
+GraphType *Moc_AssembleAdaptiveGraph(CtrlType *, GraphType *, WorkSpaceType *);
+
+/* mdiffusion.c */
+int Moc_Diffusion(CtrlType *, GraphType *, idxtype *, idxtype *, idxtype *, WorkSpaceType *, int);
+GraphType *ExtractGraph(CtrlType *, GraphType *, idxtype *, idxtype *, idxtype *);
+
+/* diffutil.c */
+void SetUpConnectGraph(GraphType *, MatrixType *, idxtype *);
+void Mc_ComputeMoveStatistics(CtrlType *, GraphType *, int *, int *, int *);
+ int Mc_ComputeSerialTotalV(GraphType *, idxtype *);
+void ComputeLoad(GraphType *, int, float *, float *, int);
+void ConjGrad2(MatrixType *, float *, float *, float, float *);
+void mvMult2(MatrixType *, float *, float *);
+void ComputeTransferVector(int, MatrixType *, float *, float *, int);
+int ComputeSerialEdgeCut(GraphType *);
+int ComputeSerialTotalV(GraphType *, idxtype *);
+
+/* akwayfm.c */
+void Moc_KWayAdaptiveRefine(CtrlType *, GraphType *, WorkSpaceType *, int);
+
+/* selectq.c */
+void Moc_DynamicSelectQueue(int, int, int, int, idxtype *, float *, int *, int *, int, float, float);
+int Moc_HashVwgts(int, float *);
+int Moc_HashVRank(int, int *);
+
+
+/* csrmatch.c */
+void CSR_Match_SHEM(MatrixType *, idxtype *, idxtype *, idxtype *, int);
+
+/* serial.c */
+void Moc_SerialKWayAdaptRefine(GraphType *, int, idxtype *, float *, int);
+void Moc_ComputeSerialPartitionParams(GraphType *, int, EdgeType *);
+int AreAllHVwgtsBelow(int, float, float *, float, float *, float *);
+void ComputeHKWayLoadImbalance(int, int, float *, float *);
+void SerialRemap(GraphType *, int, idxtype *, idxtype *, idxtype *, float *);
+int SSMIncKeyCmp(const void *, const void *);
+void Moc_Serial_FM_2WayRefine(GraphType *, float *, int);
+void Serial_SelectQueue(int, float *, float *, int *, int *, FPQueueType [MAXNCON][2]);
+int Serial_BetterBalance(int, float *, float *, float *);
+float Serial_Compute2WayHLoadImbalance(int, float *, float *);
+void Moc_Serial_Balance2Way(GraphType *, float *, float);
+void Moc_Serial_Init2WayBalance(GraphType *, float *);
+int Serial_SelectQueueOneWay(int, float *, float *, int, FPQueueType [MAXNCON][2]);
+void Moc_Serial_Compute2WayPartitionParams(GraphType *);
+int Serial_AreAnyVwgtsBelow(int, float, float *, float, float *, float *);
+
+/* weird.c */
+void PartitionSmallGraph(CtrlType *, GraphType *, WorkSpaceType *);
+void CheckInputs(int partType, int npes, int dbglvl, int *wgtflag, int *iwgtflag,
+                 int *numflag, int *inumflag, int *ncon, int *incon, int *nparts, 
+		 int *inparts, float *tpwgts, float **itpwgts, float *ubvec, 
+		 float *iubvec, float *ipc2redist, float *iipc2redist, int *options, 
+		 int *ioptions, idxtype *part, MPI_Comm *comm);
+
+/* mesh.c */
+
+/* ometis.c */
+
+/* pspases.c */
+GraphType *AssembleEntireGraph(CtrlType *, idxtype *, idxtype *, idxtype *);
+
+/* node_refine.c */
+void ComputeNodePartitionParams0(CtrlType *, GraphType *, WorkSpaceType *);
+void ComputeNodePartitionParams(CtrlType *, GraphType *, WorkSpaceType *);
+void KWayNodeRefine0(CtrlType *, GraphType *, WorkSpaceType *, int, float);
+void KWayNodeRefine(CtrlType *, GraphType *, WorkSpaceType *, int, float);
+void KWayNodeRefine2(CtrlType *, GraphType *, WorkSpaceType *, int, float);
+void PrintNodeBalanceInfo(CtrlType *, int, idxtype *, idxtype *, idxtype *, int);
+
+/* initmsection.c */
+void InitMultisection(CtrlType *, GraphType *, WorkSpaceType *);
+GraphType *AssembleMultisectedGraph(CtrlType *, GraphType *, WorkSpaceType *);
+
+/* order.c */
+void MultilevelOrder(CtrlType *, GraphType *, idxtype *, idxtype *, WorkSpaceType *);
+void LabelSeparators(CtrlType *, GraphType *, idxtype *, idxtype *, idxtype *, idxtype *, WorkSpaceType *);
+void CompactGraph(CtrlType *, GraphType *, idxtype *, WorkSpaceType *);
+void LocalOrder(CtrlType *, GraphType *, idxtype *, int, WorkSpaceType *);
+void LocalNDOrder(CtrlType *, GraphType *, idxtype *, int, WorkSpaceType *);
+void Order_Partition(CtrlType *, GraphType *, WorkSpaceType *);
+
+/* xyzpart.c */
+void Coordinate_Partition(CtrlType *, GraphType *, int, float *, int, WorkSpaceType *);
+void PartSort(CtrlType *, GraphType *, KeyValueType *, WorkSpaceType *);
+
+
+/* fpqueue.c */
+void FPQueueInit(FPQueueType *, int);
+void FPQueueReset(FPQueueType *);
+void FPQueueFree(FPQueueType *);
+int FPQueueGetSize(FPQueueType *);
+int FPQueueInsert(FPQueueType *, int, float);
+int FPQueueDelete(FPQueueType *, int);
+int FPQueueUpdate(FPQueueType *, int, float, float);
+void FPQueueUpdateUp(FPQueueType *, int, float, float);
+int FPQueueGetMax(FPQueueType *);
+int FPQueueSeeMaxVtx(FPQueueType *);
+float FPQueueSeeMaxGain(FPQueueType *);
+float FPQueueGetKey(FPQueueType *);
+int FPQueueGetQSize(FPQueueType *);
+int CheckHeapFloat(FPQueueType *);
+
+/* stat.c */
+void Moc_ComputeSerialBalance(CtrlType *, GraphType *, idxtype *, float *);
+void Moc_ComputeParallelBalance(CtrlType *, GraphType *, idxtype *, float *);
+void Moc_PrintThrottleMatrix(CtrlType *, GraphType *, float *);
+void Moc_ComputeRefineStats(CtrlType *, GraphType *, float *);
+
+/* debug.c */
+void PrintVector(CtrlType *, int, int, idxtype *, char *);
+void PrintVector2(CtrlType *, int, int, idxtype *, char *);
+void PrintPairs(CtrlType *, int, KeyValueType *, char *);
+void PrintGraph(CtrlType *, GraphType *);
+void PrintGraph2(CtrlType *, GraphType *);
+void PrintSetUpInfo(CtrlType *ctrl, GraphType *graph);
+void PrintTransferedGraphs(CtrlType *, int, idxtype *, idxtype *, idxtype *, idxtype *, idxtype *);
+void WriteMetisGraph(int, idxtype *, idxtype *, idxtype *, idxtype *);
+
+/* comm.c */
+void CommInterfaceData(CtrlType *, GraphType *, idxtype *, idxtype *, idxtype *);
+void CommChangedInterfaceData(CtrlType *, GraphType *, int, idxtype *, idxtype *, KeyValueType *, KeyValueType *, idxtype *);
+int GlobalSEMax(CtrlType *, int);
+double GlobalSEMaxDouble(CtrlType *, double);
+int GlobalSEMin(CtrlType *, int);
+int GlobalSESum(CtrlType *, int);
+float GlobalSEMaxFloat(CtrlType *, float);
+float GlobalSEMinFloat(CtrlType *, float);
+float GlobalSESumFloat(CtrlType *, float);
+
+/* util.c */
+void errexit(char *,...);
+void myprintf(CtrlType *, char *f_str,...);
+void rprintf(CtrlType *, char *f_str,...);
+#ifndef DMALLOC
+int *imalloc(int, char *);
+idxtype *idxmalloc(int, char *);
+float *fmalloc(int, char *);
+int *ismalloc(int, int, char *);
+idxtype *idxsmalloc(int, idxtype, char *);
+void *GKmalloc(int, char *);
+#endif
+/*void GKfree(void **,...); */
+int *iset(int n, int val, int *x);
+idxtype * idxset(int n, idxtype val, idxtype *x);
+int idxamax(int n, idxtype *x);
+int idxamin(int n, idxtype *x);
+int idxasum(int n, idxtype *x);
+float snorm2(int, float *);
+float sdot(int n, float *, float *);
+void saxpy(int, float, float *, float *);
+void ikeyvalsort_org(int, KeyValueType *);
+int IncKeyValueCmp(const void *, const void *);
+void dkeyvalsort(int, KeyValueType *);
+int DecKeyValueCmp(const void *, const void *);
+int BSearch(int, idxtype *, int);
+void RandomPermute(int, idxtype *, int);
+void FastRandomPermute(int, idxtype *, int);
+int ispow2(int);
+int log2Int(int);
+void BucketSortKeysDec(int, int, idxtype *, idxtype *);
+float *sset(int n, float val, float *x);
+int iamax(int, int *);
+int idxamax_strd(int, idxtype *, int);
+int idxamin_strd(int, idxtype *, int);
+int samax_strd(int, float *, int);
+int sfamax(int, float *);
+int samin_strd(int, float *, int);
+float idxavg(int, idxtype *);
+float savg(int, float *);
+int samax(int, float *);
+int sfavg(int n, float *x);
+int samax2(int, float *);
+int samin(int, float *);
+int idxsum(int, idxtype *);
+int idxsum_strd(int, idxtype *, int);
+void idxadd(int, idxtype *, idxtype *);
+float ssum(int, float *);
+float ssum_strd(int, float *, int);
+void sscale(int, float, float *);
+void saneg(int, float *);
+float BetterVBalance(int, float *, float *, float *);
+int IsHBalanceBetterTT(int, float *, float *, float *, float *);
+int IsHBalanceBetterFT(int, float *, float *, float *, float *);
+int myvalkeycompare(const void *, const void *);
+int imyvalkeycompare(const void *, const void *);
+float *fsmalloc(int, float, char *);
+void saxpy2(int, float, float *, int, float *, int);
+void GetThreeMax(int, float *, int *, int *, int *);
+
+/* qsort_special.c */
+void iidxsort(int, idxtype *);
+void iintsort(int, int *);
+void ikeysort(int, KeyValueType *);
+void ikeyvalsort(int, KeyValueType *);
+
+/* grsetup.c */
+GraphType *Moc_SetUpGraph(CtrlType *, int, idxtype *, idxtype *, idxtype *, idxtype *, idxtype *, int *);
+void SetUpCtrl(CtrlType *ctrl, int, int, MPI_Comm);
+void ChangeNumbering(idxtype *, idxtype *, idxtype *, idxtype *, int, int, int);
+void ChangeNumberingMesh(idxtype *, idxtype *, idxtype *, idxtype *, idxtype *, int, int, int, int);
+void ChangeNumberingMesh2(idxtype *elmdist, idxtype *eptr, idxtype *eind,
+                          idxtype *xadj, idxtype *adjncy, idxtype *part,
+			  int npes, int mype, int from);
+void GraphRandomPermute(GraphType *);
+void ComputeMoveStatistics(CtrlType *, GraphType *, int *, int *, int *);
+
+/* timer.c */
+void InitTimers(CtrlType *);
+void PrintTimingInfo(CtrlType *);
+void PrintTimer(CtrlType *, timer, char *);
+
+/* setup.c */
+void SetUp(CtrlType *, GraphType *, WorkSpaceType *);
+int Home_PE(int, int, idxtype *, int);
+
+
+/*********************/
+/* METIS subroutines */
+/*********************/
+void METIS_WPartGraphKway2(int *, idxtype *, idxtype *, idxtype *, idxtype *, int *, int *, int *, float *, int *, int *, idxtype *);
+void METIS_mCPartGraphRecursive2(int *, int *, idxtype *, idxtype *, idxtype *, idxtype *, int *, int *, int *, float *, int *, int *, idxtype *);
+int MCMlevelRecursiveBisection2(CtrlType *, GraphType *, int, float *, idxtype *, float, int); 
+void METIS_PartGraphKway(int *, idxtype *, idxtype *, idxtype *, idxtype *, int *, int *, int *, int *, int *, idxtype *);
+void METIS_mCPartGraphKway(int *, int *, idxtype *, idxtype *, idxtype *, idxtype *, int *, int *, int *, float *, int *, int *, idxtype *);
+void METIS_EdgeComputeSeparator(int *, idxtype *, idxtype *, idxtype *, idxtype *, int *, int *, idxtype *); 
+void METIS_NodeComputeSeparator(int *, idxtype *, idxtype *, idxtype *, idxtype *, int *, int *, idxtype *); 
+void METIS_NodeND(int *, idxtype *, idxtype *, int *, int *, idxtype *, idxtype *);
+void METIS_NodeNDP(int, idxtype *, idxtype *, int, int *, idxtype *, idxtype *, idxtype *);
+
+
+
+/***********************/
+/* TESTing subroutines */
+/***********************/
+
+/* pio.c */
+void ParallelReadGraph(GraphType *, char *, MPI_Comm);
+void Moc_ParallelWriteGraph(CtrlType *, GraphType *, char *, int, int);
+void ReadTestGraph(GraphType *, char *, MPI_Comm);
+float *ReadTestCoordinates(GraphType *, char *, int, MPI_Comm);
+void ReadMetisGraph(char *, int *, idxtype **, idxtype **);
+void Moc_SerialReadGraph(GraphType *, char *, int *, MPI_Comm);
+void Moc_SerialReadMetisGraph(char *, int *, int *, int *, int *, idxtype **, idxtype **, idxtype **, idxtype **, int *);
+
+/* adaptgraph */
+void AdaptGraph(GraphType *, int, MPI_Comm);
+void AdaptGraph2(GraphType *, int, MPI_Comm);
+void Mc_AdaptGraph(GraphType *, idxtype *, int, int, MPI_Comm);
+
+/* ptest.c */
+void TestParMetis(char *, MPI_Comm);
+
+/* NEW_ptest.c */
+void TestParMetis_V3(char *, MPI_Comm);
+int ComputeRealCut(idxtype *, idxtype *, char *, MPI_Comm);
+int ComputeRealCut2(idxtype *, idxtype *, idxtype *, idxtype *, char *, MPI_Comm);
+void TestMoveGraph(GraphType *, GraphType *, idxtype *, MPI_Comm);
+GraphType *SetUpGraph(CtrlType *, idxtype *, idxtype *, idxtype *, idxtype *, idxtype *, int);
+
+/* mienio.c */
+void mienIO(MeshType *, char *, int, int, MPI_Comm);
+
+/* meshio.c */
+void ParallelReadMesh(MeshType *, char *, MPI_Comm);
+
+/* parmetis.c */
+void ChangeToFortranNumbering(idxtype *, idxtype *, idxtype *, int, int);
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/pspases.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/pspases.c
new file mode 100644
index 0000000..8b7a182
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/pspases.c
@@ -0,0 +1,167 @@
+/*
+ * pspases.c
+ *
+ * This file contains ordering routines that are to be used with the
+ * parallel Cholesky factorization code PSPASES
+ *
+ * Started 10/14/97
+ * George
+ *
+ * $Id: pspases.c,v 1.3 2003/07/21 17:18:53 karypis Exp $
+ *
+ */
+
+#include <parmetislib.h>
+
+
+/***********************************************************************************
+* This function is the entry point of the serial ordering algorithm.
+************************************************************************************/
+void ParMETIS_SerialNodeND(idxtype *vtxdist, idxtype *xadj, idxtype *adjncy, int *numflag,
+                int *options, idxtype *order, idxtype *sizes, MPI_Comm *comm)
+{
+  int i, npes, mype, seroptions[10];
+  CtrlType ctrl;
+  GraphType *agraph;
+  idxtype *perm=NULL, *iperm=NULL;
+  int *sendcount, *displs;
+
+  MPI_Comm_size(*comm, &npes);
+  MPI_Comm_rank(*comm, &mype);
+
+  if (!ispow2(npes)) {
+    if (mype == 0)
+      printf("Error: The number of processors must be a power of 2!\n");
+    return;
+  }
+
+  if (*numflag == 1) 
+    ChangeNumbering(vtxdist, xadj, adjncy, order, npes, mype, 1);
+
+  SetUpCtrl(&ctrl, npes, options[OPTION_DBGLVL], *comm);
+
+  IFSET(ctrl.dbglvl, DBG_TIME, InitTimers(&ctrl));
+  IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
+  IFSET(ctrl.dbglvl, DBG_TIME, starttimer(ctrl.TotalTmr));
+
+  IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
+  IFSET(ctrl.dbglvl, DBG_TIME, starttimer(ctrl.MoveTmr));
+
+  agraph = AssembleEntireGraph(&ctrl, vtxdist, xadj, adjncy);
+
+  IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
+  IFSET(ctrl.dbglvl, DBG_TIME, stoptimer(ctrl.MoveTmr));
+
+
+  if (mype == 0) {
+    perm = idxmalloc(agraph->nvtxs, "PAROMETISS: perm");
+    iperm = idxmalloc(agraph->nvtxs, "PAROMETISS: iperm");
+
+    seroptions[0] = 0;
+    /*
+    seroptions[1] = 3;
+    seroptions[2] = 1;
+    seroptions[3] = 2;
+    seroptions[4] = 128;
+    seroptions[5] = 1;
+    seroptions[6] = 0;
+    seroptions[7] = 1;
+    */
+
+    METIS_NodeNDP(agraph->nvtxs, agraph->xadj, agraph->adjncy, npes, seroptions, perm, iperm, sizes);
+  }
+
+  IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
+  IFSET(ctrl.dbglvl, DBG_TIME, starttimer(ctrl.MoveTmr));
+
+  /* Broadcast the sizes array */
+  MPI_Bcast((void *)sizes, 2*npes, IDX_DATATYPE, 0, ctrl.gcomm);
+
+  /* Scatter the iperm */
+  sendcount = imalloc(npes, "PAROMETISS: sendcount");
+  displs = imalloc(npes, "PAROMETISS: displs");
+  for (i=0; i<npes; i++) {
+    sendcount[i] = vtxdist[i+1]-vtxdist[i];
+    displs[i] = vtxdist[i];
+  }
+
+  MPI_Scatterv((void *)iperm, sendcount, displs, IDX_DATATYPE, (void *)order, vtxdist[mype+1]-vtxdist[mype], IDX_DATATYPE, 0, ctrl.gcomm);
+
+  IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
+  IFSET(ctrl.dbglvl, DBG_TIME, stoptimer(ctrl.MoveTmr));
+
+  IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
+  IFSET(ctrl.dbglvl, DBG_TIME, stoptimer(ctrl.TotalTmr));
+  IFSET(ctrl.dbglvl, DBG_TIME, PrintTimingInfo(&ctrl));
+  IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
+
+  GKfree((void **)&agraph->xadj, (void **)&agraph->adjncy, (void **)&perm, (void **)&iperm, (void **)&sendcount, (void **)&displs, LTERM);
+  free(agraph);
+  FreeCtrl(&ctrl);
+
+  if (*numflag == 1) 
+    ChangeNumbering(vtxdist, xadj, adjncy, order, npes, mype, 0);
+
+}
+
+
+
+/*************************************************************************
+* This function assembles the graph into a single processor
+**************************************************************************/
+GraphType *AssembleEntireGraph(CtrlType *ctrl, idxtype *vtxdist, idxtype *xadj, idxtype *adjncy)
+{
+  int i, gnvtxs, nvtxs, gnedges, nedges;
+  int npes = ctrl->npes, mype = ctrl->mype;
+  idxtype *axadj, *aadjncy;
+  int *recvcounts, *displs;
+  GraphType *agraph;
+
+  gnvtxs = vtxdist[npes];
+  nvtxs = vtxdist[mype+1]-vtxdist[mype];
+  nedges = xadj[nvtxs];
+
+  recvcounts = imalloc(npes, "AssembleGraph: recvcounts");
+  displs = imalloc(npes+1, "AssembleGraph: displs");
+
+  /* Gather all the xadj arrays first */
+  for (i=0; i<nvtxs; i++)
+    xadj[i] = xadj[i+1]-xadj[i];
+
+  axadj = idxmalloc(gnvtxs+1, "AssembleEntireGraph: axadj");
+
+  for (i=0; i<npes; i++) {
+    recvcounts[i] = vtxdist[i+1]-vtxdist[i];
+    displs[i] = vtxdist[i];
+  }
+
+  /* Assemble the xadj and then the adjncy */
+  MPI_Gatherv((void *)xadj, nvtxs, IDX_DATATYPE, axadj, recvcounts, displs, IDX_DATATYPE, 0, ctrl->comm);
+
+  MAKECSR(i, nvtxs, xadj);
+  MAKECSR(i, gnvtxs, axadj);
+
+  /* Gather all the adjncy arrays next */
+  /* Determine the # of edges stored at each processor */
+  MPI_Allgather((void *)(&nedges), 1, MPI_INT, (void *)recvcounts, 1, MPI_INT, ctrl->comm);
+  
+  displs[0] = 0;
+  for (i=1; i<npes+1; i++) 
+    displs[i] = displs[i-1] + recvcounts[i-1];
+  gnedges = displs[npes];
+
+  aadjncy = idxmalloc(gnedges, "AssembleEntireGraph: aadjncy");
+
+  /* Assemble the xadj and then the adjncy */
+  MPI_Gatherv((void *)adjncy, nedges, IDX_DATATYPE, aadjncy, recvcounts, displs, IDX_DATATYPE, 0, ctrl->comm);
+
+  /* myprintf(ctrl, "Gnvtxs: %d, Gnedges: %d\n", gnvtxs, gnedges); */
+
+  agraph = CreateGraph();
+  agraph->nvtxs = gnvtxs;
+  agraph->nedges = gnedges;
+  agraph->xadj = axadj;
+  agraph->adjncy = aadjncy; 
+
+  return agraph;
+}
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/redomylink.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/redomylink.c
new file mode 100644
index 0000000..98b6810
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/redomylink.c
@@ -0,0 +1,175 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * redomylink.c
+ *
+ * This file contains code that implements the edge-based FM refinement
+ *
+ * Started 7/23/97
+ * George
+ *
+ * $Id: redomylink.c,v 1.2 2003/07/21 17:18:53 karypis Exp $
+ */
+
+#include <parmetislib.h>
+#define	PE	0
+
+/*************************************************************************
+* This function performs an edge-based FM refinement
+**************************************************************************/
+void RedoMyLink(CtrlType *ctrl, GraphType *graph, idxtype *home, int me,
+  int you, float *flows, float *sr_cost, float *sr_lbavg)
+{
+  int h, i, r;
+  int nvtxs, nedges, ncon;
+  int  pass, lastseed, totalv;
+  idxtype *xadj, *adjncy, *adjwgt, *where, *vsize;
+  idxtype *costwhere, *lbwhere, *selectwhere;
+  idxtype *rdata, *ed, *id, *bndptr, *bndind, *perm;
+  float *nvwgt, mycost;
+  float lbavg, lbvec[MAXNCON];
+  float best_lbavg, other_lbavg = -1.0, bestcost, othercost = -1.0;
+  float npwgts[2*MAXNCON], pwgts[MAXNCON*2], tpwgts[MAXNCON*2];
+  float ipc_factor, redist_factor, ftmp;
+int mype;
+MPI_Comm_rank(MPI_COMM_WORLD, &mype);
+
+  nvtxs = graph->nvtxs;
+  nedges = graph->nedges;
+  ncon = graph->ncon;
+  xadj = graph->xadj;
+  nvwgt = graph->nvwgt;
+  vsize = graph->vsize;
+  adjncy = graph->adjncy;
+  adjwgt = graph->adjwgt;
+  where = graph->where;
+  ipc_factor = ctrl->ipc_factor;
+  redist_factor = ctrl->redist_factor;
+
+  /**************************/
+  /* set up data structures */
+  /**************************/
+  rdata = idxmalloc(7*nvtxs, "rdata");
+  id = graph->sendind = rdata;
+  ed = graph->recvind = rdata + nvtxs;
+  bndptr = graph->sendptr = rdata + 2*nvtxs;
+  bndind = graph->recvptr = rdata + 3*nvtxs;
+  costwhere = rdata + 4*nvtxs;
+  lbwhere = rdata + 5*nvtxs;
+  perm = rdata + 6*nvtxs;
+  graph->gnpwgts = npwgts;
+
+  RandomPermute(nvtxs, perm, 1);
+  idxcopy(nvtxs, where, costwhere);
+  idxcopy(nvtxs, where, lbwhere);
+
+  /*****************************/
+  /* compute target pwgts      */
+  /*****************************/
+  sset(ncon*2, 0.0, pwgts);
+  for (h=0; h<ncon; h++) {
+    tpwgts[h] = -1.0 * flows[h];
+    tpwgts[ncon+h] = flows[h];
+  }
+
+  for (i=0; i<nvtxs; i++) {
+    if (where[i] == me) {
+      for (h=0; h<ncon; h++) {
+        tpwgts[h] += nvwgt[i*ncon+h];
+        pwgts[h] += nvwgt[i*ncon+h];
+      }
+    }
+    else {
+      ASSERTS(where[i] == you);
+      for (h=0; h<ncon; h++) {
+        tpwgts[ncon+h] += nvwgt[i*ncon+h];
+        pwgts[ncon+h] += nvwgt[i*ncon+h];
+      }
+    }
+  }
+
+  /* we don't want any weights to be less than zero */
+  for (h=0; h<ncon; h++) {
+    if (tpwgts[h] < 0.0) {
+      tpwgts[ncon+h] += tpwgts[h];
+      tpwgts[h] = 0.0;
+    }
+
+    if (tpwgts[ncon+h] < 0.0) {
+      tpwgts[h] += tpwgts[ncon+h];
+      tpwgts[ncon+h] = 0.0;
+    }
+  } 
+
+  /*****************************/
+  /* now compute new bisection */
+  /*****************************/
+  bestcost = (float)idxsum(nedges, adjwgt)*ipc_factor + (float)idxsum(nvtxs, vsize)*redist_factor;
+  best_lbavg = 10.0;
+
+  lastseed = 0;
+  for (pass = N_MOC_REDO_PASSES; pass>0; pass--) {
+    idxset(nvtxs, 1, where);
+
+    /***************************/
+    /* find seed vertices      */
+    /***************************/
+    r = perm[lastseed] % nvtxs;
+    lastseed = (lastseed+1) % nvtxs;
+    where[r] = 0;
+
+    Moc_Serial_Compute2WayPartitionParams(graph);
+    Moc_Serial_Init2WayBalance(graph, tpwgts);
+    Moc_Serial_FM_2WayRefine(graph, tpwgts, 4);
+    Moc_Serial_Balance2Way(graph, tpwgts, 1.02);
+    Moc_Serial_FM_2WayRefine(graph, tpwgts, 4);
+
+    for (i=0; i<nvtxs; i++)
+      where[i] = (where[i] == 0) ? me : you;
+
+    for (i=0; i<ncon; i++) {
+      ftmp = (pwgts[i]+pwgts[ncon+i])/2.0;
+      if (ftmp != 0.0)
+        lbvec[i] = fabs(npwgts[i]-tpwgts[i])/ftmp;
+      else
+        lbvec[i] = 0.0;
+    }
+    lbavg = savg(ncon, lbvec);
+
+    totalv = 0;
+    for (i=0; i<nvtxs; i++)
+      if (where[i] != home[i])
+        totalv += vsize[i];
+
+    mycost = (float)(graph->mincut)*ipc_factor + (float)totalv*redist_factor;
+
+    if (bestcost >= mycost) {
+      bestcost = mycost;
+      other_lbavg = lbavg;
+      idxcopy(nvtxs, where, costwhere);
+    }
+
+    if (best_lbavg >= lbavg) {
+      best_lbavg = lbavg;
+      othercost = mycost;
+      idxcopy(nvtxs, where, lbwhere);
+    }
+  }
+
+  if (other_lbavg <= .05) {
+    selectwhere = costwhere;
+    *sr_cost = bestcost;
+    *sr_lbavg = other_lbavg;
+  }
+  else {
+    selectwhere = lbwhere;
+    *sr_cost = othercost;
+    *sr_lbavg = best_lbavg;
+  }
+
+  idxcopy(nvtxs, selectwhere, where);
+
+  GKfree((void **)&rdata, LTERM);
+  return;
+}
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/remap.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/remap.c
new file mode 100644
index 0000000..31f186b
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/remap.c
@@ -0,0 +1,181 @@
+/*
+ * premap.c
+ *
+ * This file contains code that computes the assignment of processors to
+ * partition numbers so that it will minimize the redistribution cost
+ *
+ * Started 4/16/98
+ * George
+ *
+ * $Id: remap.c,v 1.2 2003/07/21 17:18:53 karypis Exp $
+ *
+ */
+
+#include <parmetislib.h>
+
+/*************************************************************************
+* This function remaps that graph so that it will minimize the 
+* redistribution cost
+**************************************************************************/
+void ParallelReMapGraph(CtrlType *ctrl, GraphType *graph, WorkSpaceType *wspace)
+{
+  int i, nvtxs, nparts;
+  idxtype *where, *vsize, *map, *lpwgts;
+
+  IFSET(ctrl->dbglvl, DBG_TIME, MPI_Barrier(ctrl->comm));
+  IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->RemapTmr));
+
+  if (ctrl->npes != ctrl->nparts) {
+    IFSET(ctrl->dbglvl, DBG_TIME, stoptimer(ctrl->RemapTmr));
+    return;
+  }
+
+  nvtxs = graph->nvtxs;
+  where = graph->where;
+  vsize = graph->vsize;
+  nparts = ctrl->nparts;
+
+  map = wspace->pv1;
+  lpwgts = idxset(nparts, 0, wspace->pv2);
+
+  for (i=0; i<nvtxs; i++)
+    lpwgts[where[i]] += (vsize == NULL) ? 1 : vsize[i];
+
+  ParallelTotalVReMap(ctrl, lpwgts, map, wspace, NREMAP_PASSES, graph->ncon);
+
+  for (i=0; i<nvtxs; i++)
+    where[i] = map[where[i]];
+
+  IFSET(ctrl->dbglvl, DBG_TIME, MPI_Barrier(ctrl->comm));
+  IFSET(ctrl->dbglvl, DBG_TIME, stoptimer(ctrl->RemapTmr));
+}
+
+
+/*************************************************************************
+* This function computes the assignment using the the objective the 
+* minimization of the total volume of data that needs to move
+**************************************************************************/
+void ParallelTotalVReMap(CtrlType *ctrl, idxtype *lpwgts, idxtype *map,
+     WorkSpaceType *wspace, int npasses, int ncon)
+{
+  int i, ii, j, k, nparts, mype;
+  int pass, maxipwgt, nmapped, oldwgt, newwgt, done;
+  idxtype *rowmap, *mylpwgts;
+  KeyValueType *recv, send;
+  int nsaved, gnsaved;
+
+  mype = ctrl->mype;
+  nparts = ctrl->nparts;
+  recv = (KeyValueType *)GKmalloc(sizeof(KeyValueType)*nparts, "remap: recv");
+  mylpwgts = idxmalloc(nparts, "mylpwgts");
+
+  done = nmapped = 0;
+  idxset(nparts, -1, map);
+  rowmap = idxset(nparts, -1, wspace->pv3);
+  idxcopy(nparts, lpwgts, mylpwgts);
+  for (pass=0; pass<npasses; pass++) {
+    maxipwgt = idxamax(nparts, mylpwgts);
+
+    if (mylpwgts[maxipwgt] > 0 && !done) {
+      send.key = -mylpwgts[maxipwgt];
+      send.val = mype*nparts+maxipwgt;
+    }
+    else {
+      send.key = 0;
+      send.val = -1;
+    }
+
+    /* each processor sends its selection */
+    MPI_Allgather((void *)&send, 2, IDX_DATATYPE, (void *)recv, 2, IDX_DATATYPE, ctrl->comm); 
+
+    ikeysort(nparts, recv);
+    if (recv[0].key == 0)
+      break;
+
+    /* now make as many assignments as possible */
+    for (ii=0; ii<nparts; ii++) {
+      i = recv[ii].val;
+
+      if (i == -1)
+        continue;
+
+      j = i % nparts;
+      k = i / nparts;
+      if (map[j] == -1 && rowmap[k] == -1 && SimilarTpwgts(ctrl->tpwgts, ncon, j, k)) {
+        map[j] = k;
+        rowmap[k] = j;
+        nmapped++;
+        mylpwgts[j] = 0;
+        if (mype == k)
+          done = 1;
+      }
+
+      if (nmapped == nparts)
+        break;
+    }
+
+    if (nmapped == nparts)
+      break;
+  }
+
+  /* Map unmapped partitions */
+  if (nmapped < nparts) {
+    for (i=j=0; j<nparts && nmapped<nparts; j++) {
+      if (map[j] == -1) {
+        for (; i<nparts; i++) {
+          if (rowmap[i] == -1 && SimilarTpwgts(ctrl->tpwgts, ncon, i, j)) {
+            map[j] = i;
+            rowmap[i] = j;
+            nmapped++;
+            break;
+          }
+        }
+      }
+    }
+  }
+
+  /* check to see if remapping fails (due to dis-similar tpwgts) */
+  /* if remapping fails, revert to original mapping */
+  if (nmapped < nparts) {
+    for (i=0; i<nparts; i++)
+      map[i] = i; 
+    IFSET(ctrl->dbglvl, DBG_REMAP, rprintf(ctrl, "Savings from parallel remapping: %0\n")); 
+  }
+  else {
+    /* check for a savings */
+    oldwgt = lpwgts[mype];
+    newwgt = lpwgts[rowmap[mype]];
+    nsaved = newwgt - oldwgt;
+    gnsaved = GlobalSESum(ctrl, nsaved);
+
+    /* undo everything if we don't see a savings */
+    if (gnsaved <= 0) {
+      for (i=0; i<nparts; i++)
+        map[i] = i;
+    }
+    IFSET(ctrl->dbglvl, DBG_REMAP, rprintf(ctrl, "Savings from parallel remapping: %d\n", amax(0,gnsaved))); 
+  }
+
+  GKfree((void **)&recv, (void **)&mylpwgts, LTERM);
+
+}
+
+
+/*************************************************************************
+* This function computes the assignment using the the objective the
+* minimization of the total volume of data that needs to move
+**************************************************************************/
+int SimilarTpwgts(float *tpwgts, int ncon, int s1, int s2)
+{
+  int i;
+
+  for (i=0; i<ncon; i++)
+    if (fabs(tpwgts[s1*ncon+i]-tpwgts[s2*ncon+i]) > SMALLFLOAT)
+      break;
+
+  if (i == ncon)
+    return 1;
+
+  return 0;
+}
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/rename.h b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/rename.h
new file mode 100644
index 0000000..d993257
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/rename.h
@@ -0,0 +1,290 @@
+/* kmetis.c */
+#define	Moc_Global_Partition Moc_Global_Partition__
+
+/* mmetis.c */
+
+/* gkmetis.c */
+
+/* match.c */
+#define	Moc_GlobalMatch_Balance Moc_GlobalMatch_Balance__
+
+/* coarsen.c */
+#define	Moc_Global_CreateCoarseGraph Moc_Global_CreateCoarseGraph__
+
+/* initpart.c */
+#define	Moc_InitPartition_RB Moc_InitPartition_RB__
+#define	Moc_KeepPart Moc_KeepPart__
+
+/* kwayrefine.c */
+#define	Moc_ProjectPartition Moc_ProjectPartition__
+#define	Moc_ComputePartitionParams Moc_ComputePartitionParams__
+
+/* kwayfm.c */
+#define	Moc_KWayFM Moc_KWayFM__
+
+/* kwaybalance.c */
+#define	Moc_KWayBalance Moc_KWayBalance__
+
+/* remap.c */
+#define	ParallelReMapGraph ParallelReMapGraph__
+#define	ParallelTotalVReMap ParallelTotalVReMap__
+#define	SimilarTpwgts SimilarTpwgts__
+
+/* move.c */
+#define	Moc_MoveGraph Moc_MoveGraph__
+#define	CheckMGraph CheckMGraph__
+#define	ProjectInfoBack ProjectInfoBack__
+#define	FindVtxPerm FindVtxPerm__
+
+/* memory.c */
+#define	PreAllocateMemory PreAllocateMemory__
+#define	FreeWSpace FreeWSpace__
+#define	FreeCtrl FreeCtrl__
+#define	CreateGraph CreateGraph__
+#define	InitGraph InitGraph__
+#define	FreeGraph FreeGraph__
+#define	FreeInitialGraphAndRemap FreeInitialGraphAndRemap__
+
+
+/************************/
+/* Adaptive subroutines */
+/************************/
+/* ametis.c */
+#define	Adaptive_Partition Adaptive_Partition__
+
+/* rmetis.c */
+
+/* lmatch.c */
+#define	Mc_LocalMatch_HEM Mc_LocalMatch_HEM__
+#define	Mc_Local_CreateCoarseGraph Mc_Local_CreateCoarseGraph__
+
+/* wave.c */
+#define	WavefrontDiffusion WavefrontDiffusion__
+
+/* balancemylink.c */
+#define	BalanceMyLink BalanceMyLink__
+
+/* redomylink.c */
+#define	RedoMyLink RedoMyLink__
+
+/* initbalance.c */
+#define	Balance_Partition Balance_Partition__
+#define	Moc_AssembleAdaptiveGraph Moc_AssembleAdaptiveGraph__
+
+/* mdiffusion.c */
+#define	Moc_Diffusion Moc_Diffusion__
+#define	ExtractGraph ExtractGraph__
+
+/* diffutil.c */
+#define	SetUpConnectGraph SetUpConnectGraph__
+#define	Mc_ComputeMoveStatistics Mc_ComputeMoveStatistics__
+#define	Mc_ComputeSerialTotalV Mc_ComputeSerialTotalV__
+#define	ComputeLoad ComputeLoad__
+#define	ConjGrad2 ConjGrad2__
+#define	mvMult2 mvMult2__
+#define	ComputeTransferVector ComputeTransferVector__
+#define	ComputeSerialEdgeCut ComputeSerialEdgeCut__
+#define	ComputeSerialTotalV ComputeSerialTotalV__
+
+/* akwayfm.c */
+#define	Moc_KWayAdaptiveRefine Moc_KWayAdaptiveRefine__
+
+/* selectq.c */
+#define	Moc_DynamicSelectQueue Moc_DynamicSelectQueue__
+#define	Moc_HashVwgts Moc_HashVwgts__
+#define	Moc_HashVRank Moc_HashVRank__
+
+/* csrmatch.c */
+#define	CSR_Match_SHEM CSR_Match_SHEM__
+
+/* serial.c */
+#define	Moc_SerialKWayAdaptRefine Moc_SerialKWayAdaptRefine__
+#define	Moc_ComputeSerialPartitionParams Moc_ComputeSerialPartitionParams__
+#define	AreAllHVwgtsBelow AreAllHVwgtsBelow__
+#define	ComputeHKWayLoadImbalance ComputeHKWayLoadImbalance__
+#define	SerialRemap SerialRemap__
+#define	SSMIncKeyCmp SSMIncKeyCmp__
+#define	Moc_Serial_FM_2WayRefine Moc_Serial_FM_2WayRefine__
+#define	Serial_SelectQueue Serial_SelectQueue__
+#define	Serial_BetterBalance Serial_BetterBalance__
+#define	Serial_Compute2WayHLoadImbalance Serial_Compute2WayHLoadImbalance__
+#define	Moc_Serial_Balance2Way Moc_Serial_Balance2Way__
+#define	Moc_Serial_Init2WayBalance Moc_Serial_Init2WayBalance__
+#define	Serial_SelectQueueOneWay Serial_SelectQueueOneWay__
+#define	Moc_Serial_Compute2WayPartitionParams Moc_Serial_Compute2WayPartitionParams__
+#define	Serial_AreAnyVwgtsBelow Serial_AreAnyVwgtsBelow__
+
+/* weird.c */
+#define	PartitionSmallGraph PartitionSmallGraph__
+#define	CheckInputs CheckInputs__
+
+
+/****************************/
+/* Mesh to Dual subroutines */
+/****************************/
+/* mesh.c */
+/* msetup.c */
+#define	SetUpMesh SetUpMesh__
+#define	CreateMesh CreateMesh__
+#define	InitMesh InitMesh__
+
+
+/************************/
+/* Ordering subroutines */
+/************************/
+/* ometis.c */
+/* pspases.c */
+#define	AssembleEntireGraph AssembleEntireGraph__
+
+/* node_refine.c */
+#define	ComputeNodePartitionParams0 ComputeNodePartitionParams0__
+#define	ComputeNodePartitionParams ComputeNodePartitionParams__
+#define	KWayNodeRefine0 KWayNodeRefine0__
+#define	KWayNodeRefine KWayNodeRefine__
+#define	KWayNodeRefine2 KWayNodeRefine2__
+#define	PrintNodeBalanceInfo PrintNodeBalanceInfo__
+
+/* initmsection.c */
+#define	InitMultisection InitMultisection__
+#define	AssembleMultisectedGraph AssembleMultisectedGraph__
+
+/* order.c */
+#define	MultilevelOrder MultilevelOrder__
+#define	LabelSeparators LabelSeparators__
+#define	CompactGraph CompactGraph__
+#define	LocalOrder LocalOrder__
+#define	LocalNDOrder LocalNDOrder__
+#define	Order_Partition Order_Partition__
+
+/* xyzpart.c */
+#define	Coordinate_Partition Coordinate_Partition__
+#define	PartSort PartSort__
+
+/***********************/
+/* Utility subroutines */
+/***********************/
+/* fpqueue.c */
+#define	FPQueueInit FPQueueInit__
+#define	FPQueueReset FPQueueReset__
+#define	FPQueueFree FPQueueFree__
+#define	FPQueueGetSize FPQueueGetSize__
+#define	FPQueueInsert FPQueueInsert__
+#define	FPQueueDelete FPQueueDelete__
+#define	FPQueueUpdate FPQueueUpdate__
+#define	FPQueueUpdateUp FPQueueUpdateUp__
+#define	FPQueueGetMax FPQueueGetMax__
+#define	FPQueueSeeMaxVtx FPQueueSeeMaxVtx__
+#define	FPQueueSeeMaxGain FPQueueSeeMaxGain__
+#define	FPQueueGetKey FPQueueGetKey__
+#define	FPQueueGetQSize FPQueueGetQSize__
+#define	CheckHeapFloat CheckHeapFloat__
+
+/* stat.c */
+#define	Moc_ComputeSerialBalance Moc_ComputeSerialBalance__
+#define	Moc_ComputeParallelBalance Moc_ComputeParallelBalance__
+#define	Moc_PrintThrottleMatrix Moc_PrintThrottleMatrix__
+#define	Moc_ComputeRefineStats Moc_ComputeRefineStats__
+
+/* debug.c */
+#define	PrintVector PrintVector__
+#define	PrintVector2 PrintVector2__
+#define	PrintPairs PrintPairs__
+#define	PrintGraph PrintGraph__
+#define	PrintGraph2 PrintGraph2__
+#define	PrintSetUpInfo PrintSetUpInfo__
+#define	PrintTransferedGraphs PrintTransferedGraphs__
+#define	WriteMetisGraph WriteMetisGraph__
+
+/* comm.c */
+#define	CommInterfaceData CommInterfaceData__
+#define	CommChangedInterfaceData CommChangedInterfaceData__
+#define	GlobalSEMax GlobalSEMax__
+#define	GlobalSEMaxDouble GlobalSEMaxDouble__
+#define	GlobalSEMin GlobalSEMin__
+#define	GlobalSESum GlobalSESum__
+#define	GlobalSEMaxFloat GlobalSEMaxFloat__
+#define	GlobalSEMinFloat GlobalSEMinFloat__
+#define	GlobalSESumFloat GlobalSESumFloat__
+
+/* util.c */
+#define	errexit errexit__
+#define	myprintf myprintf__
+#define	rprintf rprintf__
+#define	imalloc imalloc__
+#define	idxmalloc idxmalloc__
+#define	fmalloc fmalloc__
+#define	ismalloc ismalloc__
+#define	idxsmalloc idxsmalloc__
+#define	GKmalloc GKmalloc__
+#define	GKfree GKfree__
+#define	iset iset__
+#define	idxset idxset__
+#define	idxamax idxamax__
+#define	idxamin idxamin__
+#define	idxasum idxasum__
+#define	snorm2 snorm2__
+#define	sdot sdot__
+#define	saxpy saxpy__
+#define	ikeyvalsort_org ikeyvalsort_org__
+#define	IncKeyValueCmp IncKeyValueCmp__
+#define	dkeyvalsort dkeyvalsort__
+#define	DecKeyValueCmp DecKeyValueCmp__
+#define	BSearch BSearch__
+#define	RandomPermute RandomPermute__
+#define	FastRandomPermute FastRandomPermute__
+#define	ispow2 ispow2__
+#define	log2Int log2Int__
+#define	BucketSortKeysDec BucketSortKeysDec__
+#define	sset sset__
+#define	iamax iamax__
+#define	idxamax_strd idxamax_strd__
+#define	idxamin_strd idxamin_strd__
+#define	samax_strd samax_strd__
+#define	sfamax sfamax__
+#define	samin_strd samin_strd__
+#define	idxavg idxavg__
+#define	savg savg__
+#define	samax samax__
+#define	sfavg sfavg__
+#define	samax2 samax2__
+#define	samin samin__
+#define	idxsum idxsum__
+#define	idxsum_strd idxsum_strd__
+#define	idxadd idxadd__
+#define	ssum ssum__
+#define	ssum_strd ssum_strd__
+#define	sscale sscale__
+#define	saneg saneg__
+#define	BetterVBalance BetterVBalance__
+#define	IsHBalanceBetterTT IsHBalanceBetterTT__
+#define	IsHBalanceBetterFT IsHBalanceBetterFT__
+#define	myvalkeycompare myvalkeycompare__
+#define	imyvalkeycompare imyvalkeycompare__
+#define	fsmalloc fsmalloc__
+#define	saxpy2 saxpy2__
+#define	GetThreeMax GetThreeMax__
+
+/* qsort_special.c */
+#define	iidxsort iidxsort__
+#define	iintsort iintsort__
+#define	ikeysort ikeysort__
+#define	ikeyvalsort ikeyvalsort__
+
+/* grsetup.c */
+#define	Moc_SetUpGraph Moc_SetUpGraph__
+#define	SetUpCtrl SetUpCtrl__
+#define	ChangeNumbering ChangeNumbering__
+#define	ChangeNumberingMesh ChangeNumberingMesh__
+#define	GraphRandomPermute GraphRandomPermute__
+#define	ComputeMoveStatistics ComputeMoveStatistics__
+
+/* timer.c */
+#define	InitTimers InitTimers__
+#define	PrintTimingInfo PrintTimingInfo__
+#define	PrintTimer PrintTimer__
+
+/* setup.c */
+#define	SetUp SetUp__
+#define	Home_PE Home_PE__
+
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/rmetis.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/rmetis.c
new file mode 100644
index 0000000..3755209
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/rmetis.c
@@ -0,0 +1,165 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * rmetis.c
+ *
+ * This is the entry point of the partitioning refinement routine
+ *
+ * Started 10/19/96
+ * George
+ *
+ * $Id: rmetis.c,v 1.5 2003/07/25 04:01:05 karypis Exp $
+ *
+ */
+
+#include <parmetislib.h>
+
+
+
+/***********************************************************************************
+* This function is the entry point of the parallel multilevel local diffusion
+* algorithm. It uses parallel undirected diffusion followed by adaptive k-way 
+* refinement. This function utilizes local coarsening.
+************************************************************************************/
+void ParMETIS_V3_RefineKway(idxtype *vtxdist, idxtype *xadj, idxtype *adjncy,
+              idxtype *vwgt, idxtype *adjwgt, int *wgtflag, int *numflag, int *ncon, 
+	      int *nparts, float *tpwgts, float *ubvec, int *options, int *edgecut, 
+	      idxtype *part, MPI_Comm *comm)
+{
+  int h, i;
+  int npes, mype;
+  CtrlType ctrl;
+  WorkSpaceType wspace;
+  GraphType *graph;
+  int tewgt, tvsize, nmoved, maxin, maxout;
+  float gtewgt, gtvsize, avg, maximb;
+  int ps_relation, seed, dbglvl = 0;
+  int iwgtflag, inumflag, incon, inparts, ioptions[10];
+  float *itpwgts, iubvec[MAXNCON];
+
+  MPI_Comm_size(*comm, &npes);
+  MPI_Comm_rank(*comm, &mype);
+
+  /********************************/
+  /* Try and take care bad inputs */
+  /********************************/
+  if (options != NULL && options[0] == 1)
+    dbglvl = options[PMV3_OPTION_DBGLVL];
+  CheckInputs(REFINE_PARTITION, npes, dbglvl, wgtflag, &iwgtflag, numflag, &inumflag,
+              ncon, &incon, nparts, &inparts, tpwgts, &itpwgts, ubvec, iubvec, 
+              NULL, NULL, options, ioptions, part, comm);
+
+  /* ADD: take care of disconnected graph */
+  /* ADD: take care of highly unbalanced vtxdist */
+  /*********************************/
+  /* Take care the nparts = 1 case */
+  /*********************************/
+  if (inparts <= 1) {
+    idxset(vtxdist[mype+1]-vtxdist[mype], 0, part); 
+    *edgecut = 0;
+    return;
+  }
+
+  /**************************/
+  /* Set up data structures */
+  /**************************/
+  if (inumflag == 1) 
+    ChangeNumbering(vtxdist, xadj, adjncy, part, npes, mype, 1);
+
+  /*****************************/
+  /* Set up control structures */
+  /*****************************/
+  if (ioptions[0] == 1) {
+    dbglvl = ioptions[PMV3_OPTION_DBGLVL];
+    seed = ioptions[PMV3_OPTION_SEED];
+    ps_relation = (npes == inparts) ? ioptions[PMV3_OPTION_PSR] : DISCOUPLED;
+  }
+  else {
+    dbglvl = GLOBAL_DBGLVL;
+    seed = GLOBAL_SEED;
+    ps_relation = (npes == inparts) ? COUPLED : DISCOUPLED;
+  }
+
+  SetUpCtrl(&ctrl, inparts, dbglvl, *comm);
+  ctrl.CoarsenTo = amin(vtxdist[npes]+1, 50*incon*amax(npes, inparts));
+  ctrl.ipc_factor = 1000.0;
+  ctrl.redist_factor = 1.0;
+  ctrl.redist_base = 1.0;
+  ctrl.seed = (seed == 0) ? mype : seed*mype;
+  ctrl.sync = GlobalSEMax(&ctrl, seed);
+  ctrl.partType = REFINE_PARTITION;
+  ctrl.ps_relation = ps_relation;
+  ctrl.tpwgts = itpwgts;
+
+  graph = Moc_SetUpGraph(&ctrl, incon, vtxdist, xadj, vwgt, adjncy, adjwgt, &iwgtflag);
+  graph->vsize = idxsmalloc(graph->nvtxs, 1, "vsize");
+
+  graph->home = idxmalloc(graph->nvtxs, "home");
+  if (ctrl.ps_relation == COUPLED)
+    idxset(graph->nvtxs, mype, graph->home);
+  else
+    idxcopy(graph->nvtxs, part, graph->home);
+
+  tewgt   = idxsum(graph->nedges, graph->adjwgt);
+  tvsize  = idxsum(graph->nvtxs, graph->vsize);
+  gtewgt  = (float) GlobalSESum(&ctrl, tewgt) + 1.0/graph->gnvtxs;
+  gtvsize = (float) GlobalSESum(&ctrl, tvsize) + 1.0/graph->gnvtxs;
+  ctrl.edge_size_ratio = gtewgt/gtvsize;
+  scopy(incon, iubvec, ctrl.ubvec);
+
+  PreAllocateMemory(&ctrl, graph, &wspace);
+
+  /***********************/
+  /* Partition and Remap */
+  /***********************/
+  IFSET(ctrl.dbglvl, DBG_TIME, InitTimers(&ctrl));
+  IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
+  IFSET(ctrl.dbglvl, DBG_TIME, starttimer(ctrl.TotalTmr));
+
+  Adaptive_Partition(&ctrl, graph, &wspace);
+  ParallelReMapGraph(&ctrl, graph, &wspace);
+
+  IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
+  IFSET(ctrl.dbglvl, DBG_TIME, stoptimer(ctrl.TotalTmr));
+
+  idxcopy(graph->nvtxs, graph->where, part);
+  if (edgecut != NULL)
+    *edgecut = graph->mincut;
+
+  /***********************/
+  /* Take care of output */
+  /***********************/
+  IFSET(ctrl.dbglvl, DBG_TIME, PrintTimingInfo(&ctrl));
+  IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
+
+  if (ctrl.dbglvl&DBG_INFO) {
+    Mc_ComputeMoveStatistics(&ctrl, graph, &nmoved, &maxin, &maxout);
+    rprintf(&ctrl, "Final %3d-way Cut: %6d \tBalance: ", inparts, graph->mincut);
+    avg = 0.0;
+    for (h=0; h<incon; h++) {
+      maximb = 0.0;
+      for (i=0; i<inparts; i++)
+        maximb = amax(maximb, graph->gnpwgts[i*incon+h]/itpwgts[i*incon+h]);
+      avg += maximb;
+      rprintf(&ctrl, "%.3f ", maximb);
+    }
+    rprintf(&ctrl, "\nNMoved: %d %d %d %d\n", nmoved, maxin, maxout, maxin+maxout);
+  }
+
+  /*************************************/
+  /* Free memory, renumber, and return */
+  /*************************************/
+  GKfree((void **)&graph->lnpwgts, (void **)&graph->gnpwgts, (void **)&graph->nvwgt, (void **)(&graph->home), (void **)(&graph->vsize), LTERM);
+
+  GKfree((void **)&itpwgts, LTERM);
+  FreeInitialGraphAndRemap(graph, iwgtflag);
+  FreeWSpace(&wspace);
+  FreeCtrl(&ctrl);
+
+  if (inumflag == 1)
+    ChangeNumbering(vtxdist, xadj, adjncy, part, npes, mype, 0);
+
+  return;
+}
+
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/selectq.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/selectq.c
new file mode 100644
index 0000000..5ec9109
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/selectq.c
@@ -0,0 +1,340 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * selectq.c
+ *
+ * This file contains the driving routines for multilevel k-way refinement
+ *
+ * Started 7/28/97
+ * George
+ *
+ * $Id: selectq.c,v 1.2 2003/07/21 17:18:53 karypis Exp $
+ */
+
+#include <parmetislib.h>
+
+/*************************************************************************
+*  This stuff is hardcoded for up to four constraints
+**************************************************************************/
+void Moc_DynamicSelectQueue(int nqueues, int ncon, int subdomain1, int subdomain2,
+     idxtype *currentq, float *flows, int *from, int *qnum, int minval, float avgvwgt,
+     float maxdiff)
+{
+  int i, j;
+  int hash, index = -1, current;
+  int cand[MAXNCON], rank[MAXNCON], dont_cares[MAXNCON];
+  int nperms, perm[24][5];
+  float sign = 0.0;
+  KVType array[MAXNCON];
+int mype;
+MPI_Comm_rank(MPI_COMM_WORLD, &mype);
+
+  *qnum = -1;
+
+  if (*from == -1) {
+    for (i=0; i<ncon; i++) {
+      array[i].key = i;
+      array[i].val = (fabs)(flows[i]);
+    }
+
+    qsort(array, ncon, sizeof(KVType), myvalkeycompare);
+    ASSERTS(array[ncon-1].val - array[0].val <= maxdiff)
+
+    if (flows[array[ncon-1].key]>avgvwgt*MOC_GD_GRANULARITY_FACTOR) {
+      *from = subdomain1;
+      sign = 1.0;
+      index = 0;
+    }
+
+    if (flows[array[ncon-1].key]<-1.0*avgvwgt*MOC_GD_GRANULARITY_FACTOR) {
+      *from = subdomain2;
+      sign = -1.0;
+      index = nqueues;
+    }
+
+    if (*from == -1) {
+      return;
+    }
+  }
+  else {
+    ASSERTS(*from == subdomain1 || *from == subdomain2);
+
+    if (*from == subdomain1) {
+      sign = 1.0;
+      index = 0;
+    }
+    else {
+      sign = -1.0;
+      index = nqueues;
+    }
+  }
+
+  for (i=0; i<ncon; i++) {
+    array[i].key = i;
+    array[i].val = flows[i] * sign;
+  }
+
+  qsort(array, ncon, sizeof(KVType), myvalkeycompare);
+
+  iset(ncon, 1, dont_cares);
+
+  current = 0;
+  for (i=0; i<ncon-1; i++) 
+    if (array[i+1].val - array[i].val < maxdiff * MC_FLOW_BALANCE_THRESHOLD && dont_cares[current] < ncon-1) {
+      dont_cares[current]++;
+      dont_cares[i+1] = 0;
+    }
+    else
+      current = i+1;
+
+
+  switch (ncon) {
+    /***********************/
+    case 2:
+      nperms = 1;
+      perm[0][0] = 0;   perm[0][1] = 1;
+
+      break;
+    /***********************/
+    case 3:
+
+      /* if the first and second flows are close */
+      if (dont_cares[0] == 2 && dont_cares[1] == 0 && dont_cares[2] == 1) {
+        nperms = 4;
+        perm[0][0] = 0;   perm[0][1] = 1;   perm[0][2] = 2;
+        perm[1][0] = 1;   perm[1][1] = 0;   perm[1][2] = 2;
+        perm[2][0] = 0;   perm[2][1] = 2;   perm[2][2] = 1;
+        perm[3][0] = 1;   perm[3][1] = 2;   perm[3][2] = 0;
+        break;
+      }
+
+      /* if the second and third flows are close */
+      if (dont_cares[0] == 1 && dont_cares[1] == 2 && dont_cares[2] == 0) {
+        nperms = 4;
+        perm[0][0] = 0;   perm[0][1] = 1;   perm[0][2] = 2;
+        perm[1][0] = 0;   perm[1][1] = 2;   perm[1][2] = 1;
+        perm[2][0] = 1;   perm[2][1] = 0;   perm[2][2] = 2;
+        perm[3][0] = 2;   perm[3][1] = 0;   perm[3][2] = 1;
+        break;
+      }
+
+      /* all or none of the flows are close */
+      nperms = 3;
+      perm[0][0] = 0;   perm[0][1] = 1;   perm[0][2] = 2;
+      perm[1][0] = 1;   perm[1][1] = 0;   perm[1][2] = 2;
+      perm[2][0] = 0;   perm[2][1] = 2;   perm[2][2] = 1;
+
+      break;
+    /***********************/
+    case 4:
+
+      if (dont_cares[0] == 2 && dont_cares[1] == 0 &&
+          dont_cares[2] == 1 && dont_cares[3] == 1) {
+        nperms = 14;
+        perm[0][0] =  0;   perm[0][1] =  1;   perm[0][2] =  2;   perm[0][3] =  3;
+        perm[1][0] =  1;   perm[1][1] =  0;   perm[1][2] =  2;   perm[1][3] =  3;
+        perm[2][0] =  0;   perm[2][1] =  2;   perm[2][2] =  1;   perm[2][3] =  3;
+        perm[3][0] =  1;   perm[3][1] =  2;   perm[3][2] =  0;   perm[3][3] =  3;
+        perm[4][0] =  0;   perm[4][1] =  1;   perm[4][2] =  3;   perm[4][3] =  2;
+        perm[5][0] =  1;   perm[5][1] =  0;   perm[5][2] =  3;   perm[5][3] =  2;
+        
+        perm[6][0] =  0;   perm[6][1] =  3;   perm[6][2] =  1;   perm[6][3] =  2;
+        perm[7][0] =  1;   perm[7][1] =  3;   perm[7][2] =  0;   perm[7][3] =  2;
+
+        perm[8][0] =  0;   perm[8][1] =  2;   perm[8][2] =  3;   perm[8][3] =  1;
+        perm[9][0] =  1;   perm[9][1] =  2;   perm[9][2] =  3;   perm[9][3] =  0;
+
+        perm[10][0] = 2;   perm[10][1] = 0;   perm[10][2] = 1;   perm[10][3] = 3;
+        perm[11][0] = 2;   perm[11][1] = 1;   perm[11][2] = 0;   perm[11][3] = 3;
+        
+        perm[12][0] = 0;   perm[12][1] = 3;   perm[12][2] = 2;   perm[12][3] = 1;
+        perm[13][0] = 1;   perm[13][1] = 3;   perm[13][2] = 2;   perm[13][3] = 0;
+        break;
+      }
+
+      if (dont_cares[0] == 1 && dont_cares[1] == 1 &&
+          dont_cares[2] == 2 && dont_cares[3] == 0) {
+        nperms = 14;
+        perm[0][0] =  0;   perm[0][1] =  1;   perm[0][2] =  2;   perm[0][3] =  3;
+        perm[1][0] =  0;   perm[1][1] =  1;   perm[1][2] =  3;   perm[1][3] =  2;
+        perm[2][0] =  0;   perm[2][1] =  2;   perm[2][2] =  1;   perm[2][3] =  3;
+        perm[3][0] =  0;   perm[3][1] =  3;   perm[3][2] =  1;   perm[3][3] =  2;
+        perm[4][0] =  1;   perm[4][1] =  0;   perm[4][2] =  2;   perm[4][3] =  3;
+        perm[5][0] =  1;   perm[5][1] =  0;   perm[5][2] =  3;   perm[5][3] =  2;
+
+        perm[6][0] =  1;   perm[6][1] =  2;   perm[6][2] =  0;   perm[6][3] =  3;
+        perm[7][0] =  1;   perm[7][1] =  3;   perm[7][2] =  0;   perm[7][3] =  2;
+
+        perm[8][0] =  2;   perm[8][1] =  0;   perm[8][2] =  1;   perm[8][3] =  3;
+        perm[9][0] =  3;   perm[9][1] =  0;   perm[9][2] =  1;   perm[9][3] =  2;
+
+        perm[10][0] = 0;   perm[10][1] = 2;   perm[10][2] = 3;   perm[10][3] = 1;
+        perm[11][0] = 0;   perm[11][1] = 3;   perm[11][2] = 2;   perm[11][3] = 1;
+
+        perm[12][0] = 2;   perm[12][1] = 1;   perm[12][2] = 0;   perm[12][3] = 3;
+        perm[13][0] = 3;   perm[13][1] = 1;   perm[13][2] = 0;   perm[13][3] = 2;
+        break;
+      }
+
+      if (dont_cares[0] == 2 && dont_cares[1] == 0 &&
+          dont_cares[2] == 2 && dont_cares[3] == 0) {
+        nperms = 14;
+        perm[0][0] =  0;   perm[0][1] =  1;   perm[0][2] =  2;   perm[0][3] =  3;
+        perm[1][0] =  1;   perm[1][1] =  0;   perm[1][2] =  2;   perm[1][3] =  3;
+        perm[2][0] =  0;   perm[2][1] =  1;   perm[2][2] =  3;   perm[2][3] =  2;
+        perm[3][0] =  1;   perm[3][1] =  0;   perm[3][2] =  3;   perm[3][3] =  2;
+
+        perm[4][0] =  0;   perm[4][1] =  2;   perm[4][2] =  1;   perm[4][3] =  3;
+        perm[5][0] =  1;   perm[5][1] =  2;   perm[5][2] =  0;   perm[5][3] =  3;
+        perm[6][0] =  0;   perm[6][1] =  3;   perm[6][2] =  1;   perm[6][3] =  2;
+        perm[7][0] =  1;   perm[7][1] =  3;   perm[7][2] =  0;   perm[7][3] =  2;
+
+        perm[8][0] = 2;    perm[8][1] = 0;    perm[8][2] = 1;    perm[8][3] = 3;
+        perm[9][0] = 0;    perm[9][1] = 2;    perm[9][2] = 3;    perm[9][3] = 1;
+        perm[10][0] = 2;   perm[10][1] = 1;   perm[10][2] = 0;   perm[10][3] = 3;
+        perm[11][0] = 0;   perm[11][1] = 3;   perm[11][2] = 2;   perm[11][3] = 1;
+        perm[12][0] = 3;   perm[12][1] = 0;   perm[12][2] = 1;   perm[12][3] = 2;
+        perm[13][0] = 1;   perm[13][1] = 2;   perm[13][2] = 3;   perm[13][3] = 0;
+        break;
+      }
+
+      if (dont_cares[0] == 3 && dont_cares[1] == 0 &&
+          dont_cares[2] == 0 && dont_cares[3] == 1) {
+        nperms = 14;
+        perm[0][0] =  0;   perm[0][1] =  1;   perm[0][2] =  2;   perm[0][3] =  3;
+        perm[1][0] =  0;   perm[1][1] =  2;   perm[1][2] =  1;   perm[1][3] =  3;
+        perm[2][0] =  1;   perm[2][1] =  0;   perm[2][2] =  2;   perm[2][3] =  3;
+        perm[3][0] =  2;   perm[3][1] =  0;   perm[3][2] =  1;   perm[3][3] =  3;
+        perm[4][0] =  1;   perm[4][1] =  2;   perm[4][2] =  0;   perm[4][3] =  3;
+        perm[5][0] =  2;   perm[5][1] =  1;   perm[5][2] =  0;   perm[5][3] =  3;
+
+        perm[6][0] =  0;   perm[6][1] =  1;   perm[6][2] =  3;   perm[6][3] =  2;
+        perm[7][0] =  1;   perm[7][1] =  0;   perm[7][2] =  3;   perm[7][3] =  2;
+        perm[8][0] =  0;   perm[8][1] =  2;   perm[8][2] =  3;   perm[8][3] =  1;
+        perm[9][0] =  2;   perm[9][1] =  0;   perm[9][2] =  3;   perm[9][3] =  1;
+        perm[10][0] = 1;   perm[10][1] = 2;   perm[10][2] = 3;   perm[10][3] = 0;
+        perm[11][0] = 2;   perm[11][1] = 1;   perm[11][2] = 3;   perm[11][3] = 0;
+
+        perm[12][0] = 0;   perm[12][1] = 3;   perm[12][2] = 1;   perm[12][3] = 2;
+        perm[13][0] = 0;   perm[13][1] = 3;   perm[13][2] = 2;   perm[13][3] = 1;
+        break;
+      }
+
+      if (dont_cares[0] == 1 && dont_cares[1] == 3 &&
+          dont_cares[2] == 0 && dont_cares[3] == 0) {
+        nperms = 14;
+        perm[0][0] =  0;   perm[0][1] =  1;   perm[0][2] =  2;   perm[0][3] =  3;
+        perm[1][0] =  0;   perm[1][1] =  2;   perm[1][2] =  1;   perm[1][3] =  3;
+        perm[2][0] =  0;   perm[2][1] =  1;   perm[2][2] =  3;   perm[2][3] =  2;
+        perm[3][0] =  0;   perm[3][1] =  2;   perm[3][2] =  3;   perm[3][3] =  1;
+        perm[4][0] =  0;   perm[4][1] =  3;   perm[4][2] =  1;   perm[4][3] =  2;
+        perm[5][0] =  0;   perm[5][1] =  3;   perm[5][2] =  2;   perm[5][3] =  1;
+
+        perm[6][0] =  1;   perm[6][1] =  0;   perm[6][2] =  2;   perm[6][3] =  3;
+        perm[7][0] =  1;   perm[7][1] =  0;   perm[7][2] =  3;   perm[7][3] =  2;
+        perm[8][0] =  2;   perm[8][1] =  0;   perm[8][2] =  1;   perm[8][3] =  3;
+        perm[9][0] =  2;   perm[9][1] =  0;   perm[9][2] =  3;   perm[9][3] =  1;
+        perm[10][0] = 3;   perm[10][1] = 0;   perm[10][2] = 1;   perm[10][3] = 2;
+        perm[11][0] = 3;   perm[11][1] = 0;   perm[11][2] = 2;   perm[11][3] = 1;
+
+        perm[12][0] = 1;   perm[12][1] = 2;   perm[12][2] = 0;   perm[12][3] = 3;
+        perm[13][0] = 2;   perm[13][1] = 1;   perm[13][2] = 0;   perm[13][3] = 3;
+
+        break;
+      }
+
+      nperms = 14;
+      perm[0][0] =  0;   perm[0][1] =  1;   perm[0][2] =  2;   perm[0][3] =  3;
+      perm[1][0] =  1;   perm[1][1] =  0;   perm[1][2] =  2;   perm[1][3] =  3;
+      perm[2][0] =  0;   perm[2][1] =  2;   perm[2][2] =  1;   perm[2][3] =  3;
+      perm[3][0] =  0;   perm[3][1] =  1;   perm[3][2] =  3;   perm[3][3] =  2;
+      perm[4][0] =  1;   perm[4][1] =  0;   perm[4][2] =  3;   perm[4][3] =  2;
+
+      perm[5][0] =  2;   perm[5][1] =  0;   perm[5][2] =  1;   perm[5][3] =  3;
+      perm[6][0] =  0;   perm[6][1] =  2;   perm[6][2] =  3;   perm[6][3] =  1;
+
+      perm[7][0] =  1;   perm[7][1] =  2;   perm[7][2] =  0;   perm[7][3] =  3;
+      perm[8][0] =  0;   perm[8][1] =  3;   perm[8][2] =  1;   perm[8][3] =  2;
+
+      perm[9][0] =  2;   perm[9][1] =  1;   perm[9][2] =  0;   perm[9][3] =  3;
+      perm[10][0] = 0;   perm[10][1] = 3;   perm[10][2] = 2;   perm[10][3] = 1;
+      perm[11][0] = 2;   perm[11][1] = 0;   perm[11][2] = 3;   perm[11][3] = 1;
+
+      perm[12][0] = 3;   perm[12][1] = 0;   perm[12][2] = 1;   perm[12][3] = 2;
+      perm[13][0] = 1;   perm[13][1] = 2;   perm[13][2] = 3;   perm[13][3] = 0;
+      break;
+    /***********************/
+    default:
+      return;
+  }
+
+  for (i=0; i<nperms; i++) {
+    for (j=0; j<ncon; j++)
+      cand[j] = array[perm[i][j]].key;
+
+    for (j=0; j<ncon; j++)
+      rank[cand[j]] = j;
+
+
+    hash = Moc_HashVRank(ncon, rank) - minval;
+    if (currentq[hash+index] > 0) {
+      *qnum = hash;
+      return;
+    }
+  }
+
+  return;
+}
+
+
+/*************************************************************************
+*  This function sorts the nvwgts of a vertex and returns a hashed value
+**************************************************************************/
+int Moc_HashVwgts(int ncon, float *nvwgt)
+{
+  int i;
+  int multiplier, retval;
+  int rank[MAXNCON];
+  KVType array[MAXNCON];
+
+
+  for (i=0; i<ncon; i++) {
+    array[i].key = i;
+    array[i].val = nvwgt[i];
+  }
+
+  qsort(array, ncon, sizeof(KVType), myvalkeycompare);
+  for (i=0; i<ncon; i++)
+    rank[array[i].key] = i;
+
+  multiplier = 1;
+
+  retval = 0;
+  for (i=0; i<ncon; i++) {
+    multiplier *= (i+1);
+    retval += rank[ncon-i-1] * multiplier;
+  }
+
+  return retval;
+}
+
+
+/*************************************************************************
+*  This function sorts the vwgts of a vertex and returns a hashed value
+**************************************************************************/
+int Moc_HashVRank(int ncon, int *vwgt)
+{
+  int i, multiplier, retval;
+
+  multiplier = 1;
+
+  retval = 0;
+  for (i=0; i<ncon; i++) {
+    multiplier *= (i+1);
+    retval += vwgt[ncon-1-i] * multiplier;
+  }
+
+  return retval;
+}
+
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/serial.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/serial.c
new file mode 100644
index 0000000..630d4e5
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/serial.c
@@ -0,0 +1,1251 @@
+/*
+ * serial.c
+ *
+ * This file contains code that implements k-way refinement
+ *
+ * Started 7/28/97
+ * George
+ *
+ * $Id: serial.c,v 1.2 2003/07/21 17:18:53 karypis Exp $
+ *
+ */
+
+#include <parmetislib.h>
+
+
+/*************************************************************************
+* This function performs k-way refinement
+**************************************************************************/
+void Moc_SerialKWayAdaptRefine(GraphType *graph, int nparts, idxtype *home,
+     float *orgubvec, int npasses)
+{
+  int i, ii, iii, j, k;
+  int nvtxs, ncon, pass, nmoves, myndegrees;
+  int from, me, myhome, to, oldcut, gain, tmp;
+  idxtype *xadj, *adjncy, *adjwgt;
+  idxtype *where;
+  EdgeType *mydegrees;
+  RInfoType *rinfo, *myrinfo;
+  float *npwgts, *nvwgt, *minwgt, *maxwgt, ubvec[MAXNCON];
+  int gain_is_greater, gain_is_same, fit_in_to, fit_in_from, going_home;
+  int zero_gain, better_balance_ft, better_balance_tt;
+  KeyValueType *cand;
+int mype;
+MPI_Comm_rank(MPI_COMM_WORLD, &mype);
+
+  nvtxs = graph->nvtxs;
+  ncon = graph->ncon;
+  xadj = graph->xadj;
+  adjncy = graph->adjncy;
+  adjwgt = graph->adjwgt;
+  where = graph->where;
+  rinfo = graph->rinfo;
+  npwgts = graph->gnpwgts;
+  
+  /* Setup the weight intervals of the various subdomains */
+  cand = (KeyValueType *)GKmalloc(nvtxs*sizeof(KeyValueType), "cand");
+  minwgt =  fmalloc(nparts*ncon, "minwgt");
+  maxwgt = fmalloc(nparts*ncon, "maxwgt");
+
+  ComputeHKWayLoadImbalance(ncon, nparts, npwgts, ubvec);
+  for (i=0; i<ncon; i++)
+    ubvec[i] = amax(ubvec[i], orgubvec[i]);
+
+  for (i=0; i<nparts; i++) {
+    for (j=0; j<ncon; j++) {
+      maxwgt[i*ncon+j] = ubvec[j]/(float)nparts;
+      minwgt[i*ncon+j] = ubvec[j]*(float)nparts;
+    }
+  }
+
+  for (pass=0; pass<npasses; pass++) {
+    oldcut = graph->mincut;
+
+    for (i=0; i<nvtxs; i++) {
+      cand[i].key = rinfo[i].id-rinfo[i].ed;
+      cand[i].val = i;
+    }
+    ikeysort(nvtxs, cand);
+
+    nmoves = 0;
+    for (iii=0; iii<nvtxs; iii++) {
+      i = cand[iii].val;
+
+      myrinfo = rinfo+i;
+
+      if (myrinfo->ed >= myrinfo->id) {
+        from = where[i];
+        myhome = home[i];
+        nvwgt = graph->nvwgt+i*ncon;
+
+        if (myrinfo->id > 0 &&
+        AreAllHVwgtsBelow(ncon, 1.0, npwgts+from*ncon, -1.0, nvwgt, minwgt+from*ncon)) 
+          continue;
+
+        mydegrees = myrinfo->degrees;
+        myndegrees = myrinfo->ndegrees;
+
+        for (k=0; k<myndegrees; k++) {
+          to = mydegrees[k].edge;
+          gain = mydegrees[k].ewgt - myrinfo->id; 
+          if (gain >= 0 && 
+             (AreAllHVwgtsBelow(ncon, 1.0, npwgts+to*ncon, 1.0, nvwgt, maxwgt+to*ncon) ||
+             IsHBalanceBetterFT(ncon,npwgts+from*ncon,npwgts+to*ncon,nvwgt,ubvec))) {
+            break;
+          }
+        }
+
+        /* break out if you did not find a candidate */
+        if (k == myndegrees)
+          continue;
+
+        for (j=k+1; j<myndegrees; j++) {
+          to = mydegrees[j].edge;
+          going_home = (myhome == to);
+          gain_is_same = (mydegrees[j].ewgt == mydegrees[k].ewgt);
+          gain_is_greater = (mydegrees[j].ewgt > mydegrees[k].ewgt);
+          fit_in_to = AreAllHVwgtsBelow(ncon,1.0,npwgts+to*ncon,1.0,nvwgt,maxwgt+to*ncon);
+          better_balance_ft = IsHBalanceBetterFT(ncon,npwgts+from*ncon,
+                              npwgts+to*ncon,nvwgt,ubvec);
+          better_balance_tt = IsHBalanceBetterTT(ncon,npwgts+mydegrees[k].edge*ncon,
+                              npwgts+to*ncon,nvwgt,ubvec);
+
+          if (
+               (gain_is_greater &&
+                 (fit_in_to ||
+                  better_balance_ft)
+               )
+            ||
+               (gain_is_same &&
+                 (
+                   (fit_in_to &&
+                    going_home)
+                ||
+                    better_balance_tt
+                 )
+               )
+             ) {
+            k = j;
+          }
+        }
+
+        to = mydegrees[k].edge;
+        going_home = (myhome == to);
+        zero_gain = (mydegrees[k].ewgt == myrinfo->id);
+
+        fit_in_from = AreAllHVwgtsBelow(ncon,1.0,npwgts+from*ncon,0.0,npwgts+from*ncon,
+                      maxwgt+from*ncon);
+        better_balance_ft = IsHBalanceBetterFT(ncon,npwgts+from*ncon,
+                            npwgts+to*ncon,nvwgt,ubvec);
+
+        if (zero_gain &&
+            !going_home &&
+            !better_balance_ft &&
+            fit_in_from)
+          continue;
+
+        /*=====================================================================
+        * If we got here, we can now move the vertex from 'from' to 'to' 
+        *======================================================================*/
+        graph->mincut -= mydegrees[k].ewgt-myrinfo->id;
+
+        /* Update where, weight, and ID/ED information of the vertex you moved */
+        saxpy2(ncon, 1.0, nvwgt, 1, npwgts+to*ncon, 1);
+        saxpy2(ncon, -1.0, nvwgt, 1, npwgts+from*ncon, 1);
+        where[i] = to;
+        myrinfo->ed += myrinfo->id-mydegrees[k].ewgt;
+        SWAP(myrinfo->id, mydegrees[k].ewgt, tmp);
+
+        if (mydegrees[k].ewgt == 0) {
+          myrinfo->ndegrees--;
+          mydegrees[k].edge = mydegrees[myrinfo->ndegrees].edge;
+          mydegrees[k].ewgt = mydegrees[myrinfo->ndegrees].ewgt;
+        }
+        else
+          mydegrees[k].edge = from;
+
+        /* Update the degrees of adjacent vertices */
+        for (j=xadj[i]; j<xadj[i+1]; j++) {
+          ii = adjncy[j];
+          me = where[ii];
+
+          myrinfo = rinfo+ii;
+          mydegrees = myrinfo->degrees;
+
+          if (me == from) {
+            INC_DEC(myrinfo->ed, myrinfo->id, adjwgt[j]);
+          }
+          else {
+            if (me == to) {
+              INC_DEC(myrinfo->id, myrinfo->ed, adjwgt[j]);
+            }
+          }
+
+          /* Remove contribution of the ed from 'from' */
+          if (me != from) {
+            for (k=0; k<myrinfo->ndegrees; k++) {
+              if (mydegrees[k].edge == from) {
+                if (mydegrees[k].ewgt == adjwgt[j]) {
+                  myrinfo->ndegrees--;
+                  mydegrees[k].edge = mydegrees[myrinfo->ndegrees].edge;
+                  mydegrees[k].ewgt = mydegrees[myrinfo->ndegrees].ewgt;
+                }
+                else
+                  mydegrees[k].ewgt -= adjwgt[j];
+                break;
+              }
+            }
+          }
+
+          /* Add contribution of the ed to 'to' */
+          if (me != to) {
+            for (k=0; k<myrinfo->ndegrees; k++) {
+              if (mydegrees[k].edge == to) {
+                mydegrees[k].ewgt += adjwgt[j];
+                break;
+              }
+            }
+            if (k == myrinfo->ndegrees) {
+              mydegrees[myrinfo->ndegrees].edge = to;
+              mydegrees[myrinfo->ndegrees++].ewgt = adjwgt[j];
+            }
+          }
+
+        }
+        nmoves++;
+      }
+    }
+
+    if (graph->mincut == oldcut)
+      break;
+  }
+
+  GKfree((void **)&minwgt, (void **)&maxwgt, (void **)&cand, LTERM);
+
+  return;
+}
+
+
+/*************************************************************************
+* This function computes the initial id/ed
+**************************************************************************/
+void Moc_ComputeSerialPartitionParams(GraphType *graph, int nparts,
+     EdgeType *degrees)
+{
+  int i, j, k;
+  int nvtxs, nedges, ncon, mincut, me, other;
+  idxtype *xadj, *adjncy, *adjwgt, *where;
+  RInfoType *rinfo, *myrinfo;
+  EdgeType *mydegrees;
+  float *nvwgt, *npwgts;
+int mype;
+MPI_Comm_rank(MPI_COMM_WORLD, &mype);
+
+
+  nvtxs = graph->nvtxs;
+  ncon = graph->ncon;
+  xadj = graph->xadj;
+  nvwgt = graph->nvwgt;
+  adjncy = graph->adjncy;
+  adjwgt = graph->adjwgt;
+  where = graph->where;
+  rinfo = graph->rinfo;
+
+  npwgts = sset(ncon*nparts, 0.0, graph->gnpwgts);
+
+  /*------------------------------------------------------------
+  / Compute now the id/ed degrees
+  /------------------------------------------------------------*/
+  nedges = mincut = 0;
+  for (i=0; i<nvtxs; i++) {
+    me = where[i];
+    saxpy2(ncon, 1.0, nvwgt+i*ncon, 1, npwgts+me*ncon, 1);
+
+    myrinfo = rinfo+i;
+    myrinfo->id = myrinfo->ed = myrinfo->ndegrees = 0;
+    myrinfo->degrees = degrees + nedges;
+    nedges += xadj[i+1]-xadj[i];
+
+    for (j=xadj[i]; j<xadj[i+1]; j++) {
+      if (me == where[adjncy[j]]) {
+        myrinfo->id += adjwgt[j];
+      }
+      else {
+        myrinfo->ed += adjwgt[j];
+      }
+    }
+
+    mincut += myrinfo->ed;
+
+    /* Time to compute the particular external degrees */
+    if (myrinfo->ed > 0) {
+      mydegrees = myrinfo->degrees;
+
+      for (j=xadj[i]; j<xadj[i+1]; j++) {
+        other = where[adjncy[j]];
+        if (me != other) {
+          for (k=0; k<myrinfo->ndegrees; k++) {
+            if (mydegrees[k].edge == other) {
+              mydegrees[k].ewgt += adjwgt[j];
+              break;
+            }
+          }
+          if (k == myrinfo->ndegrees) {
+            mydegrees[myrinfo->ndegrees].edge = other;
+            mydegrees[myrinfo->ndegrees++].ewgt = adjwgt[j];
+          }
+        }
+      }
+    }
+  }
+
+  graph->mincut = mincut/2;
+
+  return;
+}
+
+
+/*************************************************************************
+* This function checks if the vertex weights of two vertices are below 
+* a given set of values
+**************************************************************************/
+int AreAllHVwgtsBelow(int ncon, float alpha, float *vwgt1, float beta, float *vwgt2, float *limit)
+{
+  int i;
+
+  for (i=0; i<ncon; i++)
+    if (alpha*vwgt1[i] + beta*vwgt2[i] > limit[i])
+      return 0;
+
+  return 1;
+}
+
+
+/*************************************************************************
+* This function computes the load imbalance over all the constrains
+* For now assume that we just want balanced partitionings
+**************************************************************************/ 
+void ComputeHKWayLoadImbalance(int ncon, int nparts, float *npwgts, float *lbvec)
+{
+  int i, j;
+  float max;
+
+  for (i=0; i<ncon; i++) {
+    max = 0.0;
+    for (j=0; j<nparts; j++) {
+      if (npwgts[j*ncon+i] > max)
+        max = npwgts[j*ncon+i];
+    }
+
+    lbvec[i] = max*nparts;
+  }
+}
+
+
+/**************************************************************
+*  This subroutine remaps a partitioning on a single processor
+**************************************************************/
+void SerialRemap(GraphType *graph, int nparts, idxtype *base, idxtype *scratch,
+     idxtype *remap, float *tpwgts)
+{
+  int i, ii, j, k;
+  int nvtxs, nmapped, max_mult;
+  int from, to, current_from, smallcount, bigcount;
+  KeyValueType *flowto, *bestflow;
+  KeyKeyValueType *sortvtx;
+  idxtype *vsize, *htable, *map, *rowmap;
+
+  nvtxs = graph->nvtxs;
+  vsize = graph->vsize;
+  max_mult = amin(MAX_NPARTS_MULTIPLIER, nparts);
+
+  sortvtx = (KeyKeyValueType *)GKmalloc(nvtxs*sizeof(KeyKeyValueType), "sortvtx");
+  flowto = (KeyValueType *)GKmalloc((nparts*max_mult+nparts)*sizeof(KeyValueType), "flowto");
+  bestflow = flowto+nparts;
+  map = htable = idxsmalloc(nparts*2, -1, "htable");
+  rowmap = map+nparts;
+
+  for (i=0; i<nvtxs; i++) {
+    sortvtx[i].key1 = base[i];
+    sortvtx[i].key2 = vsize[i];
+    sortvtx[i].val = i;
+  }
+
+  qsort((void *)sortvtx, (size_t)nvtxs, (size_t)sizeof(KeyKeyValueType), SSMIncKeyCmp);
+
+  for (j=0; j<nparts; j++) {
+    flowto[j].key = 0;
+    flowto[j].val = j;
+  }
+
+  /* this step has nparts*nparts*log(nparts) computational complexity */
+  bigcount = smallcount = current_from = 0;
+  for (ii=0; ii<nvtxs; ii++) {
+    i = sortvtx[ii].val;
+    from = base[i];
+    to = scratch[i];
+
+    if (from > current_from) {
+      /* reset the hash table */
+      for (j=0; j<smallcount; j++)
+        htable[flowto[j].val] = -1;
+      ASSERTS(idxsum(nparts, htable) == -nparts);
+
+      ikeysort(smallcount, flowto);
+
+      for (j=0; j<amin(smallcount, max_mult); j++, bigcount++) {
+        bestflow[bigcount].key = flowto[j].key;
+        bestflow[bigcount].val = current_from*nparts+flowto[j].val;
+      }
+
+      smallcount = 0;
+      current_from = from;
+    }
+
+    if (htable[to] == -1) {
+      htable[to] = smallcount;
+      flowto[smallcount].key = -vsize[i];
+      flowto[smallcount].val = to;
+      smallcount++;
+    }
+    else {
+      flowto[htable[to]].key += -vsize[i];
+    }
+  }
+
+  /* reset the hash table */
+  for (j=0; j<smallcount; j++)
+    htable[flowto[j].val] = -1;
+  ASSERTS(idxsum(nparts, htable) == -nparts);
+
+  ikeysort(smallcount, flowto);
+
+  for (j=0; j<amin(smallcount, max_mult); j++, bigcount++) {
+    bestflow[bigcount].key = flowto[j].key;
+    bestflow[bigcount].val = current_from*nparts+flowto[j].val;
+  }
+  ikeysort(bigcount, bestflow);
+
+  ASSERTS(idxsum(nparts, map) == -nparts);
+  ASSERTS(idxsum(nparts, rowmap) == -nparts);
+  nmapped = 0;
+
+  /* now make as many assignments as possible */
+  for (ii=0; ii<bigcount; ii++) {
+    i = bestflow[ii].val;
+    j = i % nparts;  /* to */
+    k = i / nparts;  /* from */
+
+    if (map[j] == -1 && rowmap[k] == -1 && SimilarTpwgts(tpwgts, graph->ncon, j, k)) {
+      map[j] = k;
+      rowmap[k] = j;
+      nmapped++;
+    }
+
+    if (nmapped == nparts)
+      break;
+  }
+
+
+  /* remap the rest */
+  /* it may help try remapping to the same label first */
+  if (nmapped < nparts) {
+    for (j=0; j<nparts && nmapped<nparts; j++) {
+      if (map[j] == -1) {
+        for (ii=0; ii<nparts; ii++) {
+          i = (j+ii) % nparts;
+          if (rowmap[i] == -1 && SimilarTpwgts(tpwgts, graph->ncon, i, j)) {
+            map[j] = i;
+            rowmap[i] = j;
+            nmapped++;
+            break;
+          }
+        }
+      }
+    }
+  }
+
+  /* check to see if remapping fails (due to dis-similar tpwgts) */
+  /* if remapping fails, revert to original mapping */
+  if (nmapped < nparts)
+    for (i=0; i<nparts; i++)
+      map[i] = i;
+
+  for (i=0; i<nvtxs; i++)
+    remap[i] = map[remap[i]];
+
+  GKfree((void **)&sortvtx, (void **)&flowto, (void **)&htable, LTERM);
+}
+
+
+/*************************************************************************
+*  This is a comparison function for Serial Remap
+**************************************************************************/
+int SSMIncKeyCmp(const void *fptr, const void *sptr)
+{
+  KeyKeyValueType *first, *second;
+
+  first = (KeyKeyValueType *)(fptr);
+  second = (KeyKeyValueType *)(sptr);
+
+  if (first->key1 > second->key1)
+    return 1;
+
+  if (first->key1 < second->key1)
+     return -1;
+
+  if (first->key2 < second->key2)
+    return 1;
+
+  if (first->key2 > second->key2)
+     return -1;
+
+   return 0;
+}
+
+
+/*************************************************************************
+* This function performs an edge-based FM refinement
+**************************************************************************/
+void Moc_Serial_FM_2WayRefine(GraphType *graph, float *tpwgts, int npasses)
+{
+  int i, ii, j, k;
+  int kwgt, nvtxs, ncon, nbnd, nswaps, from, to, pass, limit, tmp, cnum;
+  idxtype *xadj, *adjncy, *adjwgt, *where, *id, *ed, *bndptr, *bndind;
+  idxtype *moved, *swaps, *qnum;
+  float *nvwgt, *npwgts, mindiff[MAXNCON], origbal, minbal, newbal;
+  FPQueueType parts[MAXNCON][2];
+  int higain, oldgain, mincut, initcut, newcut, mincutorder;
+  float rtpwgts[MAXNCON*2];
+  KeyValueType *cand;
+int mype;
+MPI_Comm_rank(MPI_COMM_WORLD, &mype);
+
+  nvtxs = graph->nvtxs;
+  ncon = graph->ncon;
+  xadj = graph->xadj;
+  nvwgt = graph->nvwgt;
+  adjncy = graph->adjncy;
+  adjwgt = graph->adjwgt;
+  where = graph->where;
+  id = graph->sendind;
+  ed = graph->recvind;
+  npwgts = graph->gnpwgts;
+  bndptr = graph->sendptr;
+  bndind = graph->recvptr;
+
+  moved = idxmalloc(nvtxs, "moved");
+  swaps = idxmalloc(nvtxs, "swaps");
+  qnum = idxmalloc(nvtxs, "qnum");
+  cand = (KeyValueType *)GKmalloc(nvtxs*sizeof(KeyValueType), "cand");
+
+  limit = amin(amax(0.01*nvtxs, 25), 150);
+
+  /* Initialize the queues */
+  for (i=0; i<ncon; i++) {
+    FPQueueInit(&parts[i][0], nvtxs);
+    FPQueueInit(&parts[i][1], nvtxs);
+  }
+  for (i=0; i<nvtxs; i++)
+    qnum[i] = samax(ncon, nvwgt+i*ncon);
+
+  origbal = Serial_Compute2WayHLoadImbalance(ncon, npwgts, tpwgts);
+
+  for (i=0; i<ncon; i++) {
+    rtpwgts[i] = origbal*tpwgts[i];
+    rtpwgts[ncon+i] = origbal*tpwgts[ncon+i];
+  }
+
+  idxset(nvtxs, -1, moved);
+  for (pass=0; pass<npasses; pass++) { /* Do a number of passes */
+    for (i=0; i<ncon; i++) {
+      FPQueueReset(&parts[i][0]);
+      FPQueueReset(&parts[i][1]);
+    }
+
+    mincutorder = -1;
+    newcut = mincut = initcut = graph->mincut;
+    for (i=0; i<ncon; i++)
+      mindiff[i] = fabs(tpwgts[i]-npwgts[i]);
+    minbal = Serial_Compute2WayHLoadImbalance(ncon, npwgts, tpwgts);
+
+    /* Insert boundary nodes in the priority queues */
+    nbnd = graph->gnvtxs;
+
+    for (i=0; i<nbnd; i++) {
+      cand[i].key = id[i]-ed[i];
+      cand[i].val = i;
+    }
+    ikeysort(nbnd, cand);
+
+    for (ii=0; ii<nbnd; ii++) {
+      i = bndind[cand[ii].val];
+      FPQueueInsert(&parts[qnum[i]][where[i]], i, (float)(ed[i]-id[i]));
+    }
+
+    for (nswaps=0; nswaps<nvtxs; nswaps++) {
+      Serial_SelectQueue(ncon, npwgts, rtpwgts, &from, &cnum, parts);
+      to = (from+1)%2;
+
+      if (from == -1 || (higain = FPQueueGetMax(&parts[cnum][from])) == -1)
+        break;
+
+      saxpy2(ncon, 1.0, nvwgt+higain*ncon, 1, npwgts+to*ncon, 1);
+      saxpy2(ncon, -1.0, nvwgt+higain*ncon, 1, npwgts+from*ncon, 1);
+
+      newcut -= (ed[higain]-id[higain]);
+      newbal = Serial_Compute2WayHLoadImbalance(ncon, npwgts, tpwgts);
+
+      if ((newcut < mincut && newbal-origbal <= .00001) ||
+          (newcut == mincut && (newbal < minbal ||
+                                (newbal == minbal && Serial_BetterBalance(ncon, npwgts, tpwgts, mindiff))))) {
+        mincut = newcut;
+        minbal = newbal;
+        mincutorder = nswaps;
+        for (i=0; i<ncon; i++)
+          mindiff[i] = fabs(tpwgts[i]-npwgts[i]);
+      }
+      else if (nswaps-mincutorder > limit) { /* We hit the limit, undo last move */
+        newcut += (ed[higain]-id[higain]);
+        saxpy2(ncon, 1.0, nvwgt+higain*ncon, 1, npwgts+from*ncon, 1);
+        saxpy2(ncon, -1.0, nvwgt+higain*ncon, 1, npwgts+to*ncon, 1);
+        break;
+      }
+
+      where[higain] = to;
+      moved[higain] = nswaps;
+      swaps[nswaps] = higain;
+
+      /**************************************************************
+      * Update the id[i]/ed[i] values of the affected nodes
+      ***************************************************************/
+      SWAP(id[higain], ed[higain], tmp);
+      if (ed[higain] == 0 && xadj[higain] < xadj[higain+1])
+        BNDDelete(nbnd, bndind,  bndptr, higain);
+
+      for (j=xadj[higain]; j<xadj[higain+1]; j++) {
+        k = adjncy[j];
+        oldgain = ed[k]-id[k];
+
+        kwgt = (to == where[k] ? adjwgt[j] : -adjwgt[j]);
+        INC_DEC(id[k], ed[k], kwgt);
+
+        /* Update its boundary information and queue position */
+        if (bndptr[k] != -1) { /* If k was a boundary vertex */
+          if (ed[k] == 0) { /* Not a boundary vertex any more */
+            BNDDelete(nbnd, bndind, bndptr, k);
+            if (moved[k] == -1)  /* Remove it if in the queues */
+              FPQueueDelete(&parts[qnum[k]][where[k]], k);
+          }
+          else { /* If it has not been moved, update its position in the queue */
+            if (moved[k] == -1)
+              FPQueueUpdate(&parts[qnum[k]][where[k]], k, (float)oldgain, (float)(ed[k]-id[k]));
+          }
+        }
+        else {
+          if (ed[k] > 0) {  /* It will now become a boundary vertex */
+            BNDInsert(nbnd, bndind, bndptr, k);
+            if (moved[k] == -1)
+              FPQueueInsert(&parts[qnum[k]][where[k]], k, (float)(ed[k]-id[k]));
+          }
+        }
+      }
+    }
+
+    /****************************************************************
+    * Roll back computations
+    *****************************************************************/
+    for (i=0; i<nswaps; i++)
+      moved[swaps[i]] = -1;  /* reset moved array */
+    for (nswaps--; nswaps>mincutorder; nswaps--) {
+      higain = swaps[nswaps];
+
+      to = where[higain] = (where[higain]+1)%2;
+      SWAP(id[higain], ed[higain], tmp);
+      if (ed[higain] == 0 && bndptr[higain] != -1 && xadj[higain] < xadj[higain+1])
+        BNDDelete(nbnd, bndind,  bndptr, higain);
+      else if (ed[higain] > 0 && bndptr[higain] == -1)
+        BNDInsert(nbnd, bndind,  bndptr, higain);
+
+      saxpy2(ncon, 1.0, nvwgt+higain*ncon, 1, npwgts+to*ncon, 1);
+      saxpy2(ncon, -1.0, nvwgt+higain*ncon, 1, npwgts+((to+1)%2)*ncon, 1);
+      for (j=xadj[higain]; j<xadj[higain+1]; j++) {
+        k = adjncy[j];
+
+        kwgt = (to == where[k] ? adjwgt[j] : -adjwgt[j]);
+        INC_DEC(id[k], ed[k], kwgt);
+
+        if (bndptr[k] != -1 && ed[k] == 0)
+          BNDDelete(nbnd, bndind, bndptr, k);
+        if (bndptr[k] == -1 && ed[k] > 0)
+          BNDInsert(nbnd, bndind, bndptr, k);
+      }
+    }
+
+    graph->mincut = mincut;
+    graph->gnvtxs = nbnd;
+
+    if (mincutorder == -1 || mincut == initcut)
+      break;
+  }
+
+  for (i=0; i<ncon; i++) {
+    FPQueueFree(&parts[i][0]);
+    FPQueueFree(&parts[i][1]);
+  }
+
+  GKfree((void **)&cand, (void **)&qnum, (void **)&moved, (void **)&swaps, LTERM);
+  return;
+}
+
+/*************************************************************************
+* This function selects the partition number and the queue from which
+* we will move vertices out
+**************************************************************************/
+void Serial_SelectQueue(int ncon, float *npwgts, float *tpwgts, int *from, int *cnum,
+     FPQueueType queues[MAXNCON][2])
+{
+  int i, part;
+  float maxgain=0.0;
+  float max = -1.0, maxdiff=0.0;
+int mype;
+MPI_Comm_rank(MPI_COMM_WORLD, &mype);
+
+  *from = -1;
+  *cnum = -1;
+
+  /* First determine the side and the queue, irrespective of the presence of nodes */
+  for (part=0; part<2; part++) {
+    for (i=0; i<ncon; i++) {
+      if (npwgts[part*ncon+i]-tpwgts[part*ncon+i] >= maxdiff) {
+        maxdiff = npwgts[part*ncon+i]-tpwgts[part*ncon+i];
+        *from = part;
+        *cnum = i;
+      }
+    }
+  }
+
+  if (*from != -1 && FPQueueGetQSize(&queues[*cnum][*from]) == 0) {
+    /* The desired queue is empty, select a node from that side anyway */
+    for (i=0; i<ncon; i++) {
+      if (FPQueueGetQSize(&queues[i][*from]) > 0) {
+        max = npwgts[(*from)*ncon + i];
+        *cnum = i;
+        break;
+      }
+    }
+
+    for (i++; i<ncon; i++) {
+      if (npwgts[(*from)*ncon + i] > max && FPQueueGetQSize(&queues[i][*from]) > 0) {
+        max = npwgts[(*from)*ncon + i];
+        *cnum = i;
+      }
+    }
+  }
+
+
+  /* Check to see if you can focus on the cut */
+  if (maxdiff <= 0.0 || *from == -1) {
+    maxgain = -100000.0;
+
+    for (part=0; part<2; part++) {
+      for (i=0; i<ncon; i++) {
+        if (FPQueueGetQSize(&queues[i][part]) > 0 &&
+            FPQueueSeeMaxGain(&queues[i][part]) > maxgain) {
+          maxgain = FPQueueSeeMaxGain(&queues[i][part]);
+          *from = part;
+          *cnum = i;
+        }
+      }
+    }
+  }
+
+  return;
+}
+
+/*************************************************************************
+* This function checks if the balance achieved is better than the diff
+* For now, it uses a 2-norm measure
+**************************************************************************/
+int Serial_BetterBalance(int ncon, float *npwgts, float *tpwgts, float *diff)
+{
+  int i;
+  float ndiff[MAXNCON];
+
+  for (i=0; i<ncon; i++)
+    ndiff[i] = fabs(tpwgts[i]-npwgts[i]);
+
+  return snorm2(ncon, ndiff) < snorm2(ncon, diff);
+}
+
+
+
+/*************************************************************************
+* This function computes the load imbalance over all the constrains
+**************************************************************************/
+float Serial_Compute2WayHLoadImbalance(int ncon, float *npwgts, float *tpwgts)
+{
+  int i;
+  float max=0.0, temp;
+
+  for (i=0; i<ncon; i++) {
+    if (tpwgts[i] == 0.0)
+      temp = 0.0;
+    else
+      temp = fabs(tpwgts[i]-npwgts[i])/tpwgts[i];
+    max = (max < temp ? temp : max);
+  }
+  return 1.0+max;
+}
+
+
+
+/*************************************************************************
+* This function performs an edge-based FM refinement
+**************************************************************************/
+void Moc_Serial_Balance2Way(GraphType *graph, float *tpwgts, float lbfactor)
+{
+  int i, ii, j, k, kwgt, nvtxs, ncon, nbnd, nswaps, from, to, limit, tmp, cnum;
+  idxtype *xadj, *adjncy, *adjwgt, *where, *id, *ed, *bndptr, *bndind;
+  idxtype *moved, *swaps, *qnum;
+  float *nvwgt, *npwgts, mindiff[MAXNCON], origbal, minbal, newbal;
+  FPQueueType parts[MAXNCON][2];
+  int higain, oldgain, mincut, newcut, mincutorder;
+  int qsizes[MAXNCON][2];
+  KeyValueType *cand;
+
+  nvtxs = graph->nvtxs;
+  ncon = graph->ncon;
+  xadj = graph->xadj;
+  nvwgt = graph->nvwgt;
+  adjncy = graph->adjncy;
+  adjwgt = graph->adjwgt;
+  where = graph->where;
+  id = graph->sendind;
+  ed = graph->recvind;
+  npwgts = graph->gnpwgts;
+  bndptr = graph->sendptr;
+  bndind = graph->recvptr;
+
+  moved = idxmalloc(nvtxs, "moved");
+  swaps = idxmalloc(nvtxs, "swaps");
+  qnum = idxmalloc(nvtxs, "qnum");
+  cand = (KeyValueType *)GKmalloc(nvtxs*sizeof(KeyValueType), "cand");
+
+
+  limit = amin(amax(0.01*nvtxs, 15), 100);
+
+  /* Initialize the queues */
+  for (i=0; i<ncon; i++) {
+    FPQueueInit(&parts[i][0], nvtxs);
+    FPQueueInit(&parts[i][1], nvtxs);
+    qsizes[i][0] = qsizes[i][1] = 0;
+  }
+
+  for (i=0; i<nvtxs; i++) {
+    qnum[i] = samax(ncon, nvwgt+i*ncon);
+    qsizes[qnum[i]][where[i]]++;
+  }
+
+  for (from=0; from<2; from++) {
+    for (j=0; j<ncon; j++) {
+      if (qsizes[j][from] == 0) {
+        for (i=0; i<nvtxs; i++) {
+          if (where[i] != from)
+            continue;
+
+          k = samax2(ncon, nvwgt+i*ncon);
+          if (k == j &&
+               qsizes[qnum[i]][from] > qsizes[j][from] &&
+               nvwgt[i*ncon+qnum[i]] < 1.3*nvwgt[i*ncon+j]) {
+            qsizes[qnum[i]][from]--;
+            qsizes[j][from]++;
+            qnum[i] = j;
+          }
+        }
+      }
+    }
+  }
+
+
+  for (i=0; i<ncon; i++)
+    mindiff[i] = fabs(tpwgts[i]-npwgts[i]);
+  minbal = origbal = Serial_Compute2WayHLoadImbalance(ncon, npwgts, tpwgts);
+  newcut = mincut = graph->mincut;
+  mincutorder = -1;
+
+  idxset(nvtxs, -1, moved);
+
+  /* Insert all nodes in the priority queues */
+  nbnd = graph->gnvtxs;
+  for (i=0; i<nvtxs; i++) {
+    cand[i].key = id[i]-ed[i];
+    cand[i].val = i;
+  }
+  ikeysort(nvtxs, cand);
+
+  for (ii=0; ii<nvtxs; ii++) {
+    i = cand[ii].val;
+    FPQueueInsert(&parts[qnum[i]][where[i]], i, (float)(ed[i]-id[i]));
+  }
+
+  for (nswaps=0; nswaps<nvtxs; nswaps++) {
+    if (minbal < lbfactor)
+      break;
+
+    Serial_SelectQueue(ncon, npwgts, tpwgts, &from, &cnum, parts);
+    to = (from+1)%2;
+
+    if (from == -1 || (higain = FPQueueGetMax(&parts[cnum][from])) == -1)
+      break;
+
+    saxpy2(ncon, 1.0, nvwgt+higain*ncon, 1, npwgts+to*ncon, 1);
+    saxpy2(ncon, -1.0, nvwgt+higain*ncon, 1, npwgts+from*ncon, 1);
+    newcut -= (ed[higain]-id[higain]);
+    newbal = Serial_Compute2WayHLoadImbalance(ncon, npwgts, tpwgts);
+
+    if (newbal < minbal || (newbal == minbal &&
+        (newcut < mincut || (newcut == mincut &&
+          Serial_BetterBalance(ncon, npwgts, tpwgts, mindiff))))) {
+      mincut = newcut;
+      minbal = newbal;
+      mincutorder = nswaps;
+      for (i=0; i<ncon; i++)
+        mindiff[i] = fabs(tpwgts[i]-npwgts[i]);
+    }
+    else if (nswaps-mincutorder > limit) { /* We hit the limit, undo last move */
+      newcut += (ed[higain]-id[higain]);
+      saxpy2(ncon, 1.0, nvwgt+higain*ncon, 1, npwgts+from*ncon, 1);
+      saxpy2(ncon, -1.0, nvwgt+higain*ncon, 1, npwgts+to*ncon, 1);
+      break;
+    }
+
+    where[higain] = to;
+    moved[higain] = nswaps;
+    swaps[nswaps] = higain;
+
+    /**************************************************************
+    * Update the id[i]/ed[i] values of the affected nodes
+    ***************************************************************/
+    SWAP(id[higain], ed[higain], tmp);
+    if (ed[higain] == 0 && bndptr[higain] != -1 && xadj[higain] < xadj[higain+1])
+      BNDDelete(nbnd, bndind,  bndptr, higain);
+    if (ed[higain] > 0 && bndptr[higain] == -1)
+      BNDInsert(nbnd, bndind,  bndptr, higain);
+
+    for (j=xadj[higain]; j<xadj[higain+1]; j++) {
+      k = adjncy[j];
+      oldgain = ed[k]-id[k];
+
+      kwgt = (to == where[k] ? adjwgt[j] : -adjwgt[j]);
+      INC_DEC(id[k], ed[k], kwgt);
+
+      /* Update the queue position */
+      if (moved[k] == -1)
+        FPQueueUpdate(&parts[qnum[k]][where[k]], k, (float)(oldgain), (float)(ed[k]-id[k]));
+
+      /* Update its boundary information */
+      if (ed[k] == 0 && bndptr[k] != -1)
+        BNDDelete(nbnd, bndind, bndptr, k);
+      else if (ed[k] > 0 && bndptr[k] == -1)
+        BNDInsert(nbnd, bndind, bndptr, k);
+    }
+  }
+
+
+  /****************************************************************
+  * Roll back computations
+  *****************************************************************/
+  for (nswaps--; nswaps>mincutorder; nswaps--) {
+    higain = swaps[nswaps];
+
+    to = where[higain] = (where[higain]+1)%2;
+    SWAP(id[higain], ed[higain], tmp);
+    if (ed[higain] == 0 && bndptr[higain] != -1 && xadj[higain] < xadj[higain+1])
+      BNDDelete(nbnd, bndind,  bndptr, higain);
+    else if (ed[higain] > 0 && bndptr[higain] == -1)
+      BNDInsert(nbnd, bndind,  bndptr, higain);
+
+    saxpy2(ncon, 1.0, nvwgt+higain*ncon, 1, npwgts+to*ncon, 1);
+    saxpy2(ncon, -1.0, nvwgt+higain*ncon, 1, npwgts+((to+1)%2)*ncon, 1);
+    for (j=xadj[higain]; j<xadj[higain+1]; j++) {
+      k = adjncy[j];
+
+      kwgt = (to == where[k] ? adjwgt[j] : -adjwgt[j]);
+      INC_DEC(id[k], ed[k], kwgt);
+
+      if (bndptr[k] != -1 && ed[k] == 0)
+        BNDDelete(nbnd, bndind, bndptr, k);
+      if (bndptr[k] == -1 && ed[k] > 0)
+        BNDInsert(nbnd, bndind, bndptr, k);
+    }
+  }
+
+  graph->mincut = mincut;
+  graph->gnvtxs = nbnd;
+
+
+  for (i=0; i<ncon; i++) {
+    FPQueueFree(&parts[i][0]);
+    FPQueueFree(&parts[i][1]);
+  }
+
+  GKfree((void **)&cand, (void **)&qnum, (void **)&moved, (void **)&swaps, LTERM);
+  return;
+}
+
+/*************************************************************************
+* This function balances two partitions by moving the highest gain
+* (including negative gain) vertices to the other domain.
+* It is used only when tha unbalance is due to non contigous
+* subdomains. That is, the are no boundary vertices.
+* It moves vertices from the domain that is overweight to the one that
+* is underweight.
+**************************************************************************/
+void Moc_Serial_Init2WayBalance(GraphType *graph, float *tpwgts)
+{
+  int i, ii, j, k;
+  int kwgt, nvtxs, nbnd, ncon, nswaps, from, to, cnum, tmp;
+  idxtype *xadj, *adjncy, *adjwgt, *where, *id, *ed, *bndptr, *bndind;
+  idxtype *qnum;
+  float *nvwgt, *npwgts;
+  FPQueueType parts[MAXNCON][2];
+  int higain, oldgain, mincut;
+  KeyValueType *cand;
+
+  nvtxs = graph->nvtxs;
+  ncon = graph->ncon;
+  xadj = graph->xadj;
+  adjncy = graph->adjncy;
+  nvwgt = graph->nvwgt;
+  adjwgt = graph->adjwgt;
+  where = graph->where;
+  id = graph->sendind;
+  ed = graph->recvind;
+  npwgts = graph->gnpwgts;
+  bndptr = graph->sendptr;
+  bndind = graph->recvptr;
+
+  qnum = idxmalloc(nvtxs, "qnum");
+  cand = (KeyValueType *)GKmalloc(nvtxs*sizeof(KeyValueType), "cand");
+
+  /* This is called for initial partitioning so we know from where to pick nodes */
+  from = 1;
+  to = (from+1)%2;
+
+  for (i=0; i<ncon; i++) {
+    FPQueueInit(&parts[i][0], nvtxs);
+    FPQueueInit(&parts[i][1], nvtxs);
+  }
+
+  /* Compute the queues in which each vertex will be assigned to */
+  for (i=0; i<nvtxs; i++)
+    qnum[i] = samax(ncon, nvwgt+i*ncon);
+
+  for (i=0; i<nvtxs; i++) {
+    cand[i].key = id[i]-ed[i];
+    cand[i].val = i;
+  }
+  ikeysort(nvtxs, cand);
+
+  /* Insert the nodes of the proper partition in the appropriate priority queue */
+  for (ii=0; ii<nvtxs; ii++) {
+    i = cand[ii].val;
+    if (where[i] == from) {
+      if (ed[i] > 0)
+        FPQueueInsert(&parts[qnum[i]][0], i, (float)(ed[i]-id[i]));
+      else
+        FPQueueInsert(&parts[qnum[i]][1], i, (float)(ed[i]-id[i]));
+    }
+  }
+
+  mincut = graph->mincut;
+  nbnd = graph->gnvtxs;
+  for (nswaps=0; nswaps<nvtxs; nswaps++) {
+    if (Serial_AreAnyVwgtsBelow(ncon, 1.0, npwgts+from*ncon, 0.0, nvwgt, tpwgts+from*ncon))
+      break;
+
+    if ((cnum = Serial_SelectQueueOneWay(ncon, npwgts, tpwgts, from, parts)) == -1)
+      break;
+
+
+    if ((higain = FPQueueGetMax(&parts[cnum][0])) == -1)
+      higain = FPQueueGetMax(&parts[cnum][1]);
+
+    mincut -= (ed[higain]-id[higain]);
+    saxpy2(ncon, 1.0, nvwgt+higain*ncon, 1, npwgts+to*ncon, 1);
+    saxpy2(ncon, -1.0, nvwgt+higain*ncon, 1, npwgts+from*ncon, 1);
+
+    where[higain] = to;
+
+    /**************************************************************
+    * Update the id[i]/ed[i] values of the affected nodes
+    ***************************************************************/
+    SWAP(id[higain], ed[higain], tmp);
+    if (ed[higain] == 0 && bndptr[higain] != -1 && xadj[higain] < xadj[higain+1])
+      BNDDelete(nbnd, bndind,  bndptr, higain);
+    if (ed[higain] > 0 && bndptr[higain] == -1)
+      BNDInsert(nbnd, bndind,  bndptr, higain);
+
+    for (j=xadj[higain]; j<xadj[higain+1]; j++) {
+      k = adjncy[j];
+      oldgain = ed[k]-id[k];
+
+      kwgt = (to == where[k] ? adjwgt[j] : -adjwgt[j]);
+      INC_DEC(id[k], ed[k], kwgt);
+
+      /* Update the queue position */
+      if (where[k] == from) {
+        if (ed[k] > 0 && bndptr[k] == -1) {  /* It moves in boundary */
+          FPQueueDelete(&parts[qnum[k]][1], k);
+          FPQueueInsert(&parts[qnum[k]][0], k, (float)(ed[k]-id[k]));
+        }
+        else { /* It must be in the boundary already */
+          FPQueueUpdate(&parts[qnum[k]][0], k, (float)(oldgain), (float)(ed[k]-id[k]));
+        }
+      }
+
+      /* Update its boundary information */
+      if (ed[k] == 0 && bndptr[k] != -1)
+        BNDDelete(nbnd, bndind, bndptr, k);
+      else if (ed[k] > 0 && bndptr[k] == -1)
+        BNDInsert(nbnd, bndind, bndptr, k);
+    }
+  }
+
+  graph->mincut = mincut;
+  graph->gnvtxs = nbnd;
+
+  for (i=0; i<ncon; i++) {
+    FPQueueFree(&parts[i][0]);
+    FPQueueFree(&parts[i][1]);
+  }
+
+  GKfree((void **)&cand, (void **)&qnum, LTERM);
+}
+
+
+/*************************************************************************
+* This function selects the partition number and the queue from which
+* we will move vertices out
+**************************************************************************/
+int Serial_SelectQueueOneWay(int ncon, float *npwgts, float *tpwgts, int from,
+    FPQueueType queues[MAXNCON][2])
+{
+  int i, cnum=-1;
+  float max=0.0;
+
+  for (i=0; i<ncon; i++) {
+    if (npwgts[from*ncon+i]-tpwgts[from*ncon+i] >= max &&
+        FPQueueGetQSize(&queues[i][0]) + FPQueueGetQSize(&queues[i][1]) > 0) {
+      max = npwgts[from*ncon+i]-tpwgts[i];
+      cnum = i;
+    }
+  }
+
+  return cnum;
+}
+
+
+/*************************************************************************
+* This function computes the initial id/ed
+**************************************************************************/
+void Moc_Serial_Compute2WayPartitionParams(GraphType *graph)
+{
+  int i, j, me, nvtxs, ncon, nbnd, mincut;
+  idxtype *xadj, *adjncy, *adjwgt;
+  float *nvwgt, *npwgts;
+  idxtype *id, *ed, *where;
+  idxtype *bndptr, *bndind;
+
+  nvtxs = graph->nvtxs;
+  ncon = graph->ncon;
+  xadj = graph->xadj;
+  nvwgt = graph->nvwgt;
+  adjncy = graph->adjncy;
+  adjwgt = graph->adjwgt;
+  where = graph->where;
+
+  npwgts = sset(2*ncon, 0.0, graph->gnpwgts);
+  id = idxset(nvtxs, 0, graph->sendind);
+  ed = idxset(nvtxs, 0, graph->recvind);
+  bndptr = idxset(nvtxs, -1, graph->sendptr);
+  bndind = graph->recvptr;
+
+  /*------------------------------------------------------------
+  / Compute now the id/ed degrees
+  /------------------------------------------------------------*/
+  nbnd = mincut = 0;
+  for (i=0; i<nvtxs; i++) {
+    me = where[i];
+    saxpy2(ncon, 1.0, nvwgt+i*ncon, 1, npwgts+me*ncon, 1);
+
+    for (j=xadj[i]; j<xadj[i+1]; j++) {
+      if (me == where[adjncy[j]])
+        id[i] += adjwgt[j];
+      else
+        ed[i] += adjwgt[j];
+    }
+
+    if (ed[i] > 0 || xadj[i] == xadj[i+1]) {
+      mincut += ed[i];
+      bndptr[i] = nbnd;
+      bndind[nbnd++] = i;
+    }
+  }
+
+  graph->mincut = mincut/2;
+  graph->gnvtxs = nbnd;
+
+}
+
+/*************************************************************************
+* This function checks if the vertex weights of two vertices are below
+* a given set of values
+**************************************************************************/
+int Serial_AreAnyVwgtsBelow(int ncon, float alpha, float *vwgt1, float beta, float *vwgt2, float *limit)
+{
+  int i;
+
+  for (i=0; i<ncon; i++)
+    if (alpha*vwgt1[i] + beta*vwgt2[i] < limit[i])
+      return 1;
+
+  return 0;
+}
+
+
+/*************************************************************************
+*  This function computes the edge-cut of a serial graph.
+**************************************************************************/
+int ComputeSerialEdgeCut(GraphType *graph)
+{
+  int i, j;
+  int cut = 0;
+
+  for (i=0; i<graph->nvtxs; i++) {
+    for (j=graph->xadj[i]; j<graph->xadj[i+1]; j++)
+      if (graph->where[i] != graph->where[graph->adjncy[j]])
+        cut += graph->adjwgt[j];
+  }
+  graph->mincut = cut/2;
+
+  return graph->mincut;
+}
+
+/*************************************************************************
+*  This function computes the TotalV of a serial graph.
+**************************************************************************/
+int ComputeSerialTotalV(GraphType *graph, idxtype *home)
+{
+  int i;
+  int totalv = 0;
+
+  for (i=0; i<graph->nvtxs; i++)
+    if (graph->where[i] != home[i])
+      totalv += (graph->vsize == NULL) ? graph->vwgt[i] : graph->vsize[i];
+
+  return totalv;
+}
+
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/setup.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/setup.c
new file mode 100644
index 0000000..682a31d
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/setup.c
@@ -0,0 +1,219 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * setup.c
+ *
+ * This file contains functions that setup the various communication
+ * data structures for parallel KWAY
+ *
+ * Started 2/21/96
+ * George
+ *
+ * $Id: setup.c,v 1.3 2003/07/31 16:23:30 karypis Exp $
+ *
+ */
+
+
+#include <parmetislib.h>
+
+#define DEBUG_SETUPINFO_
+
+
+/*************************************************************************
+* This function tests the repeated shmem_put
+**************************************************************************/
+void SetUp(CtrlType *ctrl, GraphType *graph, WorkSpaceType *wspace)
+{
+  int i, j, k, islocal, penum, gnvtxs, nvtxs, nlocal, firstvtx, lastvtx, nsend, nrecv, nnbrs, nadj;
+  int npes=ctrl->npes, mype=ctrl->mype;
+  idxtype *vtxdist, *xadj, *adjncy;
+  idxtype *peind, *recvptr, *recvind, *sendptr, *sendind;
+  idxtype *receive, *pemap, *imap, *lperm;
+  idxtype *pexadj, *peadjncy, *peadjloc, *startsind;
+  KeyValueType *recvrequests, *sendrequests, *adjpairs;
+
+  IFSET(ctrl->dbglvl, DBG_TIME, MPI_Barrier(ctrl->comm));
+  IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->SetupTmr));
+
+  gnvtxs  = graph->gnvtxs;
+  nvtxs   = graph->nvtxs;
+  vtxdist = graph->vtxdist;
+  xadj    = graph->xadj;
+  adjncy  = graph->adjncy;
+
+  firstvtx = vtxdist[mype];
+  lastvtx = vtxdist[mype+1];
+
+  pemap = wspace->pv1;
+  idxset(npes, -1, pemap);
+
+  lperm = graph->lperm = idxmalloc(nvtxs, "SetUp: graph->lperm");
+  for (i=0; i<nvtxs; i++)
+    lperm[i] = i;
+
+  /************************************************************* 
+   * Determine what you need to receive 
+   *************************************************************/
+  receive  = wspace->indices;  		/* Use the large global received array for now */
+  adjpairs = wspace->pairs;
+
+  for (nlocal = nadj = i = 0; i<nvtxs; i++) {
+    islocal = 1;
+    for (j=xadj[i]; j<xadj[i+1]; j++) {
+      k = adjncy[j];
+      if (k >= firstvtx && k < lastvtx) {
+        adjncy[j] = k-firstvtx;
+        continue;  /* local vertex */
+      }
+      adjpairs[nadj].key = k;
+      adjpairs[nadj++].val = j;
+      islocal = 0;
+    }
+    if (islocal) {
+      lperm[i] = lperm[nlocal];
+      lperm[nlocal++] = i;
+    }
+  }
+
+  /* Take care the received part now */
+  ikeysort(nadj, adjpairs);
+  adjpairs[nadj].key = gnvtxs+1;  /* Boundary condition */
+  for (nrecv=i=0; i<nadj; i++) {
+    adjncy[adjpairs[i].val] = nvtxs+nrecv;
+    if (adjpairs[i].key != adjpairs[i+1].key)
+      receive[nrecv++] = adjpairs[i].key;
+  }
+
+
+  /* Allocate space for the setup info attached to this level of the graph */
+  peind = graph->peind = idxmalloc(npes, "SetUp: peind");
+  recvptr = graph->recvptr = idxmalloc(npes+1, "SetUp: recvptr");
+  recvind = graph->recvind = idxmalloc(nrecv, "SetUp: recvind");
+
+  /* Take care of the received portion */
+  idxcopy(nrecv, receive, recvind);  /* Copy the vertices to be received into recvind */
+
+  i = nnbrs = recvptr[0] = 0;
+  for (penum=0; penum<npes; penum++) {
+    for (j=i; j<nrecv; j++) {
+      if (recvind[j] >= vtxdist[penum+1])
+        break;
+    }
+    if (j > i) {
+      peind[nnbrs] = penum;
+      recvptr[++nnbrs] = j;
+      i = j;
+    }
+  }
+
+
+  /************************************************************* 
+   * Determine what you need to send 
+   *************************************************************/
+  /* Tell the other processors what they need to send you */
+  recvrequests = wspace->pepairs1;
+  sendrequests = wspace->pepairs2;
+  for (i=0; i<npes; i++)
+    recvrequests[i].key = 0;
+  for (i=0; i<nnbrs; i++) {
+    recvrequests[peind[i]].key = recvptr[i+1]-recvptr[i];
+    recvrequests[peind[i]].val = nvtxs+recvptr[i];
+  }
+  MPI_Alltoall((void *)recvrequests, 2, IDX_DATATYPE, (void *)sendrequests, 2, IDX_DATATYPE, ctrl->comm);
+
+
+  sendptr = graph->sendptr = idxmalloc(npes+1, "SetUp: sendptr");
+  startsind = wspace->pv2;
+  for (j=i=0; i<npes; i++) {
+    if (sendrequests[i].key > 0) {
+      sendptr[j] = sendrequests[i].key;
+      startsind[j] = sendrequests[i].val;
+      j++;
+    }
+  }
+  ASSERT(ctrl, nnbrs == j);
+  MAKECSR(i, j, sendptr);
+
+  nsend = sendptr[nnbrs];
+  sendind = graph->sendind = idxmalloc(nsend, "SetUp: sendind");
+
+
+  /* Issue the receives for sendind */
+  for (i=0; i<nnbrs; i++) {
+    MPI_Irecv((void *)(sendind+sendptr[i]), sendptr[i+1]-sendptr[i], IDX_DATATYPE, 
+              peind[i], 1, ctrl->comm, ctrl->rreq+i);
+  }
+
+  /* Issue the sends. My recvind[penum] becomes penum's sendind[mype] */
+  for (i=0; i<nnbrs; i++) {
+    MPI_Isend((void *)(recvind+recvptr[i]), recvptr[i+1]-recvptr[i], IDX_DATATYPE,
+              peind[i], 1, ctrl->comm, ctrl->sreq+i);
+  }
+
+  MPI_Waitall(nnbrs, ctrl->rreq, ctrl->statuses);
+  MPI_Waitall(nnbrs, ctrl->sreq, ctrl->statuses);
+
+
+
+  /* Create the peadjncy data structure for sparse boundary exchanges */
+  pexadj = graph->pexadj = idxsmalloc(nvtxs+1, 0, "SetUp: pexadj");
+  peadjncy = graph->peadjncy = idxmalloc(nsend, "SetUp: peadjncy");
+  peadjloc = graph->peadjloc = idxmalloc(nsend, "SetUp: peadjloc");
+
+  for (i=0; i<nsend; i++) {
+    ASSERTP(ctrl, sendind[i] >= firstvtx && sendind[i] < lastvtx, (ctrl, "%d %d %d\n", sendind[i], firstvtx, lastvtx));
+    pexadj[sendind[i]-firstvtx]++;
+  }
+  MAKECSR(i, nvtxs, pexadj);
+
+  for (i=0; i<nnbrs; i++) {
+    for (j=sendptr[i]; j<sendptr[i+1]; j++) {
+      k = pexadj[sendind[j]-firstvtx]++;
+      peadjncy[k] = i;  /* peind[i] is the actual PE number */
+      peadjloc[k] = startsind[i]++;
+    }
+  }
+  ASSERT(ctrl, pexadj[nvtxs] == nsend);
+
+  for (i=nvtxs; i>0; i--)
+    pexadj[i] = pexadj[i-1];
+  pexadj[0] = 0;
+
+
+  graph->nnbrs = nnbrs;
+  graph->nrecv = nrecv;
+  graph->nsend = nsend;
+  graph->nlocal = nlocal;
+
+
+  /* Create the inverse map from ladjncy to adjncy */
+  imap = graph->imap = idxmalloc(nvtxs+nrecv, "SetUp: imap");
+  for (i=0; i<nvtxs; i++)
+    imap[i] = firstvtx+i;
+  for (i=0; i<nrecv; i++)
+    imap[nvtxs+i] = recvind[i];
+
+
+  /* Check if wspace->nlarge is large enough for nrecv and nsend */
+  if (wspace->nlarge < nrecv+nsend) {
+    free(wspace->indices);
+    free(wspace->pairs);
+    wspace->nlarge = nrecv+nsend;
+    wspace->indices = idxmalloc(wspace->nlarge, "SetUp: wspace->indices");
+    wspace->pairs = (KeyValueType *)GKmalloc(sizeof(KeyValueType)*wspace->nlarge, "SetUp: wspace->pairs");
+  }
+
+  IFSET(ctrl->dbglvl, DBG_TIME, stoptimer(ctrl->SetupTmr));
+
+#ifdef DEBUG_SETUPINFO
+  rprintf(ctrl, "[%5d %5d] \tl:[%5d %5d] \ts:[%5d, %5d] \tr:[%5d, %5d]\n", 
+            GlobalSEMin(ctrl, nvtxs), GlobalSEMax(ctrl, nvtxs),
+            GlobalSEMin(ctrl, nlocal), GlobalSEMax(ctrl, nlocal),
+            GlobalSEMin(ctrl, nsend), GlobalSEMax(ctrl, nsend),
+            GlobalSEMin(ctrl, nrecv), GlobalSEMax(ctrl, nrecv));
+
+  PrintSetUpInfo(ctrl, graph);
+#endif
+}
+
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/stat.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/stat.c
new file mode 100644
index 0000000..4a32bc8
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/stat.c
@@ -0,0 +1,332 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * stat.c
+ *
+ * This file computes various statistics
+ *
+ * Started 7/25/97
+ * George
+ *
+ * $Id: stat.c,v 1.3 2003/07/23 00:54:56 karypis Exp $
+ *
+ */
+
+#include <parmetislib.h>
+
+
+
+/*************************************************************************
+* This function computes the balance of the partitioning
+**************************************************************************/
+void Moc_ComputeSerialBalance(CtrlType *ctrl, GraphType *graph, idxtype *where, float *ubvec)
+{
+  int i, j, nvtxs, ncon, nparts;
+  idxtype *pwgts, *tvwgts, *vwgt;
+  float *tpwgts, maximb;
+
+  nvtxs  = graph->nvtxs;
+  ncon   = graph->ncon;
+  vwgt   = graph->vwgt;
+  nparts = ctrl->nparts;
+  tpwgts = ctrl->tpwgts;
+
+  pwgts = idxsmalloc(nparts*ncon, 0, "pwgts");
+  tvwgts = idxsmalloc(ncon, 0, "tvwgts");
+
+  for (i=0; i<graph->nvtxs; i++) {
+    for (j=0; j<ncon; j++) {
+      pwgts[where[i]*ncon+j] += vwgt[i*ncon+j];
+      tvwgts[j] += vwgt[i*ncon+j];
+    }
+  }
+
+  /* The +1 in the following code is to deal with bad cases of tpwgts[i*ncon+j] == 0 */
+  for (j=0; j<ncon; j++) {
+    maximb = 0.0;
+    for (i=0; i<nparts; i++)
+      maximb = amax(maximb, (1.0+(float)pwgts[i*ncon+j])/(1.0+(tpwgts[i*ncon+j]*(float)tvwgts[j])));
+    ubvec[j] = maximb;
+  }
+
+  GKfree((void **)&pwgts, (void **)&tvwgts, LTERM);
+}
+
+
+/*************************************************************************
+* This function computes the balance of the partitioning
+**************************************************************************/
+void Moc_ComputeParallelBalance(CtrlType *ctrl, GraphType *graph, idxtype *where, float *ubvec)
+{
+  int i, j, nvtxs, ncon, nparts;
+  float *nvwgt, *lnpwgts, *gnpwgts;
+  float *tpwgts, maximb;
+  float lminvwgts[MAXNCON], gminvwgts[MAXNCON];
+
+  ncon   = graph->ncon;
+  nvtxs  = graph->nvtxs;
+  nvwgt  = graph->nvwgt;
+  nparts = ctrl->nparts;
+  tpwgts = ctrl->tpwgts;
+
+  lnpwgts = fmalloc(nparts*ncon, "CPB: lnpwgts");
+  gnpwgts = fmalloc(nparts*ncon, "CPB: gnpwgts");
+  sset(nparts*ncon, 0.0, lnpwgts);
+  sset(ncon, 1.0, lminvwgts);
+
+  for (i=0; i<nvtxs; i++) {
+    for (j=0; j<ncon; j++) {
+      lnpwgts[where[i]*ncon+j] += nvwgt[i*ncon+j];
+
+      /* The following is to deal with tpwgts[] that are 0.0 for certain partitions/constraints */
+      lminvwgts[j] = (nvwgt[i*ncon+j] > 0.0 && lminvwgts[j] > nvwgt[i*ncon+j] ? nvwgt[i*ncon+j] : lminvwgts[j]);
+    }
+  }
+
+  MPI_Allreduce((void *)(lnpwgts), (void *)(gnpwgts), nparts*ncon, MPI_FLOAT, MPI_SUM, ctrl->comm);
+  MPI_Allreduce((void *)(lminvwgts), (void *)(gminvwgts), ncon, MPI_FLOAT, MPI_MIN, ctrl->comm);
+
+  /* The +gminvwgts[j] in the following code is to deal with bad cases of tpwgts[i*ncon+j] == 0 */
+  for (j=0; j<ncon; j++) {
+    maximb = 0.0;
+    for (i=0; i<nparts; i++)
+      maximb = amax(maximb, (gminvwgts[j]+gnpwgts[i*ncon+j])/(gminvwgts[j]+tpwgts[i*ncon+j]));
+    ubvec[j] = maximb;
+  }
+
+  GKfree((void **)&lnpwgts, (void **)&gnpwgts, LTERM);
+
+  return;
+}
+
+
+/*************************************************************************
+* This function prints a matrix
+**************************************************************************/
+void Moc_PrintThrottleMatrix(CtrlType *ctrl, GraphType *graph, float *matrix)
+{
+  int i, j;
+
+  for (i=0; i<ctrl->npes; i++) {
+    if (i == ctrl->mype) {
+      for (j=0; j<ctrl->npes; j++)
+        printf("%.3f ", matrix[j]);
+      printf("\n");
+      fflush(stdout);
+    }
+    MPI_Barrier(ctrl->comm);
+  }
+
+  if (ctrl->mype == 0) {
+    printf("****************************\n");
+    fflush(stdout);
+  }
+  MPI_Barrier(ctrl->comm);
+
+  return;
+}
+
+
+/*************************************************************************
+*  This function computes stats for refinement
+**************************************************************************/
+void Moc_ComputeRefineStats(CtrlType *ctrl, GraphType *graph, float *ubvec)
+{
+  int h, i, j, k;
+  int nvtxs, ncon;
+  idxtype *xadj, *adjncy, *adjwgt, *where;
+  float *nvwgt, *lnpwgts, *gnpwgts;
+  RInfoType *rinfo;
+  int mype = ctrl->mype, nparts = ctrl->nparts;
+  idxtype *gborder, *border, *gfrom, *from, *gto, *to, *connect, *gconnect;
+  idxtype gain[20] = {0}, ggain[20];
+  int lnborders, gnborders;
+  int bestgain, pmoves, gpmoves, other;
+  float tpwgts[MAXNCON], badmaxpwgt[MAXNCON];
+  int HIST_FACTOR = graph->level + 1;
+
+  nvtxs = graph->nvtxs;
+  ncon = graph->ncon;
+  xadj = graph->xadj;
+  adjncy = graph->adjncy;
+  adjwgt = graph->adjwgt;
+  where = graph->where;
+  lnpwgts = graph->lnpwgts;
+  gnpwgts = graph->gnpwgts;
+  rinfo = graph->rinfo;
+
+  connect = idxsmalloc(nparts*nparts, 0, "CRS: connect");
+  gconnect = idxmalloc(nparts*nparts, "CRS: gconnect");
+  border = idxsmalloc(nparts, 0, "CRS: border");
+  gborder = idxmalloc(nparts, "CRS: gborder");
+  from = idxsmalloc(nparts, 0, "CRS: from");
+  gfrom = idxmalloc(nparts, "CRS: gfrom");
+  to = idxsmalloc(nparts, 0, "CRS: to");
+  gto = idxmalloc(nparts, "CRS: gto");
+
+  for (h=0; h<ncon; h++) {
+    tpwgts[h] = ssum_strd(nparts, gnpwgts+h, ncon)/(float)(nparts);
+    badmaxpwgt[h] = ubvec[h]*tpwgts[h];
+  }
+
+  if (mype == 0) printf("******************************\n");
+  if (mype == 0) printf("******************************\n");
+
+  /***************************************/
+  if (mype == 0) {
+    printf("subdomain weights:\n");
+    for (h=0; h<ncon; h++) {
+      for (i=0; i<nparts; i++)
+        printf("%9.3f ", gnpwgts[i*ncon+h]);
+      printf("\n");
+    }
+    printf("\n");
+  }
+
+  /***************************************/
+  if (mype == 0) {
+    printf("subdomain imbalance:\n");
+    for (h=0; h<ncon; h++) {
+      for (i=0; i<nparts; i++)
+        printf("%9.3f ", gnpwgts[i*ncon+h] * (float)(nparts));
+      printf("\n");
+    }
+    printf("\n");
+  }
+
+  /***************************************/
+  for (i=0; i<nparts; i++)
+    connect[i*nparts+i] = -1;
+
+  for (i=0; i<nvtxs; i++) {
+    for (j=xadj[i]; j<xadj[i+1]; j++) {
+      if (where[i] != where[adjncy[j]]) {
+        connect[where[i]*nparts+where[adjncy[j]]] = 1;
+        connect[where[adjncy[j]]*nparts+where[i]] = 1;
+      }
+    }
+  }
+
+  MPI_Reduce((void *)connect, (void *)gconnect, nparts*nparts, IDX_DATATYPE, MPI_MAX, 0, ctrl->comm);
+  if (mype == 0) { 
+    printf("connectivity\n");
+    for (i=0; i<nparts; i++) {
+      printf("%d: ", i);
+      for (j=0; j<nparts; j++)
+        printf("%9d ", gconnect[i*nparts+j]);
+      printf("\n");
+    }
+    printf("\n");
+  }
+
+  /***************************************/
+  lnborders = 0;
+  for (i=0; i<nvtxs; i++)
+    if (rinfo[i].ndegrees > 0) {
+      lnborders++;
+      border[where[i]]++;
+    } 
+
+  MPI_Reduce((void *)border, (void *)gborder, nparts, IDX_DATATYPE, MPI_SUM, 0, ctrl->comm);
+  gnborders = GlobalSESum(ctrl, lnborders);
+  if (mype == 0) {
+    printf("number of borders: %d\n", gnborders);
+    for (i=0; i<nparts; i++)
+      printf("%9d ", gborder[i]);
+    printf("\n\n");
+  }
+
+  /***************************************/
+  pmoves = 0;
+  for (i=0; i<nvtxs; i++) {
+    nvwgt = graph->nvwgt+i*ncon;
+
+    for (j=0; j<rinfo[i].ndegrees; j++) {
+      other = rinfo[i].degrees[j].edge;
+      for (h=0; h<ncon; h++)
+        if (gnpwgts[other*ncon+h]+nvwgt[h] > badmaxpwgt[h])
+          break;
+
+      if (h == ncon)
+        break;
+    }
+
+    if (j < rinfo[i].ndegrees) {
+      pmoves++;
+      from[where[i]]++;
+      to[other]++;
+      for (k=j+1; k<rinfo[i].ndegrees; k++) {
+        other = rinfo[i].degrees[k].edge;
+        for (h=0; h<ncon; h++)
+          if (gnpwgts[other*ncon+h]+nvwgt[h] > badmaxpwgt[h])
+            break;
+
+        if (h == ncon) {
+          pmoves++;
+          from[where[i]]++;
+          to[other]++;
+        }
+      }
+    }
+  }
+
+  gpmoves = GlobalSESum(ctrl, pmoves);
+  MPI_Reduce((void *)from, (void *)gfrom, nparts, IDX_DATATYPE, MPI_SUM, 0, ctrl->comm);
+  MPI_Reduce((void *)to, (void *)gto, nparts, IDX_DATATYPE, MPI_SUM, 0, ctrl->comm);
+
+  if (mype == 0) {
+    printf("possible moves: %d\n", gpmoves);
+    printf("from   ");
+    for (i=0; i<nparts; i++) {
+      printf("%9d ", gfrom[i]);
+    }
+    printf("\n");
+    printf("to     ");
+    for (i=0; i<nparts; i++) {
+      printf("%9d ", gto[i]);
+    }
+    printf("\n\n");
+  }
+
+  /***************************************/
+  for (i=0; i<nvtxs; i++) {
+    if (rinfo[i].ndegrees > 0) {
+      bestgain = rinfo[i].degrees[0].ewgt-rinfo[i].id;
+      for (j=0; j<rinfo[i].ndegrees; j++)
+        bestgain = amax(bestgain, rinfo[i].degrees[j].ewgt-rinfo[i].id);
+
+      if (bestgain / HIST_FACTOR >= 10) {
+        gain[19]++;
+        continue;
+      }
+
+      if (bestgain / HIST_FACTOR < -10) {
+        gain[0]++;
+        continue;
+      }
+
+      gain[(bestgain/HIST_FACTOR)+10]++;
+    }
+  }
+
+  MPI_Reduce((void *)gain, (void *)ggain, 20, IDX_DATATYPE, MPI_SUM, 0, ctrl->comm);
+  if (mype == 0) {
+    printf("gain histogram (buckets of %d)\n", HIST_FACTOR);
+    for (i=0; i<20; i++) {
+      if (i == 10 || i == 11)
+        printf("    ");
+      printf("%d ", ggain[i]);
+    }
+    printf("\n\n");
+  }
+
+
+
+
+  /***************************************/
+  if (mype == 0) printf("******************************\n");
+  if (mype == 0) printf("******************************\n");
+
+  GKfree((void **)&gconnect, (void **)&connect, (void **)&gborder, (void **)&border, (void **)&gfrom, (void **)&from, (void **)&gto, (void **)&to, LTERM);
+  return;
+}
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/stdheaders.h b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/stdheaders.h
new file mode 100644
index 0000000..e30f989
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/stdheaders.h
@@ -0,0 +1,25 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * stdheaders.h
+ *
+ * This file includes all necessary header files
+ *
+ * Started 8/27/94
+ * George
+ *
+ * $Id: stdheaders.h,v 1.4 2003/07/25 14:31:47 karypis Exp $
+ */
+
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <malloc.h>
+#include <string.h>
+#include <ctype.h>
+#include <math.h>
+#include <stdarg.h>
+#include <limits.h>
+#include <time.h>
+#include <mpi.h>
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/struct.h b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/struct.h
new file mode 100644
index 0000000..3016c5e
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/struct.h
@@ -0,0 +1,290 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * struct.h
+ *
+ * This file contains data structures for ILU routines.
+ *
+ * Started 9/26/95
+ * George
+ *
+ * $Id: struct.h,v 1.2 2003/07/21 17:50:22 karypis Exp $
+ */
+
+/* Indexes are as long as integers for now */
+#ifdef IDXTYPE_INT
+#define IDX_DATATYPE	MPI_INT
+#define MAX_INT		INT_MAX
+#define MIN_INT		INT_MIN
+#else
+#define IDX_DATATYPE	MPI_SHORT
+#define MAX_INT		SHRT_MAX
+#define MIN_INT		SHRT_MIN
+#endif
+
+
+/*************************************************************************
+* The following data structure stores key-value pair
+**************************************************************************/
+struct KeyValueType {
+  idxtype key;
+  idxtype val;
+};
+
+typedef struct KeyValueType KeyValueType;
+
+/*************************************************************************
+* The following data structure stores key-value pair
+**************************************************************************/
+struct KVType {
+  int key;
+  float val;
+};
+
+typedef struct KVType KVType;
+
+
+/*************************************************************************
+* The following data structure stores key-value pair
+**************************************************************************/
+struct FKeyValueType {
+  float key;
+  idxtype val;
+};
+
+typedef struct FKeyValueType FKeyValueType;
+
+/*************************************************************************
+* The following data structure stores key-key-value triplets
+**************************************************************************/
+struct KeyKeyValueType {
+  idxtype key1, key2;
+  idxtype val;
+};
+
+typedef struct KeyKeyValueType KeyKeyValueType;
+
+/*************************************************************************
+* The following data structure is used to store the buckets for the 
+* refinment algorithms
+**************************************************************************/
+struct PQueueType {
+  int nnodes;
+  int maxnnodes;
+  idxtype *perm, *iperm, *values;  
+  /* iperm[i] stores where the ith entry is located
+     perm[i] stores the entry that is located in the ith position */
+};
+
+typedef struct PQueueType PQueueType;
+
+
+/*************************************************************************
+* The following data structure is used to store the buckets for the
+* refinment algorithms
+**************************************************************************/
+struct FPQueueType {
+  int type;                     /* The type of the representation used */
+  int nnodes;
+  int maxnodes;
+
+  /* Heap version of the data structure */
+  FKeyValueType *heap;
+  idxtype *locator;
+};
+
+typedef struct FPQueueType FPQueueType;
+
+/*************************************************************************
+* The following data structure stores an edge
+**************************************************************************/
+struct edgedef {
+  idxtype edge;
+  idxtype ewgt;
+};
+typedef struct edgedef EdgeType;
+
+
+/*************************************************************************
+* This data structure holds various working space data
+**************************************************************************/
+struct workspacedef {
+  idxtype *core;			/* Where pairs, indices, and degrees are coming from */
+  int maxcore;
+
+  int nlarge;				/* The size of 'Large' */
+
+  KeyValueType *pairs;			/* Large pair array used during setup */
+  idxtype *indices;			/* Large array of indxtype used for various purposes */
+
+  /* Auxiliary parameters */
+  idxtype *pv1, *pv2, *pv3, *pv4;	/* Vectors of npes+1 size used in various places */
+  KeyValueType *pepairs1, *pepairs2;
+
+  EdgeType *degrees;
+};
+
+typedef struct workspacedef WorkSpaceType;
+
+
+/*************************************************************************
+* The following data structure holds information on degrees for k-way
+* partition
+**************************************************************************/
+struct rinfodef {
+ int id, ed;            /* ID/ED of edges */
+ int ndegrees;          /* The number of different ext-degrees */
+ EdgeType *degrees;     /* List of edges */
+};
+
+typedef struct rinfodef RInfoType;
+
+
+/*************************************************************************
+* The following data structure holds information on degrees for k-way
+* partition
+**************************************************************************/
+struct nrinfodef {
+ int edegrees[2];  
+};
+
+typedef struct nrinfodef NRInfoType;
+
+
+/*************************************************************************
+* The following data structure stores a sparse matrix in CSR format
+* The diagonal entry is in the first position of each row.
+**************************************************************************/
+struct matrixdef {
+  int nrows, nnzs;		/* Number of rows and nonzeros in the matrix */
+  idxtype *rowptr;
+  idxtype *colind;
+  float *values;
+  float *transfer;
+};
+
+typedef struct matrixdef MatrixType;
+
+
+/*************************************************************************
+* This data structure holds the input graph
+**************************************************************************/
+struct graphdef {
+  int gnvtxs, nvtxs, nedges, ncon, nobj;
+  idxtype *xadj;		/* Pointers to the locally stored vertices */
+  idxtype *vwgt;		/* Vertex weights */
+  float *nvwgt;		/* Vertex weights */
+  idxtype *vsize;		/* Vertex size */
+  idxtype *adjncy;		/* Array that stores the adjacency lists of nvtxs */
+  idxtype *adjwgt;		/* Array that stores the weights of the adjacency lists */
+  idxtype *vtxdist;		/* Distribution of vertices */
+
+  idxtype *match;
+  idxtype *cmap;
+
+  idxtype *label;
+
+  /* Communication/Setup parameters */
+  int nnbrs, nrecv, nsend;		/* The number of neighboring processors */
+  idxtype *peind;			/* Array of size nnbrs storing the neighboring PEs */
+  idxtype *sendptr, *sendind;		/* CSR format of the vertices that are sent */
+  idxtype *recvptr, *recvind;		/* CSR format of the vertices that are received */
+  idxtype *imap;			/* The inverse map of local to global indices */
+  idxtype *pexadj, *peadjncy, 
+          *peadjloc;			/* CSR format of the PEs each vertex is adjancent to */
+
+  int nlocal;			/* Number of interior vertices */
+  idxtype *lperm;		/* lperm[0:nlocal] points to interior vertices, the rest are interface */
+
+  /* Communication parameters for projecting the partition. 
+   * These are computed during CreateCoarseGraph and used during projection 
+   * Note that during projection, the meaning of received and sent is reversed! */
+  idxtype *rlens, *slens;	/* Arrays of size nnbrs of how many vertices you are sending and receiving */
+  KeyValueType *rcand;
+
+
+  /* Partition parameters */
+  idxtype *where, *home;
+  idxtype *lpwgts, *gpwgts;
+  float *lnpwgts, *gnpwgts;
+  RInfoType *rinfo;
+
+  /* Node refinement information */
+  NRInfoType *nrinfo;
+  int nsep;  			/* The number of vertices in the separator */
+  idxtype *sepind;		/* The indices of the vertices in the separator */
+
+  int lmincut, mincut;
+
+  int level;
+  int match_type;
+  int edgewgt_type;
+
+  struct graphdef *coarser, *finer;
+};
+
+typedef struct graphdef GraphType;
+
+
+/*************************************************************************
+* The following data type implements a timer
+**************************************************************************/
+typedef double timer;
+
+
+/*************************************************************************
+* The following structure stores information used by parallel kmetis
+**************************************************************************/
+struct controldef {
+  int mype, npes;		/* Info about the parallel system */
+  int CoarsenTo;		/* The # of vertices in the coarsest graph */
+  int dbglvl;			/* Controls the debuging output of the program */
+  int nparts;			/* The number of partitions */
+  int foldf;			/* What is the folding factor */
+  int ipart;			/* The type of initial partitioning */
+  int xyztype;			/* The type of coordinate indexing */
+  int seed;			/* Random number seed */
+  int sync;			/* Random number seed */
+  float *tpwgts;		/* Target subdomain weights */
+  int tvwgts[MAXNCON];
+  float ubvec[MAXNCON];
+  int partType;
+  int ps_relation;
+
+  float redist_factor, redist_base, ipc_factor;
+  float edge_size_ratio;
+  MatrixType *matrix;
+
+  MPI_Comm gcomm;
+  MPI_Comm comm;		/* MPI Communicator */
+  MPI_Request sreq[MAX_PES], 
+              rreq[MAX_PES];    /* MPI send and receive requests */
+  MPI_Status statuses[MAX_PES];
+  MPI_Status status;
+
+  /* Various Timers */
+  timer TotalTmr, InitPartTmr, MatchTmr, ContractTmr, CoarsenTmr, RefTmr,
+        SetupTmr, ColorTmr, ProjectTmr, KWayInitTmr, KWayTmr, MoveTmr,
+        RemapTmr, AuxTmr1, AuxTmr2, AuxTmr3, AuxTmr4, AuxTmr5, AuxTmr6;
+};
+
+typedef struct controldef CtrlType;
+
+
+
+/*************************************************************************
+* The following data structure stores a mesh.
+**************************************************************************/
+struct meshdef {
+  int etype;
+  int gnelms, gnns;
+  int nelms, nns;
+  int ncon;
+  int esize, gminnode;
+  idxtype *elmdist;
+  idxtype *elements;
+  idxtype *elmwgt;
+};
+
+typedef struct meshdef MeshType;
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/sync b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/sync
new file mode 100644
index 0000000..11a0cfb
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/sync
@@ -0,0 +1,186 @@
+adrivers.c:  ubavg   = savg(graph->ncon, ctrl->ubvec);
+adrivers.c:  ctrl->redist_factor = ctrl->redist_base * ((gtewgt/gtvsize)/ ctrl->edge_size_ratio);
+adrivers.c:  IFSET(ctrl->dbglvl, DBG_PROGRESS, rprintf(ctrl, "[%6d %8d %5d %5d][%d]\n", 
+adrivers.c:        graph->gnvtxs, GlobalSESum(ctrl, graph->nedges), GlobalSEMin(ctrl, graph->nvtxs), GlobalSEMax(ctrl, graph->nvtxs), ctrl->CoarsenTo));
+adrivers.c:  if (graph->gnvtxs < 1.3*ctrl->CoarsenTo ||
+adrivers.c:    if (lbavg > ubavg + 0.035 && ctrl->partType != REFINE_PARTITION)
+adrivers.c:    if (ctrl->dbglvl&DBG_PROGRESS) {
+adrivers.c:    switch (ctrl->ps_relation) {
+adrivers.c:    if (ctrl->dbglvl&DBG_PROGRESS) {
+akwayfm.c:  int npes = ctrl->npes, mype = ctrl->mype, nparts = ctrl->nparts;
+akwayfm.c:  IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->KWayTmr));
+akwayfm.c:  ubvec = ctrl->ubvec;
+akwayfm.c:  tpwgts = ctrl->tpwgts;
+akwayfm.c:  ipc_factor = ctrl->ipc_factor;
+akwayfm.c:  redist_factor = ctrl->redist_factor;
+akwayfm.c:    MPI_Bcast((void *)pperm, nparts, IDX_DATATYPE, 0, ctrl->comm);
+akwayfm.c:          switch (ctrl->ps_relation) {
+akwayfm.c:              switch (ctrl->ps_relation) {
+akwayfm.c:      MPI_Allreduce((void *)lnpwgts, (void *)pgnpwgts, nparts*ncon, MPI_FLOAT, MPI_SUM, ctrl->comm);
+akwayfm.c:      IFSET(ctrl->dbglvl, DBG_RMOVEINFO, rprintf(ctrl, "\t[%d %d], [%.4f],  [%d %d %d]\n",
+akwayfm.c:        MPI_Irecv((void *)(rupdate+sendptr[i]), sendptr[i+1]-sendptr[i], IDX_DATATYPE, peind[i], 1, ctrl->comm, ctrl->rreq+i);
+akwayfm.c:        MPI_Isend((void *)(supdate+j), k-j, IDX_DATATYPE, peind[i], 1, ctrl->comm, ctrl->sreq+i);
+akwayfm.c:      MPI_Waitall(nnbrs, ctrl->rreq, ctrl->statuses);
+akwayfm.c:        MPI_Get_count(ctrl->statuses+i, IDX_DATATYPE, nupds_pe+i);
+akwayfm.c:      MPI_Waitall(nnbrs, ctrl->sreq, ctrl->statuses);
+akwayfm.c:      MPI_Allreduce((void *)lnpwgts, (void *)gnpwgts, nparts*ncon, MPI_FLOAT, MPI_SUM, ctrl->comm);
+akwayfm.c:  IFSET(ctrl->dbglvl, DBG_TIME, stoptimer(ctrl->KWayTmr));
+balancemylink.c:  ipc_factor = ctrl->ipc_factor;
+balancemylink.c:  redist_factor = ctrl->redist_factor;
+coarsen.c:  int npes=ctrl->npes, mype=ctrl->mype;
+coarsen.c:  MPI_Allgather((void *)(cvtxdist+npes), 1, IDX_DATATYPE, (void *)cvtxdist, 1, IDX_DATATYPE, ctrl->comm);
+coarsen.c:      MPI_Irecv((void *)(rsizes+i), 1, IDX_DATATYPE, peind[i], 1, ctrl->comm, ctrl->rreq+i);
+coarsen.c:      MPI_Isend((void *)(ssizes+i), 1, IDX_DATATYPE, peind[i], 1, ctrl->comm, ctrl->sreq+i);
+coarsen.c:      MPI_Wait(ctrl->rreq+i, &ctrl->status);
+coarsen.c:      MPI_Wait(ctrl->sreq+i, &ctrl->status);
+coarsen.c:      MPI_Irecv((void *)(rgraph+l), (4+ncon)*(rlens[i+1]-rlens[i])+2*rsizes[i], IDX_DATATYPE, peind[i], 1, ctrl->comm, ctrl->rreq+i);
+coarsen.c:        sgraph[ll++] = (ctrl->partType == STATIC_PARTITION) ? -1 : vsize[ii];
+coarsen.c:        sgraph[ll++] = (ctrl->partType == STATIC_PARTITION) ? -1 : home[ii];
+coarsen.c:      MPI_Isend((void *)(sgraph+l), ll-l, IDX_DATATYPE, peind[i], 1, ctrl->comm, ctrl->sreq+i);
+coarsen.c:      MPI_Wait(ctrl->rreq+i, &ctrl->status);
+coarsen.c:      MPI_Wait(ctrl->sreq+i, &ctrl->status);
+coarsen.c:  if (ctrl->partType == ADAPTIVE_PARTITION || ctrl->partType == REFINE_PARTITION) {
+coarsen.c:      if (ctrl->partType == ADAPTIVE_PARTITION || ctrl->partType == REFINE_PARTITION) {
+coarsen.c:          if (ctrl->partType == ADAPTIVE_PARTITION || ctrl->partType == REFINE_PARTITION) {
+coarsen.c:            if (ctrl->partType == ADAPTIVE_PARTITION || ctrl->partType == REFINE_PARTITION) {
+coarsen.c:      cgraph->nvwgt[j*ncon+h] = (float)(cvwgt[j*ncon+h])/(float)(ctrl->tvwgts[h]);
+comm.c:  firstvtx = graph->vtxdist[ctrl->mype];
+comm.c:              peind[i], 1, ctrl->comm, ctrl->rreq+i);
+comm.c:              peind[i], 1, ctrl->comm, ctrl->sreq+i); 
+comm.c:  MPI_Waitall(nnbrs, ctrl->rreq, ctrl->statuses); 
+comm.c:  MPI_Waitall(nnbrs, ctrl->sreq, ctrl->statuses); 
+comm.c:  firstvtx = graph->vtxdist[ctrl->mype];
+comm.c:              peind[i], 1, ctrl->comm, ctrl->rreq+i);
+comm.c:    idxcopy(ctrl->npes, sendptr, psendptr);
+comm.c:                peind[i], 1, ctrl->comm, ctrl->sreq+i);
+comm.c:      MPI_Isend((void *)(sendpairs), 0, IDX_DATATYPE, peind[i], 1, ctrl->comm, ctrl->sreq+i);
+comm.c:    MPI_Wait(ctrl->rreq+i, &(ctrl->status));
+comm.c:    MPI_Get_count(&ctrl->status, IDX_DATATYPE, &n);
+comm.c:  MPI_Waitall(nnbrs, ctrl->sreq, ctrl->statuses);
+comm.c:  MPI_Allreduce((void *)&value, (void *)&max, 1, MPI_INT, MPI_MAX, ctrl->comm);
+comm.c:  MPI_Allreduce((void *)&value, (void *)&max, 1, MPI_DOUBLE, MPI_MAX, ctrl->comm);
+comm.c:  MPI_Allreduce((void *)&value, (void *)&min, 1, MPI_INT, MPI_MIN, ctrl->comm);
+comm.c:  MPI_Allreduce((void *)&value, (void *)&sum, 1, MPI_INT, MPI_SUM, ctrl->comm);
+comm.c:  MPI_Allreduce((void *)&value, (void *)&max, 1, MPI_FLOAT, MPI_MAX, ctrl->comm);
+comm.c:  MPI_Allreduce((void *)&value, (void *)&min, 1, MPI_FLOAT, MPI_MIN, ctrl->comm);
+comm.c:  MPI_Allreduce((void *)&value, (void *)&sum, 1, MPI_FLOAT, MPI_SUM, ctrl->comm);
+debug.c:  for (penum=0; penum<ctrl->npes; penum++) {
+debug.c:    if (ctrl->mype == penum) {
+debug.c:      if (ctrl->mype == 0)
+debug.c:      printf("\t%3d. ", ctrl->mype);
+debug.c:    MPI_Barrier(ctrl->comm);
+debug.c:  for (penum=0; penum<ctrl->npes; penum++) {
+debug.c:    if (ctrl->mype == penum) {
+debug.c:      if (ctrl->mype == 0)
+debug.c:      printf("\t%3d. ", ctrl->mype);
+debug.c:    MPI_Barrier(ctrl->comm);
+debug.c:  for (penum=0; penum<ctrl->npes; penum++) {
+debug.c:    if (ctrl->mype == penum) {
+debug.c:      if (ctrl->mype == 0)
+debug.c:      printf("\t%3d. ", ctrl->mype);
+debug.c:    MPI_Barrier(ctrl->comm);
+debug.c:  MPI_Barrier(ctrl->comm);
+debug.c:  firstvtx = graph->vtxdist[ctrl->mype];
+debug.c:  for (penum=0; penum<ctrl->npes; penum++) {
+debug.c:    if (ctrl->mype == penum) {
+debug.c:    MPI_Barrier(ctrl->comm);
+debug.c:  MPI_Barrier(ctrl->comm);
+debug.c:  firstvtx = graph->vtxdist[ctrl->mype];
+debug.c:  for (penum=0; penum<ctrl->npes; penum++) {
+debug.c:    if (ctrl->mype == penum) {
+debug.c:    MPI_Barrier(ctrl->comm);
+debug.c:  MPI_Barrier(ctrl->comm);
+debug.c:  for (penum=0; penum<ctrl->npes; penum++) {
+debug.c:    if (ctrl->mype == penum) {
+debug.c:      printf("PE: %d, nnbrs: %d\n", ctrl->mype, graph->nnbrs);
+debug.c:    MPI_Barrier(ctrl->comm);
+debug.c:  MPI_Barrier(ctrl->comm);
+debug.c:  for (penum=0; penum<ctrl->npes; penum++) {
+debug.c:    if (ctrl->mype == penum) {
+debug.c:      printf("PE: %d, nnbrs: %d", ctrl->mype, nnbrs);
+debug.c:    MPI_Barrier(ctrl->comm);
+diffutil.c:  nparts = ctrl->nparts;
+diffutil.c:    myhome = (ctrl->ps_relation == COUPLED) ? ctrl->mype : graph->home[i];
+diffutil.c:  /* PrintVector(ctrl, ctrl->npes, 0, lend, "Lend: "); */
+diffutil.c:  MPI_Allreduce((void *)lstart, (void *)gstart, nparts, IDX_DATATYPE, MPI_SUM, ctrl->comm);
+diffutil.c:  MPI_Allreduce((void *)lleft, (void *)gleft, nparts, IDX_DATATYPE, MPI_SUM, ctrl->comm);
+diffutil.c:  MPI_Allreduce((void *)lend, (void *)gend, nparts, IDX_DATATYPE, MPI_SUM, ctrl->comm);
+grsetup.c:  graph->gnvtxs  = vtxdist[ctrl->npes];
+grsetup.c:  graph->nvtxs   = vtxdist[ctrl->mype+1]-vtxdist[ctrl->mype];
+grsetup.c:    ctrl->tvwgts[j] = GlobalSESum(ctrl, ltvwgts[j]);
+grsetup.c:    if (ctrl->tvwgts[i] == 0) {
+grsetup.c:      graph->nvwgt[i*ncon+j] = (float)(graph->vwgt[i*ncon+j]) / (float)(ctrl->tvwgts[j]);
+grsetup.c:  srand(ctrl->seed);
+grsetup.c:  MPI_Comm_dup(comm, &(ctrl->gcomm));
+grsetup.c:  MPI_Comm_rank(ctrl->gcomm, &ctrl->mype);
+grsetup.c:  MPI_Comm_size(ctrl->gcomm, &ctrl->npes);
+grsetup.c:  ctrl->dbglvl  = dbglvl;
+grsetup.c:  ctrl->nparts  = nparts;    /* Set the # of partitions is de-coupled from the # of domains */
+grsetup.c:  ctrl->comm    = ctrl->gcomm;
+grsetup.c:  ctrl->xyztype = XYZ_SPFILL;
+grsetup.c:  srand(ctrl->mype);
+grsetup.c:  lpvtxs = idxsmalloc(ctrl->nparts, 0, "ComputeMoveStatistics: lpvtxs");
+grsetup.c:  gpvtxs = idxsmalloc(ctrl->nparts, 0, "ComputeMoveStatistics: gpvtxs");
+grsetup.c:    if (where[i] != ctrl->mype)
+grsetup.c:  /* PrintVector(ctrl, ctrl->npes, 0, lpvtxs, "Lpvtxs: "); */
+grsetup.c:  MPI_Allreduce((void *)lpvtxs, (void *)gpvtxs, ctrl->nparts, IDX_DATATYPE, MPI_SUM, ctrl->comm);
+grsetup.c:  *maxin = GlobalSEMax(ctrl, gpvtxs[ctrl->mype]-(nvtxs-j));
+initbalance.c:  IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->InitPartTmr));
+initbalance.c:  mytpwgts = fsmalloc(ctrl->nparts, 0.0, "mytpwgts");
+initbalance.c:  for (i=0; i<ctrl->nparts; i++)
+initbalance.c:      mytpwgts[i] += ctrl->tpwgts[i*ncon+j];
+initbalance.c:  for (i=0; i<ctrl->nparts; i++)
+initbalance.c:  if (ctrl->ps_relation == DISCOUPLED) {
+initbalance.c:    rcounts = imalloc(ctrl->npes, "rcounts");
+initbalance.c:    rdispls = imalloc(ctrl->npes+1, "rdispls");
+initbalance.c:    for (i=0; i<ctrl->npes; i++) {
+initbalance.c:    MAKECSR(i, ctrl->npes, rdispls);
+initbalance.c:    (void *)part, rcounts, rdispls, IDX_DATATYPE, ctrl->comm);
+initbalance.c:    for (i=0; i<ctrl->npes; i++)
+initbalance.c:    if (part[i] >= ctrl->nparts)
+initbalance.c:      part[i] = home[i] = part[i] % ctrl->nparts;
+initbalance.c:      part[i] = home[i] = (-1*part[i]) % ctrl->nparts;
+initbalance.c:  IFSET(ctrl->dbglvl, DBG_REFINEINFO, Moc_ComputeSerialBalance(ctrl, agraph, agraph->where, lbvec));
+initbalance.c:  IFSET(ctrl->dbglvl, DBG_REFINEINFO, rprintf(ctrl, "input cut: %d, balance: ", ComputeSerialEdgeCut(agraph)));
+initbalance.c:    IFSET(ctrl->dbglvl, DBG_REFINEINFO, rprintf(ctrl, "%.3f ", lbvec[i]));
+initbalance.c:  IFSET(ctrl->dbglvl, DBG_REFINEINFO, rprintf(ctrl, "\n"));
+initbalance.c:  sr = (ctrl->mype % 2 == 0) ? 1 : 0;
+initbalance.c:  gd = (ctrl->mype % 2 == 1) ? 1 : 0;
+initbalance.c:  if (graph->ncon > MAX_NCON_FOR_DIFFUSION || ctrl->npes == 1) {
+initbalance.c:  MPI_Comm_split(ctrl->gcomm, sr, 0, &ipcomm);
+initbalance.c:  myctrl.sync = ctrl->sync;
+initbalance.c:  myctrl.seed = ctrl->seed;
+initbalance.c:  myctrl.nparts = ctrl->nparts;
+initbalance.c:  myctrl.ipc_factor = ctrl->ipc_factor;
+initbalance.c:  myctrl.redist_factor = ctrl->redist_base;
+initbalance.c:  myctrl.tpwgts = ctrl->tpwgts;
+initbalance.c:  icopy(ncon, ctrl->tvwgts, myctrl.tvwgts);
+initbalance.c:  icopy(ncon, ctrl->ubvec, myctrl.ubvec);
+initbalance.c:    moptions[7] = ctrl->sync + (mype % ngroups) + 1;
+initbalance.c:    lnparts = ctrl->nparts;
+initbalance.c:    lpecost.rank = ctrl->mype;
+initbalance.c:    if (ctrl->mype == gpecost.rank && ctrl->mype != sr_pe) {
+initbalance.c:      MPI_Send((void *)part, nvtxs, IDX_DATATYPE, sr_pe, 1, ctrl->comm);
+initbalance.c:    if (ctrl->mype != gpecost.rank && ctrl->mype == sr_pe) {
+initbalance.c:      MPI_Recv((void *)part, nvtxs, IDX_DATATYPE, gpecost.rank, 1, ctrl->comm, &status);
+initbalance.c:    if (ctrl->mype == sr_pe) {
+initbalance.c:      SerialRemap(&cgraph, ctrl->nparts, home, lwhere, part, ctrl->tpwgts);
+initbalance.c:      lpecost.rank = ctrl->mype;
+initbalance.c:      if (ctrl->mype == gpecost.rank && ctrl->mype != gd_pe)
+initbalance.c:        MPI_Send((void *)part, nvtxs, IDX_DATATYPE, gd_pe, 1, ctrl->comm);
+initbalance.c:      if (ctrl->mype != gpecost.rank && ctrl->mype == gd_pe)
+initbalance.c:        MPI_Recv((void *)part, nvtxs, IDX_DATATYPE, gpecost.rank, 1, ctrl->comm, &status);
+initbalance.c:      if (ctrl->mype == gd_pe) {
+initbalance.c:        SerialRemap(&cgraph, ctrl->nparts, home, lwhere, part, ctrl->tpwgts);
+initbalance.c:    if (ctrl->mype == sr_pe  || ctrl->mype == gd_pe) {
+initbalance.c:      my_cost = ctrl->ipc_factor * my_cut + REDIST_WGT * ctrl->redist_base * my_totalv;
+initbalance.c:      IFSET(ctrl->dbglvl, DBG_REFINEINFO, printf("%s initial cut: %.1f, totalv: %.1f, balance: %.3f\n",
+initbalance.c:      (ctrl->mype == sr_pe ? "scratch-remap" : "diffusion"), my_cut, my_totalv, my_balance));
+initbalance.c:      if (ctrl->mype == gd_pe) {
+initbalance.c:        MPI_Send((void *)buffer, 2, MPI_FLOAT, sr_pe, 1, ctrl->comm);
+initbalance.c:        MPI_Recv((void *)buffer, 2, MPI_FLOAT, gd_pe, 1, ctrl->comm, &status);
+initbalance.c:    if (ctrl->mype == sr_pe) {
+initbalance.c:    MPI_Bcast((void *)&who_wins, 1, MPI_INT, sr_pe, ctrl->comm);
+initbalance.c:  MPI_Bcast((void *)part, nvtxs, IDX_DATATYPE, who_wins, ctrl->comm);
+initbalance.c:  idxcopy(graph->nvtxs, part+vtxdist[ctrl->mype], graph->where);
+initbalance.c:  IFSET(ctrl->dbglvl, DBG_TIME, stoptim
\ No newline at end of file
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/timer.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/timer.c
new file mode 100644
index 0000000..73cc11a
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/timer.c
@@ -0,0 +1,90 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * timer.c
+ *
+ * This file contain various timing routines
+ *
+ * Started 10/19/96
+ * George
+ *
+ * $Id: timer.c,v 1.2 2003/07/21 17:18:54 karypis Exp $
+ *
+ */
+
+#include <parmetislib.h>
+
+
+
+
+/*************************************************************************
+* This function initializes the various timers
+**************************************************************************/
+void InitTimers(CtrlType *ctrl)
+{
+  cleartimer(ctrl->TotalTmr); 
+  cleartimer(ctrl->InitPartTmr);
+  cleartimer(ctrl->MatchTmr); 
+  cleartimer(ctrl->ContractTmr); 
+  cleartimer(ctrl->CoarsenTmr); 
+  cleartimer(ctrl->RefTmr);
+  cleartimer(ctrl->SetupTmr); 
+  cleartimer(ctrl->ProjectTmr); 
+  cleartimer(ctrl->KWayInitTmr); 
+  cleartimer(ctrl->KWayTmr);
+  cleartimer(ctrl->MoveTmr);
+  cleartimer(ctrl->RemapTmr);
+
+  cleartimer(ctrl->AuxTmr1); 
+  cleartimer(ctrl->AuxTmr2); 
+  cleartimer(ctrl->AuxTmr3);
+  cleartimer(ctrl->AuxTmr4); 
+  cleartimer(ctrl->AuxTmr5); 
+  cleartimer(ctrl->AuxTmr6);
+}
+
+
+/*************************************************************************
+* This function prints timing information about KMETIS
+**************************************************************************/
+void PrintTimingInfo(CtrlType *ctrl)
+{
+/*  PrintTimer(ctrl, ctrl->CoarsenTmr,  " Coarsening"); */
+  PrintTimer(ctrl, ctrl->SetupTmr,    "      Setup");
+  PrintTimer(ctrl, ctrl->MatchTmr,    "   Matching");
+  PrintTimer(ctrl, ctrl->ContractTmr, "Contraction");
+  PrintTimer(ctrl, ctrl->InitPartTmr, "   InitPart");
+/*  PrintTimer(ctrl, ctrl->RefTmr,      " Refinement"); */
+  PrintTimer(ctrl, ctrl->ProjectTmr,  "    Project");
+  PrintTimer(ctrl, ctrl->KWayInitTmr, " Initialize");
+  PrintTimer(ctrl, ctrl->KWayTmr,     "      K-way");
+  PrintTimer(ctrl, ctrl->MoveTmr,     "       Move");
+  PrintTimer(ctrl, ctrl->RemapTmr,    "      Remap");
+  PrintTimer(ctrl, ctrl->TotalTmr,    "      Total");
+  PrintTimer(ctrl, ctrl->AuxTmr1,     "       Aux1");
+  PrintTimer(ctrl, ctrl->AuxTmr2,     "       Aux2");
+  PrintTimer(ctrl, ctrl->AuxTmr3,     "       Aux3");
+  PrintTimer(ctrl, ctrl->AuxTmr4,     "       Aux4");
+  PrintTimer(ctrl, ctrl->AuxTmr5,     "       Aux5");
+  PrintTimer(ctrl, ctrl->AuxTmr6,     "       Aux6");
+}
+
+
+/*************************************************************************
+* This function prints timer stat
+**************************************************************************/
+void PrintTimer(CtrlType *ctrl, timer tmr, char *msg)
+{
+  double sum, max, tsec;
+
+  tsec = gettimer(tmr);
+  MPI_Reduce((void *)&tsec, (void *)&sum, 1, MPI_DOUBLE, MPI_SUM, 0, ctrl->comm);
+
+  tsec = gettimer(tmr);
+  MPI_Reduce((void *)&tsec, (void *)&max, 1, MPI_DOUBLE, MPI_MAX, 0, ctrl->comm);
+
+  if (ctrl->mype == 0 && sum != 0.0)
+    printf("%s: Max: %7.3f, Sum: %7.3f, Balance: %7.3f\n", 
+            msg, (float)max, (float)sum, (float)(max*ctrl->npes/sum));
+}
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/util.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/util.c
new file mode 100644
index 0000000..34c657d
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/util.c
@@ -0,0 +1,983 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * util.c
+ *
+ * This function contains various utility routines
+ *
+ * Started 9/28/95
+ * George
+ *
+ * $Id: util.c,v 1.2 2003/07/21 17:18:54 karypis Exp $
+ */
+
+#include <parmetislib.h>
+
+
+/*************************************************************************
+* This function prints an error message and exits
+**************************************************************************/
+void errexit(char *f_str,...)
+{
+  va_list argp;
+  char out1[256], out2[256];
+
+  va_start(argp, f_str);
+  vsprintf(out1, f_str, argp);
+  va_end(argp);
+
+  sprintf(out2, "Error! %s", out1);
+
+  fprintf(stdout, out2);
+  fflush(stdout);
+
+  abort();
+}
+
+
+/*************************************************************************
+* This function prints an error message and exits
+**************************************************************************/
+void myprintf(CtrlType *ctrl, char *f_str,...)
+{
+  va_list argp;
+  char out1[256], out2[256];
+
+  va_start(argp, f_str);
+  vsprintf(out1, f_str, argp);
+  va_end(argp);
+
+  sprintf(out2, "[%2d] %s", ctrl->mype, out1);
+
+  fprintf(stdout, out2);
+  fflush(stdout);
+
+}
+
+
+
+/*************************************************************************
+* This function prints an error message and exits
+**************************************************************************/
+void rprintf(CtrlType *ctrl, char *f_str,...)
+{
+  va_list argp;
+
+  if (ctrl->mype == 0) {
+    va_start(argp, f_str);
+    vfprintf(stdout, f_str, argp);
+    va_end(argp);
+  }
+
+  fflush(stdout);
+
+  MPI_Barrier(ctrl->comm);
+
+}
+
+
+#ifndef DMALLOC
+/*************************************************************************
+* The following function allocates an array of integers
+**************************************************************************/
+int *imalloc(int n, char *msg)
+{
+  if (n == 0)
+    return NULL;
+
+  return (int *)GKmalloc(sizeof(int)*n, msg);
+}
+
+
+/*************************************************************************
+* The following function allocates an array of integers
+**************************************************************************/
+idxtype *idxmalloc(int n, char *msg)
+{
+  if (n == 0)
+    return NULL;
+
+  return (idxtype *)GKmalloc(sizeof(idxtype)*n, msg);
+}
+
+
+/*************************************************************************
+* The following function allocates an array of float 
+**************************************************************************/
+float *fmalloc(int n, char *msg)
+{
+  if (n == 0)
+    return NULL;
+
+  return (float *)GKmalloc(sizeof(float)*n, msg);
+}
+
+
+/*************************************************************************
+* The follwoing function allocates an array of integers
+**************************************************************************/
+int *ismalloc(int n, int ival, char *msg)
+{
+  if (n == 0)
+    return NULL;
+
+  return iset(n, ival, (int *)GKmalloc(sizeof(int)*n, msg));
+}
+
+
+
+/*************************************************************************
+* The follwoing function allocates an array of integers
+**************************************************************************/
+idxtype *idxsmalloc(int n, idxtype ival, char *msg)
+{
+  if (n == 0)
+    return NULL;
+
+  return idxset(n, ival, (idxtype *)GKmalloc(sizeof(idxtype)*n, msg));
+}
+
+
+/*************************************************************************
+* This function is my wrapper around malloc
+**************************************************************************/
+void *GKmalloc(int nbytes, char *msg)
+{
+  void *ptr;
+
+  if (nbytes == 0)
+    return NULL;
+
+  ptr = (void *)malloc(nbytes);
+  if (ptr == NULL) 
+    errexit("***Memory allocation failed for %s. Requested size: %d bytes", msg, nbytes);
+
+  return ptr;
+}
+#endif
+
+/*************************************************************************
+* This function is my wrapper around free, allows multiple pointers    
+**************************************************************************/
+void GKfree(void **ptr1,...)
+{
+  va_list plist;
+  void **ptr;
+
+  if (*ptr1 != NULL)
+    free(*ptr1);
+  *ptr1 = NULL;
+
+  va_start(plist, ptr1);
+
+  while ((ptr = va_arg(plist, void **)) != LTERM) {
+    if (*ptr != NULL)
+      free(*ptr);
+    *ptr = NULL;
+  }
+
+  va_end(plist);
+}            
+
+
+/*************************************************************************
+* These functions set the values of a vector
+**************************************************************************/
+int *iset(int n, int val, int *x)
+{
+  int i;
+
+  for (i=0; i<n; i++)
+    x[i] = val;
+
+  return x;
+}
+
+
+/*************************************************************************
+* These functions set the values of a vector
+**************************************************************************/
+idxtype *idxset(int n, idxtype val, idxtype *x)
+{
+  int i;
+
+  for (i=0; i<n; i++)
+    x[i] = val;
+
+  return x;
+}
+
+
+
+/*************************************************************************
+* These functions return the index of the maximum element in a vector
+**************************************************************************/
+int idxamax(int n, idxtype *x)
+{
+  int i, max=0;
+
+  for (i=1; i<n; i++)
+    max = (x[i] > x[max] ? i : max);
+
+  return max;
+}
+
+
+/*************************************************************************
+* These functions return the index of the minimum element in a vector
+**************************************************************************/
+int idxamin(int n, idxtype *x)
+{
+  int i, min=0;
+
+  for (i=1; i<n; i++)
+    min = (x[i] < x[min] ? i : min);
+
+  return min;
+}
+
+
+/*************************************************************************
+* This function sums the entries in an array
+**************************************************************************/
+int idxsum(int n, idxtype *x)
+{
+  int i, sum = 0;
+
+  for (i=0; i<n; i++)
+    sum += x[i];
+
+  return sum;
+}
+
+
+/*************************************************************************
+* This function sums the entries in an array
+**************************************************************************/
+int charsum(int n, char *x)
+{
+  int i, sum = 0;
+
+  for (i=0; i<n; i++)
+    sum += x[i];
+
+  return sum;
+}
+
+/*************************************************************************
+* This function sums the entries in an array
+**************************************************************************/
+int isum(int n, int *x)
+{
+  int i, sum = 0;
+
+  for (i=0; i<n; i++)
+    sum += x[i];
+
+  return sum;
+}
+
+
+/*************************************************************************
+* This function computes a 2-norm
+**************************************************************************/
+float snorm2(int n, float *v)
+{
+  int i;
+  float partial = 0;
+ 
+  for (i = 0; i<n; i++)
+    partial += v[i] * v[i];
+
+  return sqrt(partial);
+}
+
+
+
+/*************************************************************************
+* This function computes a 2-norm
+**************************************************************************/
+float sdot(int n, float *x, float *y)
+{
+  int i;
+  float partial = 0;
+ 
+  for (i = 0; i<n; i++)
+    partial += x[i] * y[i];
+
+  return partial;
+}
+
+
+/*************************************************************************
+* This function computes a 2-norm
+**************************************************************************/
+void saxpy(int n, float alpha, float *x, float *y)
+{
+  int i;
+ 
+  for (i=0; i<n; i++)
+    y[i] += alpha*x[i];
+}
+
+
+
+
+
+
+/*************************************************************************
+* This function sorts an array of type KeyValueType in increasing order
+**************************************************************************/
+void ikeyvalsort_org(int n, KeyValueType *nodes)
+{
+  qsort((void *)nodes, (size_t)n, (size_t)sizeof(KeyValueType), IncKeyValueCmp);
+}
+
+
+/*************************************************************************
+* This function compares 2 KeyValueType variables for sorting in inc order
+**************************************************************************/
+int IncKeyValueCmp(const void *v1, const void *v2)
+{
+  KeyValueType *n1, *n2;
+
+  n1 = (KeyValueType *)v1;
+  n2 = (KeyValueType *)v2;
+
+  return (n1->key != n2->key ? n1->key - n2->key : n1->val - n2->val);
+}
+
+
+
+/*************************************************************************
+* This function sorts an array of type KeyValueType in increasing order
+**************************************************************************/
+void dkeyvalsort(int n, KeyValueType *nodes)
+{
+  qsort((void *)nodes, (size_t)n, (size_t)sizeof(KeyValueType), DecKeyValueCmp);
+}
+
+
+/*************************************************************************
+* This function compares 2 KeyValueType variables for sorting in inc order
+**************************************************************************/
+int DecKeyValueCmp(const void *v1, const void *v2)
+{
+  KeyValueType *n1, *n2;
+
+  n1 = (KeyValueType *)v1;
+  n2 = (KeyValueType *)v2;
+
+  return n2->key - n1->key;
+
+}
+
+
+
+/*************************************************************************
+* This function does a binary search on an array for a key and returns
+* the index
+**************************************************************************/
+int BSearch(int n, idxtype *array, int key)
+{
+  int a=0, b=n, c;
+
+  while (b-a > 8) {
+    c = (a+b)>>1;
+    if (array[c] > key)
+      b = c;
+    else
+      a = c;
+  }
+
+  for (c=a; c<b; c++) {
+    if (array[c] == key)
+      return c;
+  }
+
+  errexit("Key %d not found!\n", key);
+
+  return 0;
+}
+
+
+
+/*************************************************************************
+* This file randomly permutes the contents of an array.
+* flag == 0, don't initialize perm
+* flag == 1, set p[i] = i 
+**************************************************************************/
+void RandomPermute(int n, idxtype *p, int flag)
+{
+  int i, u, v;
+  idxtype tmp;
+
+  if (flag == 1) {
+    for (i=0; i<n; i++)
+      p[i] = i;
+  }
+
+  for (i=0; i<n; i++) {
+    v = RandomInRange(n);
+    u = RandomInRange(n);
+    SWAP(p[v], p[u], tmp);
+  }
+}
+
+
+/*************************************************************************
+* This file randomly permutes the contents of an array.
+* flag == 0, don't initialize perm
+* flag == 1, set p[i] = i 
+**************************************************************************/
+void FastRandomPermute(int n, idxtype *p, int flag)
+{
+  int i, u, v;
+  idxtype tmp;
+
+  /* this is for very small arrays */
+  if (n < 25) {
+    RandomPermute(n, p, flag);
+    return;
+  }
+
+  if (flag == 1) {
+    for (i=0; i<n; i++)
+      p[i] = i;
+  }
+
+  for (i=0; i<n; i+=8) {
+    v = RandomInRange(n-4);
+    u = RandomInRange(n-4);
+    SWAP(p[v], p[u], tmp);
+    SWAP(p[v+1], p[u+1], tmp);
+    SWAP(p[v+2], p[u+2], tmp);
+    SWAP(p[v+3], p[u+3], tmp);
+  }
+}
+
+/*************************************************************************
+* This function returns true if the a is a power of 2
+**************************************************************************/
+int ispow2(int a)
+{
+  for (; a%2 != 1; a = a>>1);
+  return (a > 1 ? 0 : 1);
+}
+
+/*************************************************************************
+* This function returns the log2(x)
+**************************************************************************/
+int log2Int(int a)
+{
+  int i;
+
+  for (i=1; a > 1; i++, a = a>>1);
+  return i-1;
+}
+
+
+/*************************************************************************
+* These functions set the values of a vector
+**************************************************************************/
+float *sset(int n, float val, float *x)
+{
+  int i;
+
+  for (i=0; i<n; i++)
+    x[i] = val;
+
+  return x;
+}
+
+
+
+/*************************************************************************
+* These functions return the index of the maximum element in a vector
+**************************************************************************/
+int iamax(int n, int *x)
+{
+  int i, max=0;
+
+  for (i=1; i<n; i++)
+    max = (x[i] > x[max] ? i : max);
+
+  return max;
+}
+
+
+/*************************************************************************
+* These functions return the index of the maximum element in a vector
+**************************************************************************/
+int samax_strd(int n, float *x, int incx)
+{
+  int i;
+  int max=0;
+
+  n *= incx;
+  for (i=incx; i<n; i+=incx)
+    max = (x[i] > x[max] ? i : max);
+
+  return max/incx;
+}
+
+
+/*************************************************************************
+* These functions return the index of the maximum element in a vector
+**************************************************************************/
+int sfamax(int n, float *x)
+{
+  int i;
+  int max=0;
+
+  for (i=1; i<n; i++)
+    max = (fabs(x[i]) > fabs(x[max]) ? i : max);
+
+  return max;
+}
+
+
+
+/*************************************************************************
+* These functions return the index of the maximum element in a vector
+**************************************************************************/
+int samin_strd(int n, float *x, int incx)
+{
+  int i;
+  int min=0;
+
+  n *= incx;
+  for (i=incx; i<n; i+=incx)
+    min = (x[i] < x[min] ? i : min);
+
+  return min/incx;
+}
+
+
+/*************************************************************************
+* These functions return the index of the maximum element in a vector
+**************************************************************************/
+int idxamax_strd(int n, idxtype *x, int incx)
+{
+  int i, max=0;
+
+  n *= incx;
+  for (i=incx; i<n; i+=incx)
+    max = (x[i] > x[max] ? i : max);
+
+  return max/incx;
+}
+
+
+/*************************************************************************
+* These functions return the index of the maximum element in a vector
+**************************************************************************/
+int idxamin_strd(int n, idxtype *x, int incx)
+{
+  int i, min=0;
+
+  n *= incx;
+  for (i=incx; i<n; i+=incx)
+    min = (x[i] < x[min] ? i : min);
+
+  return min/incx;
+}
+
+
+/*************************************************************************
+* This function returns the average value of an array
+**************************************************************************/
+float idxavg(int n, idxtype *x)
+{
+  int i;
+  float retval = 0.0;
+
+  for (i=0; i<n; i++)
+    retval += (float)(x[i]);
+
+  return retval / (float)(n);
+}
+
+
+/*************************************************************************
+* This function returns the average value of an array
+**************************************************************************/
+float savg(int n, float *x)
+{
+  int i;
+  float retval = 0.0;
+
+  for (i=0; i<n; i++)
+    retval += x[i];
+
+  return retval / (float)(n);
+}
+
+
+/*************************************************************************
+* These functions return the index of the maximum element in a vector
+**************************************************************************/
+int samax(int n, float *x)
+{
+  int i, max=0;
+
+  for (i=1; i<n; i++)
+    max = (x[i] > x[max] ? i : max);
+
+  return max;
+}
+
+
+/*************************************************************************
+* These functions return the index of the maximum element in a vector
+**************************************************************************/
+int sfavg(int n, float *x)
+{
+  int i;
+  float total = 0.0;
+
+  if (n == 0)
+    return 0.0;
+
+  for (i=0; i<n; i++)
+    total += fabs(x[i]);
+
+  return total / (float) n;
+}
+
+
+/*************************************************************************
+* These functions return the index of the almost maximum element in a vector
+**************************************************************************/
+int samax2(int n, float *x)
+{
+  int i, max1, max2;
+
+  if (x[0] > x[1]) {
+    max1 = 0;
+    max2 = 1;
+  }
+  else {
+    max1 = 1;
+    max2 = 0;
+  }
+
+  for (i=2; i<n; i++) {
+    if (x[i] > x[max1]) {
+      max2 = max1;
+      max1 = i;
+    }
+    else if (x[i] > x[max2])
+      max2 = i;
+  }
+
+  return max2;
+}
+
+
+/*************************************************************************
+* These functions return the index of the minimum element in a vector
+**************************************************************************/
+int samin(int n, float *x)
+{
+  int i, min=0;
+
+  for (i=1; i<n; i++)
+    min = (x[i] < x[min] ? i : min);
+
+  return min;
+}
+
+
+/*************************************************************************
+* This function sums the entries in an array
+**************************************************************************/
+int idxsum_strd(int n, idxtype *x, int incx)
+{
+  int i, sum = 0;
+
+  for (i=0; i<n; i++, x+=incx) {
+    sum += *x;
+  }
+
+  return sum;
+}
+
+
+/*************************************************************************
+* This function sums the entries in an array
+**************************************************************************/
+void idxadd(int n, idxtype *x, idxtype *y)
+{
+  for (n--; n>=0; n--)
+    y[n] += x[n];
+}
+
+
+/*************************************************************************
+* This function sums the entries in an array
+**************************************************************************/
+float ssum(int n, float *x)
+{
+  int i;
+  float sum = 0.0;
+
+  for (i=0; i<n; i++)
+    sum += x[i];
+
+  return sum;
+}
+
+/*************************************************************************
+* This function sums the entries in an array
+**************************************************************************/
+float ssum_strd(int n, float *x, int incx)
+{
+  int i;
+  float sum = 0.0;
+
+  for (i=0; i<n; i++, x+=incx)
+    sum += *x;
+
+  return sum;
+}
+
+/*************************************************************************
+* This function sums the entries in an array
+**************************************************************************/
+void sscale(int n, float alpha, float *x)
+{
+  int i;
+
+  for (i=0; i<n; i++)
+    x[i] *= alpha;
+}
+
+
+/*************************************************************************
+* This function negates the entries in an array
+**************************************************************************/
+void saneg(int n, float *x)
+{
+  int i;
+
+  for (i=0; i<n; i++)
+    x[i] = -1.0*x[i];
+}
+
+
+
+/*************************************************************************
+* This function checks if v+u2 provides a better balance in the weight
+* vector that v+u1
+**************************************************************************/
+float BetterVBalance(int ncon, float *vwgt, float *u1wgt, float *u2wgt)
+{
+  int i;
+  float sum1, sum2, diff1, diff2;
+
+  if (ncon == 1)
+    return u1wgt[0] - u1wgt[0];
+
+  sum1 = sum2 = 0.0;
+  for (i=0; i<ncon; i++) {
+    sum1 += vwgt[i]+u1wgt[i];
+    sum2 += vwgt[i]+u2wgt[i];
+  }
+  sum1 = sum1/(1.0*ncon);
+  sum2 = sum2/(1.0*ncon);
+
+  diff1 = diff2 = 0.0;
+  for (i=0; i<ncon; i++) {
+    diff1 += fabs(sum1 - (vwgt[i]+u1wgt[i]));
+    diff2 += fabs(sum2 - (vwgt[i]+u2wgt[i]));
+  }
+
+  return diff1 - diff2;
+
+}
+
+
+/*************************************************************************
+* This function checks if the pairwise balance of the between the two
+* partitions will improve by moving the vertex v from pfrom to pto,
+* subject to the target partition weights of tfrom, and tto respectively
+**************************************************************************/
+int IsHBalanceBetterFT(int ncon, float *pfrom, float *pto, float *nvwgt, float *ubvec)
+{
+  int i;
+  float blb1=0.0, alb1=0.0, sblb=0.0, salb=0.0;
+  float blb2=0.0, alb2=0.0;
+  float temp;
+
+  for (i=0; i<ncon; i++) {
+    temp = amax(pfrom[i], pto[i])/ubvec[i];
+    if (blb1 < temp) {
+      blb2 = blb1;
+      blb1 = temp;
+    }
+    else if (blb2 < temp)
+      blb2 = temp;
+    sblb += temp;
+
+    temp = amax(pfrom[i]-nvwgt[i], pto[i]+nvwgt[i])/ubvec[i];
+    if (alb1 < temp) {
+      alb2 = alb1;
+      alb1 = temp;
+    }
+    else if (alb2 < temp)
+      alb2 = temp;
+    salb += temp;
+  }
+
+  if (alb1 < blb1)
+    return 1;
+  if (blb1 < alb1)
+    return 0;
+  if (alb2 < blb2)
+    return 1;
+  if (blb2 < alb2)
+    return 0;
+
+  return salb < sblb;
+
+}
+
+/*************************************************************************
+* This function checks if it will be better to move a vertex to pt2 than
+* to pt1 subject to their target weights of tt1 and tt2, respectively
+* This routine takes into account the weight of the vertex in question
+**************************************************************************/
+int IsHBalanceBetterTT(int ncon, float *pt1, float *pt2, float *nvwgt, float *ubvec)
+{
+  int i;
+  float m11=0.0, m12=0.0, m21=0.0, m22=0.0, sm1=0.0, sm2=0.0, temp;
+
+  for (i=0; i<ncon; i++) {
+    temp = (pt1[i]+nvwgt[i])/ubvec[i];
+    if (m11 < temp) {
+      m12 = m11;
+      m11 = temp;
+    }
+    else if (m12 < temp)
+      m12 = temp;
+    sm1 += temp;
+    temp = (pt2[i]+nvwgt[i])/ubvec[i];
+    if (m21 < temp) {
+      m22 = m21;
+      m21 = temp;
+    }
+    else if (m22 < temp)
+      m22 = temp;
+    sm2 += temp;
+  }
+  if (m21 < m11)
+    return 1;
+  if (m21 > m11)
+    return 0;
+  if (m22 < m12)
+    return 1;
+  if (m22 > m12)
+    return 0;
+
+  return sm2 < sm1;
+}
+
+/*************************************************************************
+*  This is a comparison function
+**************************************************************************/
+int myvalkeycompare(const void *fptr, const void *sptr)
+{
+  KVType *first, *second;
+
+  first = (KVType *)(fptr);
+  second = (KVType *)(sptr);
+
+  if (first->val > second->val)
+    return 1;
+
+  if (first->val < second->val)
+    return -1;
+
+  return 0;
+}
+
+/*************************************************************************
+*  This is the inverse comparison function
+**************************************************************************/
+int imyvalkeycompare(const void *fptr, const void *sptr)
+{
+  KVType *first, *second;
+
+  first = (KVType *)(fptr);
+  second = (KVType *)(sptr);
+
+  if (first->val > second->val)
+    return -1;
+
+  if (first->val < second->val)
+    return 1;
+
+  return 0;
+}
+
+
+/*************************************************************************
+* The following function allocates and sets an array of floats
+**************************************************************************/
+float *fsmalloc(int n, float fval, char *msg)
+{
+  if (n == 0)
+    return NULL;
+
+  return sset(n, fval, (float *)GKmalloc(sizeof(float)*n, msg));
+}
+
+
+/*************************************************************************
+* This function computes a 2-norm
+**************************************************************************/
+void saxpy2(int n, float alpha, float *x, int incx, float *y, int incy)
+{
+  int i;
+
+  for (i=0; i<n; i++, x+=incx, y+=incy)
+    *y += alpha*(*x);
+}
+
+
+/*************************************************************************
+* This function computes the top three values of a float array
+**************************************************************************/
+void GetThreeMax(int n, float *x, int *first, int *second, int *third)
+{
+  int i;
+
+  if (n <= 0) {
+    *first = *second = *third = -1;
+    return;
+  }
+
+  *second = *third = -1;
+  *first = 0;
+
+  for (i=1; i<n; i++) {
+    if (x[i] > x[*first]) {
+      *third = *second;
+      *second = *first;
+      *first = i;
+      continue;
+    }
+
+    if (*second == -1 || x[i] > x[*second]) {
+      *third = *second;
+      *second = i;
+      continue;
+    }
+
+    if (*third == -1 || x[i] > x[*third])
+      *third = i;
+  }
+
+  return;
+}
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/wave.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/wave.c
new file mode 100644
index 0000000..0f1cb3f
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/wave.c
@@ -0,0 +1,241 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * wave.c 
+ *
+ * This file contains code for directed diffusion at the coarsest graph
+ *
+ * Started 5/19/97, Kirk, George
+ *
+ * $Id: wave.c,v 1.3 2003/07/22 21:47:18 karypis Exp $
+ *
+ */
+
+#include <parmetislib.h>
+
+/*************************************************************************
+* This function performs a k-way directed diffusion
+**************************************************************************/
+float WavefrontDiffusion(CtrlType *ctrl, GraphType *graph, idxtype *home)
+{
+  int ii, i, j, k, l, nvtxs, nedges, nparts;
+  int from, to, edge, done, nswaps, noswaps, totalv, wsize;
+  int npasses, first, second, third, mind, maxd;
+  idxtype *xadj, *adjncy, *adjwgt, *where, *perm;
+  idxtype *rowptr, *colind, *ed, *psize;
+  float *transfer, *tmpvec;
+  float balance = -1.0, *load, *solution, *workspace;
+  float *nvwgt, *npwgts, flowFactor, cost, ubfactor;
+  MatrixType matrix;
+  KeyValueType *cand;
+  int ndirty, nclean, dptr, clean;
+
+  nvtxs        = graph->nvtxs;
+  nedges       = graph->nedges;
+  xadj         = graph->xadj;
+  nvwgt        = graph->nvwgt;
+  adjncy       = graph->adjncy;
+  adjwgt       = graph->adjwgt;
+  where        = graph->where;
+  nparts       = ctrl->nparts;
+  ubfactor     = ctrl->ubvec[0];
+  matrix.nrows = nparts;
+
+  flowFactor = 0.35;
+  flowFactor = (ctrl->mype == 2) ? 0.50 : flowFactor;
+  flowFactor = (ctrl->mype == 3) ? 0.75 : flowFactor;
+  flowFactor = (ctrl->mype == 4) ? 1.00 : flowFactor;
+
+  /* allocate memory */
+  solution                   = fmalloc(4*nparts+2*nedges, "WavefrontDiffusion: solution");
+  tmpvec                     = solution + nparts;
+  npwgts                     = solution + 2*nparts;
+  load                       = solution + 3*nparts;
+  matrix.values              = solution + 4*nparts;
+  transfer = matrix.transfer = solution + 4*nparts + nedges;
+
+  perm                   = idxmalloc(2*nvtxs+2*nparts+nedges+1, "WavefrontDiffusion: perm");
+  ed                     = perm + nvtxs;
+  psize                  = perm + 2*nvtxs;
+  rowptr = matrix.rowptr = perm + 2*nvtxs + nparts;
+  colind = matrix.colind = perm + 2*nvtxs + 2*nparts + 1;
+
+  wsize     = amax(sizeof(float)*nparts*6, sizeof(idxtype)*(nvtxs+nparts*2+1));
+  workspace = (float *)GKmalloc(wsize, "WavefrontDiffusion: workspace");
+  cand      = (KeyValueType *)GKmalloc(nvtxs*sizeof(KeyValueType), "WavefrontDiffusion: cand");
+
+
+  /*****************************/
+  /* Populate empty subdomains */
+  /*****************************/
+  idxset(nparts, 0, psize);
+  for (i=0; i<nvtxs; i++) 
+    psize[where[i]]++;
+
+  mind = idxamin(nparts, psize);
+  maxd = idxamax(nparts, psize);
+  if (psize[mind] == 0) {
+    for (i=0; i<nvtxs; i++) {
+      k = (RandomInRange(nvtxs)+i)%nvtxs; 
+      if (where[k] == maxd) {
+        where[k] = mind;
+        psize[mind]++;
+        psize[maxd]--;
+        break;
+      }
+    }
+  }
+  idxset(nvtxs, 0, ed);
+  sset(nparts, 0.0, npwgts);
+  for (i=0; i<nvtxs; i++) {
+    npwgts[where[i]] += nvwgt[i];
+    for (j=xadj[i]; j<xadj[i+1]; j++)
+      ed[i] += (where[i] != where[adjncy[j]] ? adjwgt[j] : 0);
+  }
+
+  ComputeLoad(graph, nparts, load, ctrl->tpwgts, 0);
+  done = 0;
+
+  npasses = amin(nparts/2, NGD_PASSES);
+  for (l=0; l<npasses; l++) {
+    /* Set-up and solve the diffusion equation */
+    nswaps = 0;
+
+    /************************/
+    /* Solve flow equations */
+    /************************/
+    SetUpConnectGraph(graph, &matrix, (idxtype *)workspace);
+
+    /* check for disconnected subdomains */
+    for(i=0; i<matrix.nrows; i++) {
+      if (matrix.rowptr[i]+1 == matrix.rowptr[i+1]) {
+        cost = (float)(ctrl->mype); 
+	goto CleanUpAndExit;
+      }
+    }
+
+    ConjGrad2(&matrix, load, solution, 0.001, workspace);
+    ComputeTransferVector(1, &matrix, solution, transfer, 0);
+
+    GetThreeMax(nparts, load, &first, &second, &third);
+
+    if (l%3 == 0) {
+      FastRandomPermute(nvtxs, perm, 1);
+    }
+    else {
+      /*****************************/
+      /* move dirty vertices first */
+      /*****************************/
+      ndirty = 0;
+      for (i=0; i<nvtxs; i++)
+        if (where[i] != home[i])
+          ndirty++;
+
+      dptr = 0;
+      for (i=0; i<nvtxs; i++)
+        if (where[i] != home[i])
+          perm[dptr++] = i;
+        else
+          perm[ndirty++] = i;
+
+      ASSERT(ctrl, ndirty == nvtxs);
+      ndirty = dptr;
+      nclean = nvtxs-dptr;
+      FastRandomPermute(ndirty, perm, 0);
+      FastRandomPermute(nclean, perm+ndirty, 0);
+    }
+
+    if (ctrl->mype == 0) {
+      for (j=nvtxs, k=0, ii=0; ii<nvtxs; ii++) {
+        i = perm[ii];
+        if (ed[i] != 0) {
+          cand[k].key = -ed[i];
+          cand[k++].val = i;
+        }
+        else {
+          cand[--j].key = 0;
+          cand[j].val = i;
+        }
+      }
+      ikeysort(k, cand);
+    }
+
+    for (ii=0; ii<nvtxs/3; ii++) {
+      i = (ctrl->mype == 0) ? cand[ii].val : perm[ii];
+      from = where[i];
+
+      /* don't move out the last vertex in a subdomain */
+      if (psize[from] == 1)
+        continue;
+
+      clean = (from == home[i]) ? 1 : 0;
+
+      /* only move from top three or dirty vertices */
+      if (from != first && from != second && from != third && clean)
+        continue;
+
+      /* Scatter the sparse transfer row into the dense tmpvec row */
+      for (j=rowptr[from]+1; j<rowptr[from+1]; j++)
+        tmpvec[colind[j]] = transfer[j];
+
+      for (j=xadj[i]; j<xadj[i+1]; j++) {
+        to = where[adjncy[j]];
+        if (from != to) {
+          if (tmpvec[to] > (flowFactor * nvwgt[i])) {
+            tmpvec[to] -= nvwgt[i];
+            INC_DEC(psize[to], psize[from], 1);
+            INC_DEC(npwgts[to], npwgts[from], nvwgt[i]);
+            INC_DEC(load[to], load[from], nvwgt[i]);
+            where[i] = to;
+            nswaps++;
+
+            /* Update external degrees */
+            ed[i] = 0;
+            for (k=xadj[i]; k<xadj[i+1]; k++) {
+              edge = adjncy[k];
+              ed[i] += (to != where[edge] ? adjwgt[k] : 0);
+
+              if (where[edge] == from)
+                ed[edge] += adjwgt[k];
+              if (where[edge] == to)
+                ed[edge] -= adjwgt[k];
+            }
+            break;
+          }
+        }
+      }
+
+      /* Gather the dense tmpvec row into the sparse transfer row */
+      for (j=rowptr[from]+1; j<rowptr[from+1]; j++) {
+        transfer[j] = tmpvec[colind[j]];
+        tmpvec[colind[j]] = 0.0;
+      }
+      ASSERTS(fabs(ssum(nparts, tmpvec)) < .0001)
+    }
+
+    if (l % 2 == 1) {
+      balance = npwgts[samax(nparts, npwgts)] * (float)nparts;
+      if (balance < ubfactor + 0.035)
+        done = 1;
+
+      if (GlobalSESum(ctrl, done) > 0)
+        break;
+
+      noswaps = (nswaps > 0) ? 0 : 1;
+      if (GlobalSESum(ctrl, noswaps) > ctrl->npes/2)
+        break;
+
+    }
+  }
+
+  graph->mincut = ComputeSerialEdgeCut(graph);
+  totalv        = Mc_ComputeSerialTotalV(graph, home);
+  cost          = ctrl->ipc_factor * (float)graph->mincut + ctrl->redist_factor * (float)totalv;
+
+
+CleanUpAndExit:
+  GKfree((void **)&solution, (void **)&perm, (void **)&workspace, (void **)&cand, LTERM);
+
+  return cost;
+}
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/weird.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/weird.c
new file mode 100644
index 0000000..69d0e5d
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/weird.c
@@ -0,0 +1,275 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * weird.c
+ *
+ * This file contain various graph setting up routines
+ *
+ * Started 10/19/96
+ * George
+ *
+ * $Id: weird.c,v 1.9 2003/07/31 16:27:28 karypis Exp $
+ *
+ */
+
+#include <parmetislib.h>
+
+
+
+/*************************************************************************
+* This function computes a partitioning of a small graph
+**************************************************************************/
+void PartitionSmallGraph(CtrlType *ctrl, GraphType *graph, WorkSpaceType *wspace)
+{
+  int i, h, ncon, nparts, npes, mype;
+  int moptions[10];
+  int mynumflag, mywgtflag, me;
+  idxtype *mypart;
+  int lpecut[2], gpecut[2];
+  GraphType *agraph;
+  int *sendcounts, *displs;
+  float *mytpwgts, *gnpwgts, *lnpwgts;
+
+  ncon = graph->ncon;
+  nparts = ctrl->nparts;
+
+  MPI_Comm_size(ctrl->comm, &npes);
+  MPI_Comm_rank(ctrl->comm, &mype);
+
+  SetUp(ctrl, graph, wspace);
+  graph->where = idxmalloc(graph->nvtxs+graph->nrecv, "PartitionSmallGraph: where");
+  agraph       = Moc_AssembleAdaptiveGraph(ctrl, graph, wspace);
+  mypart       = idxmalloc(agraph->nvtxs, "mypart");
+
+  moptions[0] = 0;
+  moptions[7] = ctrl->sync + mype;
+  mynumflag = 0;
+  mywgtflag = 3;
+  if (ncon == 1) {
+    METIS_WPartGraphKway2(&agraph->nvtxs, agraph->xadj, agraph->adjncy, agraph->vwgt, 
+          agraph->adjwgt, &mywgtflag, &mynumflag, &nparts, ctrl->tpwgts, moptions, 
+	  &graph->mincut, mypart);
+  }
+  else {
+    mytpwgts = fmalloc(nparts, "mytpwgts");
+    for (i=0; i<nparts; i++)
+      mytpwgts[i] = ctrl->tpwgts[i*ncon];
+
+    METIS_mCPartGraphRecursive2(&agraph->nvtxs, &ncon, agraph->xadj, agraph->adjncy, 
+          agraph->vwgt, agraph->adjwgt, &mywgtflag, &mynumflag, &nparts, mytpwgts, 
+	  moptions, &graph->mincut, mypart);
+
+    free(mytpwgts);
+  }
+
+  lpecut[0] = graph->mincut;
+  lpecut[1] = mype;
+  MPI_Allreduce(lpecut, gpecut, 1, MPI_2INT, MPI_MINLOC, ctrl->comm);
+  graph->mincut = gpecut[0];
+
+  if (lpecut[1] == gpecut[1] && gpecut[1] != 0)
+    MPI_Send((void *)mypart, agraph->nvtxs, IDX_DATATYPE, 0, 1, ctrl->comm);
+  if (lpecut[1] == 0 && gpecut[1] != 0)
+    MPI_Recv((void *)mypart, agraph->nvtxs, IDX_DATATYPE, gpecut[1], 1, ctrl->comm, &ctrl->status);
+
+  sendcounts = imalloc(npes, "sendcounts");
+  displs     = imalloc(npes, "displs");
+
+  for (i=0; i<npes; i++) {
+    sendcounts[i] = graph->vtxdist[i+1]-graph->vtxdist[i];
+    displs[i] = graph->vtxdist[i];
+  }
+
+  MPI_Scatterv((void *)mypart, sendcounts, displs, IDX_DATATYPE,
+               (void *)graph->where, graph->nvtxs, IDX_DATATYPE, 0, ctrl->comm);
+
+  lnpwgts = graph->lnpwgts = fmalloc(nparts*ncon, "lnpwgts");
+  gnpwgts = graph->gnpwgts = fmalloc(nparts*ncon, "gnpwgts");
+  sset(nparts*ncon, 0, lnpwgts);
+  for (i=0; i<graph->nvtxs; i++) {
+    me = graph->where[i];
+    for (h=0; h<ncon; h++)
+      lnpwgts[me*ncon+h] += graph->nvwgt[i*ncon+h];
+  }
+  MPI_Allreduce((void *)lnpwgts, (void *)gnpwgts, nparts*ncon, MPI_FLOAT, MPI_SUM, ctrl->comm);
+  GKfree((void**)&mypart, (void**)&sendcounts, (void**)&displs, LTERM);
+  FreeGraph(agraph);
+
+  return;
+}
+
+
+
+/*************************************************************************
+* This function checks the inputs for the partitioning routines
+**************************************************************************/
+void CheckInputs(int partType, int npes, int dbglvl, int *wgtflag, int *iwgtflag,
+                 int *numflag, int *inumflag, int *ncon, int *incon, int *nparts, 
+		 int *inparts, float *tpwgts, float **itpwgts, float *ubvec, 
+		 float *iubvec, float *ipc2redist, float *iipc2redist, int *options, 
+		 int *ioptions, idxtype *part, MPI_Comm *comm)
+{
+  int i, j;
+  int doweabort, doiabort = 0;
+  float tsum, *myitpwgts;
+  int mgcnums[5] = {-1, 2, 3, 4, 2};
+
+  /**************************************/
+  if (part == NULL) {
+    doiabort = 1;
+    IFSET(dbglvl, DBG_INFO, printf("ERROR: part array is set to NULL.\n"));
+  }
+  /**************************************/
+
+
+  /**************************************/
+  if (wgtflag == NULL) {
+    *iwgtflag = 0;
+    IFSET(dbglvl, DBG_INFO, printf("WARNING: wgtflag is NULL.  Using a value of 0.\n"));
+  }
+  else {
+    *iwgtflag = *wgtflag;
+  }
+  /**************************************/
+
+
+  /**************************************/
+  if (numflag == NULL) {
+    *inumflag = 0;
+    IFSET(dbglvl, DBG_INFO, printf("WARNING: numflag is NULL.  Using a value of 0.\n"));
+  }
+  else {
+    if (*numflag != 0 && *numflag != 1) {
+      IFSET(dbglvl, DBG_INFO, printf("WARNING: bad value for numflag %d.  Using a value of 0.\n", *numflag));
+      *inumflag = 0;
+    }
+    else {
+      *inumflag = *numflag;
+    }
+  }
+  /**************************************/
+
+
+  /**************************************/
+  if (ncon == NULL) {
+    *incon = 1;
+    IFSET(dbglvl, DBG_INFO, printf("WARNING: ncon is NULL.  Using a value of 1.\n"));
+  }
+  else {
+    if (*ncon < 1 || *ncon > MAXNCON) {
+      IFSET(dbglvl, DBG_INFO, printf("WARNING: bad value for ncon %d.  Using a value of 1.\n", *ncon));
+      *incon = 1;
+    }
+    else {
+      *incon = *ncon;
+    }
+  }
+  /**************************************/
+
+
+  /**************************************/
+  if (nparts == NULL) {
+    *inparts = npes;
+    IFSET(dbglvl, DBG_INFO, printf("WARNING: nparts is NULL.  Using a value of %d.\n", npes));
+  }
+  else {
+    if (*nparts < 1 || *nparts > MAX_NPARTS) {
+      IFSET(dbglvl, DBG_INFO, printf("WARNING: bad value for nparts %d.  Using a value of %d.\n", *nparts, npes));
+      *inparts = npes;
+    }
+    else {
+      *inparts = *nparts;
+    }
+  }
+  /**************************************/
+
+
+  /**************************************/
+  myitpwgts = *itpwgts = fmalloc((*inparts)*(*incon), "CheckInputs: itpwgts");
+  if (tpwgts == NULL) {
+    sset((*inparts)*(*incon), 1.0/(float)(*inparts), myitpwgts);
+    IFSET(dbglvl, DBG_INFO, printf("WARNING: tpwgts is NULL.  Setting all array elements to %.3f.\n", 1.0/(float)(*inparts)));
+  }
+  else {
+    for (i=0; i<*incon; i++) {
+      tsum = 0.0;
+      for (j=0; j<*inparts; j++) {
+        tsum += tpwgts[j*(*incon)+i];
+      } 
+      if (fabs(1.0-tsum) < SMALLFLOAT)
+        tsum = 1.0;
+      for (j=0; j<*inparts; j++)
+       myitpwgts[j*(*incon)+i] = tpwgts[j*(*incon)+i] / tsum;
+    }
+  }
+  /**************************************/
+
+
+  /**************************************/
+  if (ubvec == NULL) {
+    sset(*incon, 1.05, iubvec);
+    IFSET(dbglvl, DBG_INFO, printf("WARNING: ubvec is NULL.  Setting all array elements to 1.05.\n"));
+  }
+  else {
+    for (i=0; i<*incon; i++) {
+      if (ubvec[i] < 1.0 || ubvec[i] > (float)(*inparts)) {
+        iubvec[i] = 1.05;
+        IFSET(dbglvl, DBG_INFO, printf("WARNING: bad value for ubvec[%d]: %.3f.  Setting value to 1.05.[%d]\n", i, ubvec[i], *inparts));
+      }
+      else {
+        iubvec[i] = ubvec[i];
+      }
+    }
+  }
+  /**************************************/
+
+
+  /**************************************/
+  if (partType == ADAPTIVE_PARTITION) {
+    if (ipc2redist != NULL) {
+      if (*ipc2redist < SMALLFLOAT || *ipc2redist > 1000000.0) {
+        IFSET(dbglvl, DBG_INFO, printf("WARNING: bad value for ipc2redist %.3f.  Using a value of 1000.\n", *ipc2redist));
+        *iipc2redist = 1000.0;
+      }
+      else {
+        *iipc2redist = *ipc2redist;
+      }
+    }
+    else {
+      IFSET(dbglvl, DBG_INFO, printf("WARNING: ipc2redist is NULL.  Using a value of 1000.\n"));
+      *iipc2redist = 1000.0;
+    }
+  }
+  /**************************************/
+
+
+  /**************************************/
+  if (options == NULL) {
+    ioptions[0] = 0;
+    IFSET(dbglvl, DBG_INFO, printf("WARNING: options is NULL.  Using defaults\n"));
+  }
+  else {
+    ioptions[0] = options[0];
+    ioptions[1] = options[1];
+    ioptions[2] = options[2];
+    if (partType == ADAPTIVE_PARTITION || partType == REFINE_PARTITION)
+      ioptions[3] = options[3];
+  }
+  /**************************************/
+
+
+  /**************************************/
+  if (comm == NULL) {
+    IFSET(dbglvl, DBG_INFO, printf("ERROR: comm is NULL.  Aborting\n"));
+    abort();
+  }
+  else {
+    MPI_Allreduce((void *)&doiabort, (void *)&doweabort, 1, MPI_INT, MPI_MAX, *comm);
+    if (doweabort > 0)
+      abort();
+  }
+  /**************************************/
+
+}
+
+
diff --git a/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/xyzpart.c b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/xyzpart.c
new file mode 100644
index 0000000..e538034
--- /dev/null
+++ b/benchmarks/CUDA/DG/3rdParty/ParMetis-3.1/ParMETISLib/xyzpart.c
@@ -0,0 +1,257 @@
+/*
+ * Copyright 1997, Regents of the University of Minnesota
+ *
+ * xyzpart.c
+ *
+ * This file contains code that implements a coordinate based partitioning
+ *
+ * Started 7/11/97
+ * George
+ *
+ * $Id: xyzpart.c,v 1.3 2003/07/30 18:37:59 karypis Exp $
+ *
+ */
+
+#include <parmetislib.h>
+
+
+/*************************************************************************
+* This function implements a simple coordinate based partitioning
+**************************************************************************/
+void Coordinate_Partition(CtrlType *ctrl, GraphType *graph, int ndims, float *xyz, 
+                          int setup, WorkSpaceType *wspace)
+{
+  int i, j, k, nvtxs, firstvtx, icoord, coords[3];
+  idxtype *vtxdist;
+  float max[3], min[3], gmin[3], gmax[3], shift[3], scale[3];
+  KeyValueType *cand;
+
+  if (setup)
+    SetUp(ctrl, graph, wspace);
+  else
+    graph->nrecv = 0;
+
+  nvtxs = graph->nvtxs;
+  vtxdist = graph->vtxdist;
+
+  firstvtx = vtxdist[ctrl->mype];
+
+  cand = (KeyValueType *)GKmalloc(nvtxs*sizeof(KeyValueType), "Coordinate_Partition: cand");
+
+  /* Compute parameters for coordinate transformation */
+  for (k=0; k<ndims; k++) {
+    min[k] = +10000000;
+    max[k] = -10000000;
+  }
+  for (i=0; i<nvtxs; i++) {
+    for (k=0; k<ndims; k++) {
+      if (xyz[i*ndims+k] < min[k])
+        min[k] = xyz[i*ndims+k];
+      if (xyz[i*ndims+k] > max[k])
+        max[k] = xyz[i*ndims+k];
+    }
+  }
+
+  /* Compute global min and max */
+  MPI_Allreduce((void *)min, (void *)gmin, ndims, MPI_FLOAT, MPI_MIN, ctrl->comm);
+  MPI_Allreduce((void *)max, (void *)gmax, ndims, MPI_FLOAT, MPI_MAX, ctrl->comm);
+
+  /* myprintf(ctrl, "Coordinate Range: %e %e, Global %e %e\n", min[0], max[0], gmin[0], gmax[0]); */
+
+  for (k=0; k<ndims; k++) {
+    /* rprintf(ctrl, "Dim#%d: %e %e, span: %e\n", k, gmin[k], gmax[k], gmax[k]-gmin[k]); */
+    shift[k] = -gmin[k];
+    if (gmax[k] != gmin[k])
+      scale[k] = 1.0/(gmax[k]-gmin[k]);
+    else
+      scale[k] = 1.0;
+  }
+
+  switch (ctrl->xyztype) {
+    case XYZ_XCOORD:
+      for (i=0; i<nvtxs; i++) {
+        cand[i].key = 1000000*((xyz[i*ndims]+shift[0])*scale[0]);
+        ASSERT(ctrl, cand[i].key>=0 && cand[i].key<=1000000);
+        cand[i].val = firstvtx+i;
+      }
+      break;
+    case XYZ_SPFILL:
+      for (i=0; i<nvtxs; i++) {
+        for (k=0; k<ndims; k++)
+          coords[k] = 1024*((xyz[i*ndims+k]+shift[k])*scale[k]);
+        for (icoord=0, j=9; j>=0; j--) {
+          for (k=0; k<ndims; k++)
+            icoord = (icoord<<1) + (coords[k]&(1<<j) ? 1 : 0);
+        }
+        cand[i].key = icoord;
+        cand[i].val = firstvtx+i;
+      }
+      break;
+    default:
+      errexit("Unknown XYZ_Type type!\n");
+  }
+
+
+  /* Partition using sorting */
+  PartSort(ctrl, graph, cand, wspace);
+
+  free(cand);
+
+}
+
+
+
+/*************************************************************************
+* This function sorts a distributed list of KeyValueType in increasing 
+* order, and uses it to compute a partition. It uses samplesort. 
+**************************************************************************/
+void PartSort(CtrlType *ctrl, GraphType *graph, KeyValueType *elmnts, WorkSpaceType *wspace)
+{
+  int i, j, k, nvtxs, nrecv, npes=ctrl->npes, mype=ctrl->mype, firstvtx, lastvtx;
+  idxtype *scounts, *rcounts, *vtxdist, *perm;
+  KeyValueType *relmnts, *mypicks, *allpicks;
+
+  nvtxs   = graph->nvtxs;
+  vtxdist = graph->vtxdist;
+
+  scounts = wspace->pv1;
+  rcounts = wspace->pv2;
+
+  /* Allocate memory for the splitters */
+  mypicks  = (KeyValueType *)GKmalloc(sizeof(KeyValueType)*(npes+1), "ParSort: mypicks");
+  allpicks = (KeyValueType *)GKmalloc(sizeof(KeyValueType)*npes*npes, "ParSort: allpicks");
+
+  /* Sort the local elements */
+  ikeysort(nvtxs, elmnts);
+
+  /* Select the local npes-1 equally spaced elements */
+  for (i=1; i<npes; i++) { 
+    mypicks[i-1].key = elmnts[i*(nvtxs/npes)].key;
+    mypicks[i-1].val = elmnts[i*(nvtxs/npes)].val;
+  }
+
+  /* PrintPairs(ctrl, npes-1, mypicks, "Mypicks"); */
+
+  /* Gather the picks to all the processors */
+  MPI_Allgather((void *)mypicks, 2*(npes-1), IDX_DATATYPE, (void *)allpicks, 2*(npes-1), IDX_DATATYPE, ctrl->comm);
+
+  /* PrintPairs(ctrl, npes*(npes-1), allpicks, "Allpicks"); */
+
+  /* Sort all the picks */
+  ikeyvalsort(npes*(npes-1), allpicks);
+
+  /* PrintPairs(ctrl, npes*(npes-1), allpicks, "Allpicks"); */
+
+  /* Select the final splitters. Set the boundaries to simplify coding */
+  for (i=1; i<npes; i++)
+    mypicks[i] = allpicks[i*(npes-1)];
+  mypicks[0].key    = MIN_INT;
+  mypicks[npes].key = MAX_INT;
+
+  /* PrintPairs(ctrl, npes+1, mypicks, "Mypicks"); */
+
+  /* Compute the number of elements that belong to each bucket */
+  idxset(npes, 0, scounts);
+  for (j=i=0; i<nvtxs; i++) {
+    if (elmnts[i].key < mypicks[j+1].key || (elmnts[i].key == mypicks[j+1].key && elmnts[i].val < mypicks[j+1].val))
+      scounts[j]++;
+    else
+      scounts[++j]++;
+  }
+  MPI_Alltoall(scounts, 1, IDX_DATATYPE, rcounts, 1, IDX_DATATYPE, ctrl->comm);
+
+/*
+  PrintVector(ctrl, npes, 0, scounts, "Scounts");
+  PrintVector(ctrl, npes, 0, rcounts, "Rcounts");
+*/
+
+  /* Allocate memory for sorted elements and receive them */
+  MAKECSR(i, npes, scounts);
+  MAKECSR(i, npes, rcounts);
+  nrecv = rcounts[npes];
+  if (wspace->nlarge >= nrecv)
+    relmnts = (KeyValueType *)wspace->pairs;
+  else
+    relmnts = (KeyValueType *)GKmalloc(sizeof(KeyValueType)*nrecv, "ParSort: relmnts");
+
+  /* Issue the receives first */
+  for (i=0; i<npes; i++) 
+    MPI_Irecv((void *)(relmnts+rcounts[i]), 2*(rcounts[i+1]-rcounts[i]), IDX_DATATYPE, i, 1, ctrl->comm, ctrl->rreq+i);
+
+  /* Issue the sends next */
+  for (i=0; i<npes; i++) 
+    MPI_Isend((void *)(elmnts+scounts[i]), 2*(scounts[i+1]-scounts[i]), IDX_DATATYPE, i, 1, ctrl->comm, ctrl->sreq+i);
+
+  MPI_Waitall(npes, ctrl->rreq, ctrl->statuses);
+  MPI_Waitall(npes, ctrl->sreq, ctrl->statuses);
+
+
+  /* OK, now do the local sort of the relmnts. Use perm to keep track original order */
+  perm = idxmalloc(nrecv, "ParSort: perm");
+  for (i=0; i<nrecv; i++) {
+    perm[i] = relmnts[i].val;
+    relmnts[i].val = i;
+  }
+  ikeysort(nrecv, relmnts);
+
+
+  /* Compute what needs to be shifted */
+  MPI_Scan((void *)(&nrecv), (void *)(&lastvtx), 1, MPI_INT, MPI_SUM, ctrl->comm);
+  firstvtx = lastvtx-nrecv;  
+
+  /*myprintf(ctrl, "first, last: %d %d\n", firstvtx, lastvtx); */
+
+  for (j=0, i=0; i<npes; i++) {
+    if (vtxdist[i+1] > firstvtx) {  /* Found the first PE that is passed me */
+      if (vtxdist[i+1] >= lastvtx) {
+        /* myprintf(ctrl, "Shifting %d elements to processor %d\n", lastvtx-firstvtx, i); */
+        for (k=0; k<lastvtx-firstvtx; k++, j++) 
+          relmnts[relmnts[j].val].key = i;
+      }
+      else {
+        /* myprintf(ctrl, "Shifting %d elements to processor %d\n", vtxdist[i+1]-firstvtx, i); */
+        for (k=0; k<vtxdist[i+1]-firstvtx; k++, j++) 
+          relmnts[relmnts[j].val].key = i;
+
+        firstvtx = vtxdist[i+1];
+      }
+    }
+    if (vtxdist[i+1] >= lastvtx)
+      break;
+  }
+
+  /* Reverse the ordering on the relmnts[].val */
+  for (i=0; i<nrecv; i++) {
+    ASSERTP(ctrl, relmnts[i].key>=0 && relmnts[i].key<npes, (ctrl, "%d %d\n", i, relmnts[i].key));
+    relmnts[i].val = perm[i];
+  }
+
+  /* OK, now sent it back */
+  /* Issue the receives first */
+  for (i=0; i<npes; i++) 
+    MPI_Irecv((void *)(elmnts+scounts[i]), 2*(scounts[i+1]-scounts[i]), IDX_DATATYPE, i, 1, ctrl->comm, ctrl->rreq+i);
+
+  /* Issue the sends next */
+  for (i=0; i<npes; i++) 
+    MPI_Isend((void *)(relmnts+rcounts[i]), 2*(rcounts[i+1]-rcounts[i]), IDX_DATATYPE, i, 1, ctrl->comm, ctrl->sreq+i);
+
+  MPI_Waitall(npes, ctrl->rreq, ctrl->statuses);
+  MPI_Waitall(npes, ctrl->sreq, ctrl->statuses);
+
+
+  /* Construct a partition for the graph */
+  graph->where = idxmalloc(graph->nvtxs+graph->nrecv, "PartSort: graph->where");
+  firstvtx = vtxdist[mype];
+  for (i=0; i<nvtxs; i++) {
+    ASSERTP(ctrl, elmnts[i].key>=0 && elmnts[i].key<npes, (ctrl, "%d %d\n", i, elmnts[i].key));
+    ASSERTP(ctrl, elmnts[i].val>=vtxdist[mype] && elmnts[i].val<vtxdist[mype+1], (ctrl, "%d %d %d %d\n", i, vtxdist[mype], vtxdist[mype+1], elmnts[i].val));
+    graph->where[elmnts[i].val-firstvtx] = elmnts[i].key;
+  }
+
+
+  GKfree((void **)&mypicks, (void **)&allpicks, (void **)&perm, LTERM);
+  if (wspace->nlarge < nrecv)
+    free(relmnts);
+
+}
+