Пример использования MPIX-расширений на системе Blue Gene

/*
 * io.c
 *
 * Example of Blue Gene MPIX MPI-extensions usage along with MPI-IO
 * /bgsys/drivers/ppcfloor/comm/bin/mpicc io.c -Wall -Wextra -ansi
 *
 * Alexander Pozdneev
 * http://hpc.cmc.msu.ru
 */

#include <stdio.h>
#include <stdlib.h>
#include <mpi.h>
#include <mpix.h>

int main(int argc, char **argv)
{
    const MPI_Comm world_comm = MPI_COMM_WORLD;
    int world_rank;

    MPI_Comm pset_comm;
    int pset_rank, pset_size, pset_root = 0;

    MPI_Comm io_comm;
    int io_rank;

    int *buf = NULL;
    int buf_size;

    MPI_Init(&argc, &argv);
    MPI_Comm_rank(world_comm, &world_rank);

    /* Create separate communicator for each pset */
    MPIX_Pset_same_comm_create(&pset_comm);
    MPI_Comm_rank(pset_comm, &pset_rank);
    MPI_Comm_size(pset_comm, &pset_size);

    /* Let's gather some data (say, world_rank) from each process of
     * pset_comm on the pset_root */
    if (pset_rank == pset_root)
    {
        buf_size = pset_size;
        buf = malloc(buf_size * sizeof(buf[0]));
    }

    MPI_Gather(&world_rank, 1, MPI_INT, buf, 1, MPI_INT,
            pset_root, pset_comm);

    /* Create separate communicators where no process share the same I/O node */
    MPIX_Pset_diff_comm_create(&io_comm);
    MPI_Comm_rank(io_comm, &io_rank);
   
    /* In each pset, only pset_root process works; these processes are
     * in the same io_comm and use it for MPI-IO collectives */
    if (pset_rank == pset_root)
    {
        char *fname = "mpix.bin";
        MPI_File fout;

        MPI_File_open(io_comm, fname, MPI_MODE_CREATE | MPI_MODE_WRONLY,
                MPI_INFO_NULL, &fout);
        MPI_File_set_view(fout, io_rank * buf_size * sizeof(buf[0]),
                MPI_INT, MPI_INT, "native", MPI_INFO_NULL);
        MPI_File_write_all(fout, buf, buf_size, MPI_INT, MPI_STATUS_IGNORE);
        MPI_File_close(&fout);
    }
   
    if (pset_rank == pset_root)
        free(buf);

    MPI_Finalize();

    return 0;
}