Ignore:
File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/oar/mpilauncher.cpp

    r28 r17  
    1414
    1515/*
    16  * These two variables sets the command to launch in each sub-directory
    17  * and the arguments required. The name of the root directory of the
    18  * datasets is given as an argument when starting the MPIlauncher. const
    19  * string
    20  * commandToLaunch("$HOME/SOURCES/MES-PROJETS/MPILAUNCHER/appli.exe");
    21  * const string("-l -info");
    22  */
     16These two variables sets the command to launch in each sub-directory and the arguments required. The name of the root directory of the datasets is given as an argument when starting the MPIlauncher.
     17const string commandToLaunch("/HA/sources/begou/SOURCES/MES-PROJETS/MPILAUNCHER/appli.exe");
     18const string("-l -info");
     19*/
    2320
    2421/*
    25  * getListOfDir(string rep, vector<string> &files)
    26  *
    27  * Gets the list of tjhe subdirectories in the directory rep and returns
    28  * them in a vector array of strings. Directories ";" and ".." are removed
    29  * from the vector array.
    30  */
    31 int getListOfDir(string rep, vector < string > &files) {
    32    DIR *dir;
    33    struct dirent *ent;
    34    int count;
     22getListOfDir(string rep, vector<string> &files)
    3523
    36    count = 0;
    37    dir = opendir(rep.c_str());
    38    if (dir != NULL) {
    39       /*
    40        * search for the files within directory
    41        */
    42       while ((ent = readdir(dir)) != NULL)
    43          if (ent->d_type == DT_DIR) {
    44             if (strcmp(ent->d_name, ".") *
    45                   strcmp(ent->d_name, "..")) {
    46                count++;
    47                files.push_back(string(ent->d_name));
    48                }
    49             }
    50       closedir(dir);
     24Gets the list of tjhe subdirectories in the directory rep and returns them
     25in a vector array of strings. Directories ";" and ".." are removed from the
     26vector array.
     27*/
     28int getListOfDir(string rep, vector<string> &files)
     29{
     30  DIR *dir;
     31  struct dirent *ent;
     32  int count;
     33
     34  count=0;
     35  dir = opendir (rep.c_str());
     36  if (dir != NULL) {
     37    /* search for the files within directory */
     38    while ((ent = readdir (dir)) != NULL)
     39      if (ent->d_type == DT_DIR) {
     40        if (strcmp(ent->d_name,".") * strcmp(ent->d_name,"..")) {
     41           count++;
     42           files.push_back(string(ent->d_name));
     43           }
     44        }
     45    closedir (dir);
     46    }
     47  else {
     48    cerr<<"Directory "<<rep.c_str()<<" not found"<<endl;
     49    }
     50   return count;
     51}
     52/* getListOfCommand(string rep, vector<string> &files)
     53Gets the list of commands in the ascii file fich. One command per line (no wrap)
     54in this first version.
     55*/
     56int getListOfCommand(const string & fich, vector<string> &commands)
     57{
     58  int count;
     59  string line;
     60 
     61  count=0;
     62  commands.clear();
     63
     64  std::ifstream infile (fich.c_str(), std::ios_base::in);
     65  while (getline(infile, line, '\n'))
     66  {
     67    // remove all trailing blanks
     68    while(line.size() >0 && isspace(line[line.size() -1])) line.erase(line.size() -1);
     69   
     70    // no empty line
     71    if (line.size() > 0) {
     72       commands.push_back (line);
     73       count++;
     74       }
     75  }
     76  return count;
     77}
     78
     79
     80
     81/*
     82Main program
     83*/
     84int main(int argc, char **argv)
     85{
     86vector<string> files = vector<string>();
     87string rep; //root directory where to find subdirectories
     88int rank, size, nbdir, stride;
     89int nbcmd;
     90/*
     91These two variables sets the command to launch in each sub-directory and the arguments required. The name of the root directory of the datasets is given as an argument when starting the MPIlauncher.
     92*/
     93string commandToLaunch;
     94string arguments;
     95string commandfile;
     96
     97string finalCommand; //command to execute
     98
     99MPI_Init (&argc, &argv);                      /* starts MPI */
     100MPI_Comm_rank (MPI_COMM_WORLD, &rank);        /* get current process id */
     101MPI_Comm_size (MPI_COMM_WORLD, &size);        /* get number of processes */
     102
     103if (argc<3){
     104   cout<<"USAGE:"<<endl;
     105   cout<<argv[0]<<" root_directory  command [arguments]"<<endl;
     106   cout<<argv[0]<<" -f  command_file"<<endl;
     107  }
     108else {   
     109   if (strcmp(argv[1],"-f")==0) {
     110     /* processing a command file now. */
     111     commandfile=argv[2];
     112     nbcmd=getListOfCommand(commandfile, files);
     113//     if (rank==0){
     114//       cout<<"Command file is "<<commandfile<<endl;
     115//       for (unsigned int i = 0;i < nbcmd;i++)
     116//          cout<<"["<<files[i]<<"]"<<endl; 
     117//       }
     118
     119
     120     //Number of commands should divide by number of cpus
     121//     if (nbcmd%size != 0) {
     122//        if(rank ==0)
     123//        cerr <<nbcmd<<" command(s) to process in "<<commandfile
     124//           <<" cannot fit on "<<size<<" processe(s)"
     125//           <<endl<<"FAILED"<<endl;
     126//        }
     127//     else {
     128//         execute the command
     129        int reste;
     130        stride=nbcmd/size;
     131        for (unsigned int i = 0;i < stride;i++) {
     132          cerr<<"Process "<<rank<<" execute "<<files[stride*rank+i] <<endl;
     133          system(files[stride*rank+i].c_str() );
     134          }
     135        //remaining command lines
     136        reste=nbcmd-stride*size;
     137        if(rank>0 && rank<=reste){
     138             cerr<<"Process "<<rank<<" execute "<<files[nbcmd-rank] <<endl;
     139             system(files[nbcmd-rank].c_str() );
     140             }
     141//      }
    51142      }
    52143   else {
    53       cerr << "Directory " << rep.c_str() << " not found" << endl;
    54       }
    55    return count;
    56    }
     144     /* processing a list of dir now */
     145     rep=string(argv[1]);
     146     commandToLaunch=string(argv[2]);
     147     for (int i=3; i<argc; i++) (arguments+=argv[i])+=" ";
     148       
     149     nbdir=getListOfDir(rep, files);
     150     
     151     //Number of dir should divide by number of cpus
     152     if (nbdir%size != 0) {
     153        if(rank ==0)
     154        cerr <<nbdir<<" dataset(s) to process cannot fit on "<<size<<" processes"<<endl<<"FAILED"<<endl;
     155        }
     156     else {
     157        // execute the command
     158        stride=nbdir/size;
     159        for (unsigned int i = 0;i < stride;i++) {
     160          string finalCommand("cd ");
     161          finalCommand +=rep;
     162          finalCommand +="/";
     163          finalCommand += files[stride*rank+i];
     164          finalCommand +=";";
     165          finalCommand +=commandToLaunch;
     166          finalCommand +=" ";
     167          finalCommand +=arguments;
     168          //cout<<"On "<<rank<<" execute "<<finalCommand<<endl;
     169          system(finalCommand.c_str() );
     170          }
     171        }
     172      } 
     173   }   
     174MPI_Finalize();
    57175
    58 /*
    59  * getListOfCommand(string rep, vector<string> &files) Gets the list of
    60  * commands in the ascii file fich. One command per line (no wrap) in this
    61  * first version.
    62  */
    63 int getListOfCommand(const string & fich, vector < string > &commands) {
    64    int count;
    65    string line;
    66 
    67    count = 0;
    68    commands.clear();
    69 
    70    std::ifstream infile(fich.c_str(), std::ios_base::in);
    71    while (getline(infile, line, '\n')) {
    72       // remove all trailing blanks
    73       while (line.size() > 0 && isspace(line[line.size() - 1]))
    74          line.erase(line.size() - 1);
    75 
    76       // no empty line
    77       if (line.size() > 0) {
    78          commands.push_back(line);
    79          count++;
    80          }
    81       }
    82    return count;
    83    }
    84 
    85 /*
    86  * Main program
    87  */
    88 int main(int argc, char **argv) {
    89    vector < string > files = vector < string > ();
    90    string rep; // root folder where to find subfolders
    91    int rank, size, nbdir, stride;
    92    int nbcmd;
    93    /*
    94     * These two variables sets the command to launch in each
    95     * sub-directory and the arguments required. The name of the root
    96     * directory of the datasets is given as an argument when starting the
    97     * MPIlauncher.
    98     */
    99    string commandToLaunch;
    100    string arguments;
    101    string commandfile;
    102 
    103    string finalCommand; // command to execute
    104 
    105    MPI_Init(&argc, &argv);      // starts MPI
    106    MPI_Comm_rank(MPI_COMM_WORLD, &rank); // get current process id
    107    MPI_Comm_size(MPI_COMM_WORLD, &size); // get number of processes
    108 
    109    if (argc < 3) {
    110       cout << "USAGE:" << endl;
    111       cout << argv[0] << " root_directory  command [arguments]" <<
    112            endl;
    113       cout << argv[0] << " -f  command_file" << endl;
    114       }
    115    else {
    116       if (strcmp(argv[1], "-f") == 0) {
    117          /*
    118           * processing a command file now.
    119           */
    120          commandfile = argv[2];
    121          nbcmd = getListOfCommand(commandfile, files);
    122          int reste;
    123          stride = nbcmd / size;
    124          for (unsigned int i = 0; i < stride; i++) {
    125             cerr << "Process " << rank << " execute " <<
    126                  files[stride * rank + i] << endl;
    127             system(files[stride * rank + i].c_str());
    128             }
    129          // remaining command lines
    130          reste = nbcmd - stride * size;
    131          if (rank > 0 && rank <= reste) {
    132             cerr << "Process " << rank << " execute " <<
    133                  files[nbcmd - rank] << endl;
    134             system(files[nbcmd - rank].c_str());
    135             }
    136          }
    137       else {
    138          /*
    139           * processing a list of dir now
    140           */
    141          rep = string(argv[1]);
    142          commandToLaunch = string(argv[2]);
    143          for (int i = 3; i < argc; i++)
    144             (arguments += argv[i]) += " ";
    145 
    146          nbdir = getListOfDir(rep, files);
    147 
    148          // Number of dir should divide by number of cpus
    149          if (nbdir % size != 0) {
    150             if (rank == 0)
    151                cerr << nbdir <<
    152                     " dataset(s) to process cannot fit on "
    153                     << size << " processes" << endl <<
    154                     "FAILED" << endl;
    155             }
    156          else {
    157             // execute the command
    158             stride = nbdir / size;
    159             for (unsigned int i = 0; i < stride; i++) {
    160                string finalCommand("cd ");
    161                finalCommand += rep;
    162                finalCommand += "/";
    163                finalCommand +=
    164                   files[stride * rank + i];
    165                finalCommand += ";";
    166                finalCommand += commandToLaunch;
    167                finalCommand += " ";
    168                finalCommand += arguments;
    169                // cout<<"On "<<rank<<" execute "<<finalCommand<<endl;
    170                system(finalCommand.c_str());
    171                }
    172             }
    173          }
    174       }
    175    MPI_Finalize();
    176 
    177    return 0;
    178    }
    179 
    180 /*
    181  * Documentation in Perl POD format (man perlpod)
    182  *
    183  * =head1 NAME
    184  *
    185  *  mpilauncher - parallel execute lot of small job via mpi
    186  *
    187  * =head1 SYNOPSIS
    188  *
    189  *  mpilauncher root_folder command [args]
    190  *  mpilauncher -f command_file
    191  *
    192  * =head1 DESCRIPTION
    193  *
    194  * C<mpilauncher> need to be executed inside an MPI environment (mpirun),
    195  * typically a cluster... Job process are divide by the number of core and
    196  * are launched on each core via the "system" command one after other.
    197  *
    198  * There is two case: jobs are list and define in a file (option -f) or
    199  * one command is launched inside a lot of folder. In this last case, you
    200  * need to give the root folder path in which you have all your subfolder...
    201  *
    202  *
    203  * =head1 SEE ALSO
    204  *
    205  * oar-dispatch, oar-parexec
    206  *
    207  *
    208  * =head1 AUTHORS
    209  *
    210  * Written by Patrick Begou - Gabriel Moreau, Grenoble - France
    211  *
    212  *
    213  * =head1 LICENSE AND COPYRIGHT
    214  *
    215  * GPL version 2 or later
    216  *
    217  * Copyright (C) 2011 Patrick Begou / LEGI - CNRS UMR 5519 - France
    218  *
    219  * =cut
    220  *
    221  */
     176return 0;
     177}
Note: See TracChangeset for help on using the changeset viewer.