Changes in / [30:20]


Ignore:
Location:
/trunk/oar
Files:
5 edited

Legend:

Unmodified
Added
Removed

  • /trunk/oar/Makefile

    r30 r20  
    22CC=mpiCC
    33CCFLAGS=-O2
    4 PODFLAG=-c 'User Contributed OAR Documentation' -r 'LEGI Trokata'
    5 
    64
    75.PHONY: all clean distclean
    86
    9 all: mpilauncher mpilauncher.1 oar-envsh.1 oar-dispatch.1 oar-parexec.1
     7all: mpilauncher
    108
    119clean:
    … …  
    2119%.o: %.cpp
    2220        $(CC) $(CCFLAGS) -c $<
    23 
    24 %.1: %.cpp
    25         sed -re 's#^[/ ]\* ?##; s#^// ?##' $< | pod2man -n $* $(PODFLAG) > $@
    26 
    27 %.1: %
    28         pod2man -n $* $(PODFLAG) $< > $@
    29 
    30 # astyle --style=banner -s3 mpilauncher.cpp

  • /trunk/oar/mpilauncher.cpp

    r30 r20  
    1414
    1515/*
    16  * These two variables sets the command to launch in each sub-directory
    17  * and the arguments required. The name of the root directory of the
    18  * datasets is given as an argument when starting the MPIlauncher. const
    19  * string
    20  * commandToLaunch("$HOME/SOURCES/MES-PROJETS/MPILAUNCHER/appli.exe");
    21  * const string("-l -info");
    22  */
     16These two variables sets the command to launch in each sub-directory and the arguments required. The name of the root directory of the datasets is given as an argument when starting the MPIlauncher.
     17const string commandToLaunch("/HA/sources/begou/SOURCES/MES-PROJETS/MPILAUNCHER/appli.exe");
     18const string("-l -info");
     19*/
    2320
    2421/*
    25  * getListOfDir(string rep, vector<string> &files)
    26  *
    27  * Gets the list of tjhe subdirectories in the directory rep and returns
    28  * them in a vector array of strings. Directories ";" and ".." are removed
    29  * from the vector array.
    30  */
    31 int getListOfDir(string rep, vector < string > &files) {
    32    DIR *dir;
    33    struct dirent *ent;
    34    int count;
     22getListOfDir(string rep, vector<string> &files)
    3523
    36    count = 0;
    37    dir = opendir(rep.c_str());
    38    if (dir != NULL) {
    39       /*
    40        * search for the files within directory
    41        */
    42       while ((ent = readdir(dir)) != NULL)
    43          if (ent->d_type == DT_DIR) {
    44             if (strcmp(ent->d_name, ".") *
    45                   strcmp(ent->d_name, "..")) {
    46                count++;
    47                files.push_back(string(ent->d_name));
    48                }
    49             }
    50       closedir(dir);
     24Gets the list of tjhe subdirectories in the directory rep and returns them
     25in a vector array of strings. Directories ";" and ".." are removed from the
     26vector array.
     27*/
     28int getListOfDir(string rep, vector<string> &files)
     29{
     30  DIR *dir;
     31  struct dirent *ent;
     32  int count;
     33
     34  count=0;
     35  dir = opendir (rep.c_str());
     36  if (dir != NULL) {
     37    /* search for the files within directory */
     38    while ((ent = readdir (dir)) != NULL)
     39      if (ent->d_type == DT_DIR) {
     40        if (strcmp(ent->d_name,".") * strcmp(ent->d_name,"..")) {
     41           count++;
     42           files.push_back(string(ent->d_name));
     43           }
     44        }
     45    closedir (dir);
     46    }
     47  else {
     48    cerr<<"Directory "<<rep.c_str()<<" not found"<<endl;
     49    }
     50   return count;
     51}
     52/* getListOfCommand(string rep, vector<string> &files)
     53Gets the list of commands in the ascii file fich. One command per line (no wrap)
     54in this first version.
     55*/
     56int getListOfCommand(const string & fich, vector<string> &commands)
     57{
     58  int count;
     59  string line;
     60 
     61  count=0;
     62  commands.clear();
     63
     64  std::ifstream infile (fich.c_str(), std::ios_base::in);
     65  while (getline(infile, line, '\n'))
     66  {
     67    // remove all trailing blanks
     68    while(line.size() >0 && isspace(line[line.size() -1])) line.erase(line.size() -1);
     69   
     70    // no empty line
     71    if (line.size() > 0) {
     72       commands.push_back (line);
     73       count++;
     74       }
     75  }
     76  return count;
     77}
     78
     79
     80
     81/*
     82Main program
     83*/
     84int main(int argc, char **argv)
     85{
     86vector<string> files = vector<string>();
     87string rep; //root directory where to find subdirectories
     88int rank, size, nbdir, stride;
     89int nbcmd;
     90/*
     91These two variables sets the command to launch in each sub-directory and the arguments required. The name of the root directory of the datasets is given as an argument when starting the MPIlauncher.
     92*/
     93string commandToLaunch;
     94string arguments;
     95string commandfile;
     96
     97string finalCommand; //command to execute
     98
     99MPI_Init (&argc, &argv);                      /* starts MPI */
     100MPI_Comm_rank (MPI_COMM_WORLD, &rank);        /* get current process id */
     101MPI_Comm_size (MPI_COMM_WORLD, &size);        /* get number of processes */
     102
     103if (argc<3){
     104   cout<<"USAGE:"<<endl;
     105   cout<<argv[0]<<" root_directory  command [arguments]"<<endl;
     106   cout<<argv[0]<<" -f  command_file"<<endl;
     107  }
     108else {   
     109   if (strcmp(argv[1],"-f")==0) {
     110     /* processing a command file now. */
     111     commandfile=argv[2];
     112     nbcmd=getListOfCommand(commandfile, files);
     113//     if (rank==0){
     114//       cout<<"Command file is "<<commandfile<<endl;
     115//       for (unsigned int i = 0;i < nbcmd;i++)
     116//          cout<<"["<<files[i]<<"]"<<endl; 
     117//       }
     118
     119
     120     //Number of commands should divide by number of cpus
     121//     if (nbcmd%size != 0) {
     122//        if(rank ==0)
     123//        cerr <<nbcmd<<" command(s) to process in "<<commandfile
     124//           <<" cannot fit on "<<size<<" processe(s)"
     125//           <<endl<<"FAILED"<<endl;
     126//        }
     127//     else {
     128//         execute the command
     129        int reste;
     130        stride=nbcmd/size;
     131        for (unsigned int i = 0;i < stride;i++) {
     132          cerr<<"Process "<<rank<<" execute "<<files[stride*rank+i] <<endl;
     133          system(files[stride*rank+i].c_str() );
     134          }
     135        //remaining command lines
     136        reste=nbcmd-stride*size;
     137        if(rank>0 && rank<=reste){
     138             cerr<<"Process "<<rank<<" execute "<<files[nbcmd-rank] <<endl;
     139             system(files[nbcmd-rank].c_str() );
     140             }
     141//      }
    51142      }
    52143   else {
    53       cerr << "Directory " << rep.c_str() << " not found" << endl;
    54       }
    55    return count;
    56    }
     144     /* processing a list of dir now */
     145     rep=string(argv[1]);
     146     commandToLaunch=string(argv[2]);
     147     for (int i=3; i<argc; i++) (arguments+=argv[i])+=" ";
     148       
     149     nbdir=getListOfDir(rep, files);
     150     
     151     //Number of dir should divide by number of cpus
     152     if (nbdir%size != 0) {
     153        if(rank ==0)
     154        cerr <<nbdir<<" dataset(s) to process cannot fit on "<<size<<" processes"<<endl<<"FAILED"<<endl;
     155        }
     156     else {
     157        // execute the command
     158        stride=nbdir/size;
     159        for (unsigned int i = 0;i < stride;i++) {
     160          string finalCommand("cd ");
     161          finalCommand +=rep;
     162          finalCommand +="/";
     163          finalCommand += files[stride*rank+i];
     164          finalCommand +=";";
     165          finalCommand +=commandToLaunch;
     166          finalCommand +=" ";
     167          finalCommand +=arguments;
     168          //cout<<"On "<<rank<<" execute "<<finalCommand<<endl;
     169          system(finalCommand.c_str() );
     170          }
     171        }
     172      } 
     173   }   
     174MPI_Finalize();
    57175
    58 /*
    59  * getListOfCommand(string rep, vector<string> &files) Gets the list of
    60  * commands in the ascii file fich. One command per line (no wrap) in this
    61  * first version.
    62  */
    63 int getListOfCommand(const string & fich, vector < string > &commands) {
    64    int count;
    65    string line;
    66 
    67    count = 0;
    68    commands.clear();
    69 
    70    std::ifstream infile(fich.c_str(), std::ios_base::in);
    71    while (getline(infile, line, '\n')) {
    72       // remove all trailing blanks
    73       while (line.size() > 0 && isspace(line[line.size() - 1]))
    74          line.erase(line.size() - 1);
    75 
    76       // no empty line
    77       if (line.size() > 0) {
    78          commands.push_back(line);
    79          count++;
    80          }
    81       }
    82    return count;
    83    }
    84 
    85 /*
    86  * Main program
    87  */
    88 int main(int argc, char **argv) {
    89    vector < string > files = vector < string > ();
    90    string rep; // root folder where to find subfolders
    91    int rank, size, nbdir, stride;
    92    int nbcmd;
    93    /*
    94     * These two variables sets the command to launch in each
    95     * sub-directory and the arguments required. The name of the root
    96     * directory of the datasets is given as an argument when starting the
    97     * MPIlauncher.
    98     */
    99    string commandToLaunch;
    100    string arguments;
    101    string commandfile;
    102 
    103    string finalCommand; // command to execute
    104 
    105    MPI_Init(&argc, &argv);      // starts MPI
    106    MPI_Comm_rank(MPI_COMM_WORLD, &rank); // get current process id
    107    MPI_Comm_size(MPI_COMM_WORLD, &size); // get number of processes
    108 
    109    if (argc < 3) {
    110       cout << "USAGE:" << endl;
    111       cout << argv[0] << " root_directory  command [arguments]" <<
    112            endl;
    113       cout << argv[0] << " -f  command_file" << endl;
    114       }
    115    else {
    116       if (strcmp(argv[1], "-f") == 0) {
    117          /*
    118           * processing a command file now.
    119           */
    120          commandfile = argv[2];
    121          nbcmd = getListOfCommand(commandfile, files);
    122          int reste;
    123          stride = nbcmd / size;
    124          for (unsigned int i = 0; i < stride; i++) {
    125             cerr << "Process " << rank << " execute " <<
    126                  files[stride * rank + i] << endl;
    127             system(files[stride * rank + i].c_str());
    128             }
    129          // remaining command lines
    130          reste = nbcmd - stride * size;
    131          if (rank > 0 && rank <= reste) {
    132             cerr << "Process " << rank << " execute " <<
    133                  files[nbcmd - rank] << endl;
    134             system(files[nbcmd - rank].c_str());
    135             }
    136          }
    137       else {
    138          /*
    139           * processing a list of dir now
    140           */
    141          rep = string(argv[1]);
    142          commandToLaunch = string(argv[2]);
    143          for (int i = 3; i < argc; i++)
    144             (arguments += argv[i]) += " ";
    145 
    146          nbdir = getListOfDir(rep, files);
    147 
    148          // Number of dir should divide by number of cpus
    149          if (nbdir % size != 0) {
    150             if (rank == 0)
    151                cerr << nbdir <<
    152                     " dataset(s) to process cannot fit on "
    153                     << size << " processes" << endl <<
    154                     "FAILED" << endl;
    155             }
    156          else {
    157             // execute the command
    158             stride = nbdir / size;
    159             for (unsigned int i = 0; i < stride; i++) {
    160                string finalCommand("cd ");
    161                finalCommand += rep;
    162                finalCommand += "/";
    163                finalCommand +=
    164                   files[stride * rank + i];
    165                finalCommand += ";";
    166                finalCommand += commandToLaunch;
    167                finalCommand += " ";
    168                finalCommand += arguments;
    169                // cout<<"On "<<rank<<" execute "<<finalCommand<<endl;
    170                system(finalCommand.c_str());
    171                }
    172             }
    173          }
    174       }
    175    MPI_Finalize();
    176 
    177    return 0;
    178    }
    179 
    180 /*
    181  * Documentation in Perl POD format (man perlpod)
    182  *
    183  * =head1 NAME
    184  *
    185  *  mpilauncher - parallel execute lot of small job via mpi
    186  *
    187  * =head1 SYNOPSIS
    188  *
    189  *  mpilauncher root_folder command [args]
    190  *  mpilauncher -f command_file
    191  *
    192  * =head1 DESCRIPTION
    193  *
    194  * C<mpilauncher> need to be executed inside an MPI environment (mpirun),
    195  * typically a cluster... Job process are divide by the number of core and
    196  * are launched on each core via the "system" command one after other.
    197  *
    198  * There is two case: jobs are list and define in a file (option -f) or
    199  * one command is launched inside a lot of folder. In this last case, you
    200  * need to give the root folder path in which you have all your subfolder...
    201  *
    202  *
    203  * =head1 SEE ALSO
    204  *
    205  * oar-dispatch, oar-parexec
    206  *
    207  *
    208  * =head1 AUTHORS
    209  *
    210  * Written by Patrick Begou - Gabriel Moreau, Grenoble - France
    211  *
    212  *
    213  * =head1 LICENSE AND COPYRIGHT
    214  *
    215  * GPL version 2 or later
    216  *
    217  * Copyright (C) 2011 Patrick Begou / LEGI - CNRS UMR 5519 - France
    218  *
    219  * =cut
    220  *
    221  */
     176return 0;
     177}

  • /trunk/oar/oar-dispatch

    r30 r20  
    106106=head1 SYNOPSIS
    107107
    108  oar-dispatch [--task integer] [--overload real] --file filepath [--verbose]
     108 oar-dispatch [--core integer] [--overload real] --file filepath [--verbose]
    109109 oar-dispatch --help
    110110
    111111=head1 OPTIONS
    112112
    113 =over 12
     113 --task number of task to do in parallel.
     114                        default to line number of file OAR_NODE_FILE.
     115 
     116 --overload     number of OAR job to create / number of task.
     117                        Some job are create in advance to start whenever it's possible.
     118                        1.1 by default.
    114119
    115 =item B<[-t|--task integer]>
     120 --file file name which content OAR job list
    116121
    117 Number of task to do in parallel.
    118 Default to the line number of the file OAR_NODE_FILE.
     122 --verbose
    119123 
    120 =item B<[-o|--overload real]>
    121 
    122 Number of OAR job to create / number of task.
    123 Some job are create in advance to start whenever it's possible.
    124 1.1 by default.
    125 
    126 =item B<[-f|--file filepath]>
    127 
    128 File name which content OAR job list
    129 
    130 =item B<[-v|--verbose]>
    131  
    132 =item B<[-h|--help]>
    133 
    134 =back
     124 --help
    135125
    136126Input job file name content can have
    … …  
    143133just after C<oarsub>.
    144134
    145 =head1 EXAMPLE
    146 
    147 Example where the file F<$HOME/test/subjob.txt> is a list of OAR script job (and can be executable but not need here).
     135Example where F<$HOME/test/subjob.txt> is a list of OAR script job (and can be executable but not need here).
    148136
    149137 oarsub -n test -l /core=1,walltime=00:05:00 $HOME/test/subjob1.oar
    … …  
    173161
    174162
    175 =head1 SEE ALSO
    176 
    177 oar-parexec, mpilauncher
    178 
    179 
    180163=head1 AUTHORS
    181164
    182 Written by Gabriel Moreau, Grenoble - France
    183 
    184 
    185 =head1 LICENSE AND COPYRIGHT
    186 
    187 GPL version 2 or later and Perl equivalent
    188 
    189 Copyright (C) 2011 Gabriel Moreau / LEGI - CNRS UMR 5519 - France
    190 
     165Gabriel Moreau (C) 2011

  • /trunk/oar/oar-envsh

    r30 r20  
    2828
    2929exec /opt/oar/current/bin/oarsh $HOST "$MYENV $@"
    30 
    31 exit
    32 
    33 ################################################################
    34 
    35 Documentation in Perl POD format (man perlpod)
    36 
    37 =head1 NAME
    38 
    39  oar-envsh - oarsh with env variable transmit
    40 
    41 =head1 SYNOPSIS
    42 
    43  oar-envsh node command
    44 
    45 =head1 DESCRIPTION
    46 
    47 C<oar-envsh> need to be executed inside an OAR cluster.
    48 It's a simple wrapper around C<oarsh> command
    49 which is a wrapper around C<ssh> command!
    50 
    51 With C<oar-envsh>, almost all env variable are export
    52 in the new shell except the following:
    53 
    54  USER TERM OAR* SGE_* LS_COLORS
    55  ENV BASH_ENV HOSTNAME LOGNAME MAIL
    56  MANPATH OMPI_MCA_plm_rsh_agent PWD
    57  SHELL SSH_* SUDO_COMMAND= HOME DISPLAY
    58  SHLVL
    59 
    60 Alway use C<oarsh>,
    61 only use C<oar-envsh> if really needed!
    62 
    63 
    64 =head1 SEE ALSO
    65 
    66 oarsh
    67 
    68 
    69 =head1 AUTHORS
    70 
    71 Written by :
    72 
    73  Nicolas Capit, Grenoble - France
    74  Patrick Begou - Gabriel Moreau, Grenoble - France
    75 
    76 
    77 =head1 LICENSE AND COPYRIGHT
    78 
    79 GPL version 2 or later
    80 
    81 Copyright (C) 2011 Patrick Begou / LEGI - CNRS UMR 5519 - France
    82 
    83 =cut

  • /trunk/oar/oar-parexec

    r30 r20  
    2929   ) || pod2usage( -verbose => 0 );
    3030pod2usage( -verbose => 2 ) if $help;
    31 pod2usage( -verbose => 2 ) if not -e $file;
     31pod2usage( -verbose => 2 ) if -e $file;
    3232
    3333my @job = ();
    … …  
    147147 --help
    148148
    149 
    150 =head1 DESCRIPTION
    151 
    152 C<oar-parexec> need to be executed inside an OAR job environment.
    153 because it need the two environment variable that OAR define by
    154 default:
    155 
    156  OAR_NODE_FILE path to a file which content one node by line
    157 
    158  OAR_WORKDIR   dir to launch job and do a chdir inside
    159 
    160 Content for the job file (option C<--file>) could have:
     149File name content can have
    161150
    162151 - empty line
    … …  
    179168 oarsub -n test -l /core=6,walltime=00:35:00 "oar-parexec -f ./subjob.list.txt"
    180169
    181 
    182 =head1 SEE ALSO
    183 
    184 oar-dispatch, mpilauncher
    185 
    186 
    187170=head1 AUTHORS
    188171
    189 Written by Gabriel Moreau, Grenoble - France
     172Gabriel Moreau (C) 2011
    190173
    191 
    192 =head1 LICENSE AND COPYRIGHT
    193 
    194 GPL version 2 or later and Perl equivalent
    195 
    196 Copyright (C) 2011 Gabriel Moreau / LEGI - CNRS UMR 5519 - France
    197 
Note: See TracChangeset for help on using the changeset viewer.