Alert.png The wiki is deprecated and due to be decommissioned by the end of September 2022.
The content is being migrated to other supports, new updates will be ignored and lost.
If needed you can get in touch with EGI SDIS team using operations @ egi.eu.

Difference between revisions of "HOWTO14 How to use COMPSs"

From EGIWiki
Jump to navigation Jump to search
Line 1,026: Line 1,026:
<br>
<br>


=== '''&nbsp;Tomcat http secure channel configuration:''' ===
=== '''&nbsp;Tomcat http secure channel configuration:''' ===


vi /home/pmes/apache-tomcat/conf/server.xml
vi /home/pmes/apache-tomcat/conf/server.xml  


*
<font color="#222222">In Tomcat service.xml configuration file add:</font>  
<font color="#222222">In Tomcat service.xml configuration file add:</font>


<font color="#222222"><font face="monospace, serif"><font size="2">&lt;Connector port=</font></font></font><font color="#000000"><font face="monospace, serif"><font size="2"><span lang="en-US">"</span></font></font></font><font color="#222222"><font face="monospace, serif"><font size="2">8443</font></font></font><font color="#000000"><font face="monospace, serif"><font size="2"><span lang="en-US">"</span></font></font></font><font color="#222222">&nbsp;</font><font color="#222222"><font face="monospace, serif"><font size="2">SSLEnabled="true</font></font></font><font color="#000000"><font face="monospace, serif"><font size="2"><span lang="en-US">"</span></font></font></font>
<font color="#222222"><font face="monospace, serif"><font size="2">&lt;Connector port=</font></font></font><font color="#000000"><font face="monospace, serif"><font size="2"><span lang="en-US">"</span></font></font></font><font color="#222222"><font face="monospace, serif"><font size="2">8443</font></font></font><font color="#000000"><font face="monospace, serif"><font size="2"><span lang="en-US">"</span></font></font></font><font color="#222222">&nbsp;</font><font color="#222222"><font face="monospace, serif"><font size="2">SSLEnabled="true</font></font></font><font color="#000000"><font face="monospace, serif"><font size="2"><span lang="en-US">"</span></font></font></font>  


<font color="#222222"><font face="monospace, serif"><font size="2">maxThreads=</font></font></font><font color="#000000"><font face="monospace, serif"><font size="2"><span lang="en-US">"</span></font></font></font><font color="#222222"><font face="monospace, serif"><font size="2">150</font></font></font><font color="#000000"><font face="monospace, serif"><font size="2"><span lang="en-US">"</span></font></font></font><font color="#222222">&nbsp;</font><font color="#222222"><font face="monospace, serif"><font size="2">scheme="https</font></font></font><font color="#000000"><font face="monospace, serif"><font size="2"><span lang="en-US">"</span></font></font></font><font color="#222222">&nbsp;</font><font color="#222222"><font face="monospace, serif"><font size="2">secure="true</font></font></font><font color="#000000"><font face="monospace, serif"><font size="2"><span lang="en-US">"</span></font></font></font>
<font color="#222222"><font face="monospace, serif"><font size="2">maxThreads=</font></font></font><font color="#000000"><font face="monospace, serif"><font size="2"><span lang="en-US">"</span></font></font></font><font color="#222222"><font face="monospace, serif"><font size="2">150</font></font></font><font color="#000000"><font face="monospace, serif"><font size="2"><span lang="en-US">"</span></font></font></font><font color="#222222">&nbsp;</font><font color="#222222"><font face="monospace, serif"><font size="2">scheme="https</font></font></font><font color="#000000"><font face="monospace, serif"><font size="2"><span lang="en-US">"</span></font></font></font><font color="#222222">&nbsp;</font><font color="#222222"><font face="monospace, serif"><font size="2">secure="true</font></font></font><font color="#000000"><font face="monospace, serif"><font size="2"><span lang="en-US">"</span></font></font></font>  


<font color="#222222"><font face="monospace, serif"><font size="2">clientAuth="false</font></font></font><font color="#000000"><font face="monospace, serif"><font size="2"><span lang="en-US">"</span></font></font></font><font color="#222222">&nbsp;</font><font color="#222222"><font face="monospace, serif"><font size="2">sslProtocol="TLS</font></font></font><font color="#000000"><font face="monospace, serif"><font size="2"><span lang="en-US">"</span></font></font></font><font color="#222222"></font>
<font color="#222222"><font face="monospace, serif"><font size="2">clientAuth="false</font></font></font><font color="#000000"><font face="monospace, serif"><font size="2"><span lang="en-US">"</span></font></font></font><font color="#222222">&nbsp;</font><font color="#222222"><font face="monospace, serif"><font size="2">sslProtocol="TLS</font></font></font><font color="#000000"><font face="monospace, serif"><font size="2"><span lang="en-US">"</span></font></font></font><font color="#222222"></font>  


<font color="#222222"><font face="monospace, serif"><font size="2">keystoreFile="/home/pmes/certs/keystore</font></font></font><font color="#000000"><font face="monospace, serif"><font size="2"><span lang="en-US">"</span></font></font></font>
<font color="#222222"><font face="monospace, serif"><font size="2">keystoreFile="/home/pmes/certs/keystore</font></font></font><font color="#000000"><font face="monospace, serif"><font size="2"><span lang="en-US">"</span></font></font></font>
 
<font color="#222222"><font face="monospace, serif"><font size="2">keystorePass="&lt;KeystorePassword&gt;"/&gt;</font></font></font>
 
<font color="#222222"></font>
 
<font color="#222222">Then start the service:</font>  


<font color="#222222"><font face="monospace, serif"><font size="2">keystorePass="&lt;KeystorePassword&gt;"/&gt;</font></font></font>


<font color="#222222">Then start the service:</font>


*
<font color="#222222"><font face="monospace, serif"><font size="2">source</font></font></font><font color="#222222">&nbsp;</font><font color="#222222"><font face="monospace, serif"><font size="2">~/.bashrc</font></font></font>
<font color="#222222"><font face="monospace, serif"><font size="2">source</font></font></font><font color="#222222">&nbsp;</font><font color="#222222"><font face="monospace, serif"><font size="2">~/.bashrc</font></font></font>


*
<font color="#222222"><font face="monospace, serif"><font size="2">/home/pmes/apache-tomcat/bin/catalina.sh start</font></font></font>
<font color="#222222"><font face="monospace, serif"><font size="2">/home/pmes/apache-tomcat/bin/catalina.sh start</font></font></font>
<br>

Revision as of 15:02, 11 June 2014

Introduction

 COMPSs is a programming framework, composed of a programming model and an execution runtime which supports it, whose main objective is to ease the development of applications for distributed environments. On the one hand, the COMPSs programming model aims to keep the programmers unaware of the execution environment and parallelization details. 

On the other hand, the COMPSs runtime is in charge of optimizing the performance of the application by exploiting its inherent concurrency. The runtime intercepts any call to a selected method creating a representative task and finding the data dependencies with all the previous ones that must be considered along the application run. 

One important feature of the COMPSs runtime is the ability to exploit the cloud elasticity by adjusting the amount of resources to the current workload. When the number of tasks is higher than the available cores, the runtime turns to the cloud looking for the provider that offers the resources that better fit the requirements of the application with a lower economical cost. Symmetrically, when the runtime detects an excess of resources it powers off unused instances in a cost-efficient way. Such decisions are based on information provided by the user who describes the details of the software images and instance templates available for every cloud provider. In order to make COMPSs interoperable with different providers, a common interface is used, which implements the specific cloud provider API. Currently, there exist connectors for Amazon EC2, Azure and for providers that implement the Open Cloud Computing Interface (OCCI) and the Open Virtualization Format (OVF) specifications for resource management.

Integration of COMPSs in the EGI FedCloud

In order to support the usage of EGI Federated Cloud IaaS resources, a specific connector is adopted to operate with the interfaces and protocols offered by EGI providers. The support for the VM management is implemented using the rOCCI client that transparently interoperates with the rOCCI servers deployed in the testbed on top of the different middlewares. The connector supports different authentication methods, including X509 type through VOMS proxy certificates. To setup each connector instance, the user indicates which virtual images and instance types are offered by the specific provider; thus, when the runtime asks for the creation of a VM, the connector selects the appropriate image and resource template according to the requirements (in terms of cpu, memory, disk, etc) and invokes the rOCCI client through Mixins.


Compss egi.png

Developing COMPSs applications

The COMPSs PMES

The Programming Model Enactment Service (PMES) is a web service that allows the execution of applications programmed through the COMPSs programming model. A client is used to contact an OGF BES compliant endpoint in order to submit the execution of the application. This request is expressed through a JSDL document containing the application name, the input parameters and data references. When a request is received, a virtual machine is requested to the Cloud Provider in order to deploy the application downloading its package from a storage location. Once the runtime machine is deployed the COMPSs application is remotely started. In its turn, the COMPSs runtime will schedule the tasks on a set of machines created on demand. This phase includes the staging of input files from remote locations as specified in the execution request. The COMPSs-PMES service can be also configured to boot a predefined number of VMs on the provider where the service is deployed; this solution allows the serving of requests that involve the execution of smaller operations in a reasonable time avoiding the overhead of VM creation; if the number of requests exceeds the available resources, the service is still able to dynamically deploy new instances in order to cope with the burst of load.

Compss pmes.png


Manual installation of COMPSs PMES

A PMES-COMPSs package is available for Debian and RedHat distributions

Debian based distributions:

Checkout the DEB package from: http://compss.bsc.es

Install package dependencies

  • sudo apt-get update
  • sudo apt-get install gdebi

Install PMES on the system:

  • sudo gdebi pmes-compss.deb
  • rm -rf pmes-compss.deb

RedHat based distributions (CentOS):

Checkout the RPM package from: http://compss.bsc.es

Install package dependencies

  • yum update

Install PMES on the system:

  • yum -y install pmes-compss.rpm
  • rm -rf pmes-compss.rpm

Deployment of the COMPSs PMES Virtual Appliance from the AppDB

A Virtual Appliance is available in the EGI AppDB appdb.egi.eu/store/vappliance/compss.pmes. This appliance comes with two flavors, set through contextualization scripts: one version offers the PMES service that deploys and starts COMPSs applications, while the COMPSs flavor is used by the COMPSs runtime to create VMs to execute the tasks. The cloud-init script for PMES also starts a Tomcat container from the /home/pmes/apache-tomcat folder. 


PMES configuration

PMES-COMPSs service configuration 

The configuration file in the available VA is located at: /home/pmes/conf/PMESConfig.xml

    <!-- PMES Configuration -->

    <PMES>

        <Address>https://took25.ics.muni.cz:8443/pmes/factory</Address>

        <User>pmes</User>

        <TempDir>/home/pmes/tmp</TempDir>

        <!-- Default SSH key type: DSA -->

        <!--<SSHKeyName>id_rsa</SSHKeyName>-->

        <UserDBFile>/home/pmes/conf/usersdb</UserDBFile>

        <!--<UsersMapFile>/home/pmes/conf/mapfile</UsersMapFile>-->

        <Scheduler>com.bsc.pmes.jobmanager.jobscheduler.SJNScheduler</Scheduler>


        <!-- Max allowed time per job (minutes) -->

        <MaxJobWallClock>360</MaxJobWallClock>


        <!-- Update logs time (minutes) -->

        <UpdateLogsTime>1</UpdateLogsTime>


        <!-- Job expiration control (minutes) -->

        <FinishedExpTime>120</FinishedExpTime>

        <CancelledExpTime>120</CancelledExpTime>

        <FailedExpTime>120</FailedExpTime>

        <ExpCheckTime>5</ExpCheckTime>

    </PMES>

   <!-- Resources Configuration -->

    <RESOURCES>

        <!-- CLOUD Provider Configuration -->

        <CLOUD>

           <Connectors>

                  <Provider>

                          <Name>CESNET</Name>

                          <Server>https://carach5.ics.muni.cz:11443/</Server>

                          <Implementation>com.bsc.pmes.connectors.rocci.ROCCI</Implementation>

                          <Type>main</Type>

                          <Properties>

                                <Property>

 Templates files are used to map the requirements of the applications to the type of resource templates (small, medium, large...) offered by a provider 

                                        <Name>templates</Name>

                                        <Value>/home/pmes/conf/connectors/rocci/templates/cesnet.xml</Value>

                                        <Deploy>/home/user/certs/</Deploy>

                                </Property>


                                <Property>

                                        <Name>voms</Name>

                                        <Value>true</Value>

                                </Property>


                                <Property>

                                        <Name>auth</Name>

                                        <Value>x509</Value>

                                </Property>


                                <Property>

                                        <Name>ca-path</Name>

                                        <Value>/etc/grid-security/certificates</Value>

                                </Property>


                                <Property>

A proxy file has to be provided to the PMES in order to create the VMs

                                        <Name>user-cred</Name>

                                        <Value>/home/pmes/certs/egi/x509up_u1002</Value>

                                        <Deploy>/home/user/certs/</Deploy>

                                </Property>

                          </Properties>

            <User>user</User>


            <!-- Maximum allowed VM creation time in minutes -->

            <MaxVMCreationTime>10</MaxVMCreationTime>


            <!-- Application deploy path on provider VM -->

            <DeploymentPath>/home/user/apps</DeploymentPath>


           <!-- Shared space mountpoint on virtual instances -->

           <WorkingDir>

                <Path>/tmp</Path>

                <Shared>false</Shared>

           </WorkingDir>


            <Resources>

              <!-- Defines a pool of pre-started VMs for speeding up job submissions -->

              <StaticPool>

The static pool is useful to serve those requests that don't involve complex computational runs. In the client a parameter is used to specify if the run is simple or complex. In the latter case the COMPSs runtime is deployed and used to elasticly execute the tasks of the applications.

                   <Resource image="uuid_egi_compss_debian_7_0_x86_64_0001_fedcloud_dukan_74">

                       <Instances>2</Instances>

                       <CPU>1</CPU>

                       <MEM>1.7</MEM>

                       <Disk>10</Disk>

                       <Arch>x86_64</Arch>

                   </Resource>

              </StaticPool>

<span style="line-height: 1.5em;" />

 This parameter defines the number of cores managed by the PMES instance'

              <!-- Maximum of system cores -->

              <MaxCores>36</MaxCores>

              <!-- Maximum of system memory -->

              <MaxMemory>135</MaxMemory>

          </Resources>


      </CLOUD>

    </RESOURCES>

<span style="line-height: 1.5em;" />

This section is used to specify the location of the COMPSs runtime and the specification of the VM needed to deploy it

    <!-- COMPSs Configuration -->

    <COMPSs>

       <Location>/opt/COMPSs/Runtime</Location>

       <Connector>integratedtoolkit.connectors.rocci.ROCCI</Connector>

       <WorkingDir>/home/user/IT</WorkingDir>


       <!-- COMPSs job monitoring frequency (seconds) -->

       <MonitorFreq>3</MonitorFreq>


       <!-- Master VM Specs -->

       <!--<Master>

           <CPU>1</CPU>

           <MEM>2.0</MEM>

           <Disk>2.0</Disk>

           <Arch>x86_64</Arch>

       </Master>-->


       <!-- COMPSs service resources -->

       <!--<Services>

           <Service>

               <Name>HmmerObjects</Name>

               <WSDL>http://bscgrid05.bsc.es:20390/hmmerobj/hmmerobj?wsdl</WSDL>

               <Namespace>http://hmmerobj.worker</Namespace>

               <Port>HmmerObjectsPort</Port>

               <Tasks>3</Tasks>

           </Service>

        </Services>-->


    </COMPSs>

   <!-- Storage adaptors map -->

    <ADAPTORS>

        <Adaptor>

            <Protocol>ftp</Protocol>

            <Implementation>com.bsc.pmes.jobmanager.datastager.FTPDataStager</Implementation>

        </Adaptor>

        <Adaptor>

            <Protocol>http</Protocol>

            <Implementation>com.bsc.pmes.jobmanager.datastager.CDMIDataStager</Implementation>

        </Adaptor>

        <Adaptor>

            <Protocol>gcubestorage</Protocol>

            <Implementation>com.bsc.pmes.jobmanager.datastager.GStorageDataStager</Implementation>

        </Adaptor>

    </ADAPTORS>


</PMESConfig>


 Tomcat http secure channel configuration:

vi /home/pmes/apache-tomcat/conf/server.xml

In Tomcat service.xml configuration file add:

<Connector port="8443" SSLEnabled="true"

maxThreads="150" scheme="https" secure="true"

clientAuth="false" sslProtocol="TLS"

keystoreFile="/home/pmes/certs/keystore"

keystorePass="<KeystorePassword>"/>

Then start the service:


source ~/.bashrc

/home/pmes/apache-tomcat/bin/catalina.sh start