Difference between revisions of "Federated Cloud OCCI to IM Migration"
(6 intermediate revisions by the same user not shown) | |||
Line 88: | Line 88: | ||
Creation of a VM using OCCI requires a <code>PUT</code> request to the <code>/compute</code> URL of the service with the right templates as mixins and a title. With IM you need to define a RADL description of the VM that includes similar information. See the comparison below: | Creation of a VM using OCCI requires a <code>PUT</code> request to the <code>/compute</code> URL of the service with the right templates as mixins and a title. With IM you need to define a RADL description of the VM that includes similar information. See the comparison below: | ||
OCCI: | |||
occi --endpoint <ENDPOINT> --auth x509 --user-cred $X509_USER_PROXY | occi --endpoint <ENDPOINT> --auth x509 --user-cred $X509_USER_PROXY | ||
--voms \ --action create --resource compute | --voms \ --action create --resource compute | ||
--attribute occi.core.title="<TITLE>" \ | --attribute occi.core.title="<TITLE>" \ | ||
--mixin <OS_TPL> --mixin <RES_TPL> | --mixin <OS_TPL> --mixin <RES_TPL> | ||
IM: | |||
<pre> | |||
$ cat infra.radl | |||
network public (outbound = 'yes' ) | network public (outbound = 'yes' ) | ||
system vm ( | system vm ( | ||
Line 110: | Line 107: | ||
) | ) | ||
deploy vm 1; | deploy vm 1; | ||
$ im_client.py -r https://appsgrycap.i3m.upv.es:31443/im/ -a auth.txt create infra.radl | |||
Secure connection with: https://appsgrycap.i3m.upv.es:31443/im/ | |||
Infrastructure successfully created with ID: 6a5f9a60-755e-11e8-b537-0a580af4031d | |||
</pre> | |||
== Contextualisation == | |||
OCCI allows two different contextualisation options: | |||
* public key: this is added to the default user in the image by cloud init | |||
* user data: blob interpreted by cloud-init to perform contextualisation actions | |||
These are specified during creation of the VM with the <code>--context</code> option to the rOCCI client. | |||
With IM you can have two contextualisation methods, cloud-init and ansible, both options are described with the <code>configure</code> section of your RADL. In any case IM creates a default user and a ssh-key pair for it. You can get the ssh-key with: | |||
The following example uses cloud-init: | |||
<pre> | |||
network private () | |||
system node ( | |||
cpu.count>=1 and | |||
... | |||
) | |||
configure node ( | |||
@begin | |||
runcmd: | |||
- [ wget, "http://slashdot.org", -O, /tmp/index.html ] | |||
@end | |||
) | |||
deploy node 1 | |||
contextualize ( | |||
system node configure node with cloud_init | |||
) | |||
</pre> | |||
= jOCCI = | |||
Users of jOCCI can use the Java API of IM, available at https://github.com/indigo-dc/im-java-api. See the following working example that creates, modifies and destroy a given infrastructure with that library: | |||
<pre> | |||
/* | |||
* (c) Copyright 2018 EGI Foundation | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* | |||
*/ | |||
package eu.egi.fedcloud; | |||
import java.util.concurrent.TimeUnit; | |||
import java.net.URL; | |||
import java.net.MalformedURLException; | |||
import java.nio.file.Paths; | |||
import es.upv.i3m.grycap.file.DoubleEscapeNewLinesFile; | |||
import es.upv.i3m.grycap.file.NoNullOrEmptyFile; | |||
import es.upv.i3m.grycap.file.Utf8File; | |||
import es.upv.i3m.grycap.im.InfrastructureManager; | |||
import es.upv.i3m.grycap.im.auth.credentials.AuthorizationHeader; | |||
import es.upv.i3m.grycap.im.auth.credentials.providers.OcciCredentials; | |||
import es.upv.i3m.grycap.im.auth.credentials.providers.ImCredentials; | |||
import es.upv.i3m.grycap.im.exceptions.ImClientException; | |||
import es.upv.i3m.grycap.im.pojo.InfrastructureUri; | |||
import es.upv.i3m.grycap.im.pojo.Property; | |||
import es.upv.i3m.grycap.im.rest.client.BodyContentType; | |||
import es.upv.i3m.grycap.im.States; | |||
import es.upv.i3m.grycap.im.VmProperties; | |||
public class ImExample | |||
{ | |||
public static void waitForInfrastructure(InfrastructureManager im, InfrastructureUri infraUri) throws ImClientException | |||
{ | |||
try { | |||
States st; | |||
do { | |||
TimeUnit.SECONDS.sleep(60); | |||
st = im.getInfrastructureState(infraUri.getInfrastructureId()).getEnumState(); | |||
System.out.println("STATE: " + st); | |||
} while (st != States.CONFIGURED && st != States.FAILED); | |||
// | |||
// show some info about the VM (e.g. IP) | |||
for (InfrastructureUri vmUri: im.getInfrastructureInfo(infraUri.getInfrastructureId()).getUris()) { | |||
String vmId = Paths.get((new URL(vmUri.getUri())).getPath()).getFileName().toString(); | |||
Property vmIP = im.getVmProperty(infraUri.getInfrastructureId(), vmId, VmProperties.NET_INTERFACE_0_IP); | |||
System.out.println("IP of VM " + vmId + ": " + vmIP.getValue()); | |||
} | |||
} catch (InterruptedException | MalformedURLException ex) { | |||
throw new RuntimeException(ex); | |||
} | |||
} | |||
public static void main (String[] args) | |||
{ | |||
// IM Server endpoint | |||
String IM_ENDPOINT = "https://servproject.i3m.upv.es:8800"; | |||
// This is a sample code, do not hardcode credentials in your code! | |||
String IM_USER = "<your user>"; // <= Change here! | |||
String IM_PASSWD = "<your password>"; // <= Change here! | |||
String X509_USER_PROXY = "/tmp/x509up_u501"; // <= Change to match your proxy location (voms-proxy-info -path) | |||
// Change as needed | |||
String OCCI_ENDPOINT = "https://sbgcloud.in2p3.fr:8787/occi1.2"; | |||
try { | |||
// Prepare authentication header to IM | |||
String proxy = new DoubleEscapeNewLinesFile( | |||
new NoNullOrEmptyFile( | |||
new Utf8File(Paths.get(X509_USER_PROXY)) | |||
)).read(); | |||
AuthorizationHeader ah = new AuthorizationHeader(); | |||
ah.addCredential(ImCredentials.buildCredentials() | |||
.withUsername(IM_USER) | |||
.withPassword(IM_PASSWD)); | |||
// This can be changed to other type of credentials as needed | |||
ah.addCredential(OcciCredentials.buildCredentials() | |||
.withHost(OCCI_ENDPOINT) | |||
.withProxy(proxy)); | |||
InfrastructureManager im = new InfrastructureManager(IM_ENDPOINT, ah); | |||
// List existing infras, should be empty... | |||
System.out.println("Existing infrastructures:"); | |||
for (InfrastructureUri uri: im.getInfrastructureList().getUris()) { | |||
System.out.println(uri.getInfrastructureId()); | |||
} | |||
// create a simple VM | |||
// Check RADL specification at http://imdocs.readthedocs.io/en/latest/radl.html | |||
String vmRadl = "network public (outbound = 'yes' )\n" | |||
+ "system vm (\n" | |||
// this is taken from AppDB IS | |||
+ "instance_type = '2' and\n" | |||
+ "net_interface.0.connection = 'public' and\n" | |||
+ "net_interface.0.dns_name = 'master' and\n" | |||
+ "disk.0.os.name = 'linux' and\n" | |||
// Again ids from AppDB IS | |||
+ "disk.0.image.url= ['https://sbgcloud.in2p3.fr:8787/occi1.2/9e7c0d7c-84c6-4b1f-95a1-c541b4a8310d'] and\n" | |||
+ "disk.0.os.credentials.username = 'cloudadm'\n" | |||
+ ")\n" | |||
+ "deploy vm 1\n"; | |||
InfrastructureUri infraUri = im.createInfrastructure(vmRadl, BodyContentType.RADL); | |||
waitForInfrastructure(im, infraUri); | |||
// modify the infrastructure, add one VM more | |||
String newVMRadl = "network public\n" | |||
+ "system vm\n" | |||
+ "deploy vm 1\n"; | |||
InfrastructureUri vm2Uri = im.addResource(infraUri.getInfrastructureId(), | |||
newVMRadl, BodyContentType.RADL).getUris().get(0); | |||
waitForInfrastructure(im, infraUri); | |||
// remove the second VM | |||
im.removeResource(infraUri.getInfrastructureId(), | |||
Paths.get((new URL(vm2Uri.getUri())).getPath()).getFileName().toString()); | |||
waitForInfrastructure(im, infraUri); | |||
// delete the whole infrastructure | |||
im.destroyInfrastructure(infraUri.getInfrastructureId()); | |||
} catch (ImClientException | MalformedURLException ex) { | |||
throw new RuntimeException(ex); | |||
} | |||
} | |||
} | |||
</pre> |
Latest revision as of 16:25, 21 June 2018
Overview | For users | For resource providers | Infrastructure status | Site-specific configuration | Architecture |
This page gives instructions on how to migrate from OCCI to IM for using EGI FedCloud resources
Setup of IM
UPV offers a public instance of IM, you can register a new account. Documentation is available at [1]
Once you have an account, you can either interact with IM via the web GUI or using the command line tool. Installation of the command line can be easily done with pip, a virtualenv
(and virtualenvwrapper
is recommended for the installation):
# create a "im" virtualenv $ mkvirtualenv im # now we are already in the im virtualenv, install im (im)$ pip install im_client
Whenever you want to use the client tool, just enter the virtualenv and it will be available on your path
$ workon im (im) $ which im_client.py /Users/enol/.virtualenvs/im/bin/im_client.py
Authentication
IM uses a file with the credentials used to access the IM server and the providers. See below an example with two OCCI providers:
$ cat ~/.im_client_auth id = im; type = InfrastructureManager; username = <your_im_user>; password = <your_im_password> id = occi_bari; type = OCCI; proxy = file(/tmp/x509up_u1000); host = http://cloud.recas.ba.infn.it:8787/occi/ id = occi_cesnet; type = OCCI; proxy = file(/tmp/x509up_u1000); host = https://carach5.ics.muni.cz:11443/
You can get the URLs for the OCCI endpoints at AppDB or GOCDB, check the Federated_Cloud_APIs_and_SDKs#Discovery_of_resources Discovery of Resources page for more information.
Commands issued need to have the URL of the server and the authentication file as parameters like this
(im) $ im_client.py -u http://servproject.i3m.upv.es:8899 -a ~/.im_client_auth <command> <command options>
For example, listing your deployed infrastructures
(im) $ im_client.py -u http://servproject.i3m.upv.es:8899 -a ~/.im_client_auth list Connected with: http://servproject.i3m.upv.es:8899 Infrastructure IDs: e3a6f3ca-0965-11e7-a466-300000000002
RADL
IM native language for specifying the deployments is called RADL (Resource and Application Description Language), it has sections to specify the VMs to be deployed and the configuration to be applied on them with tight integration with Ansible. The following example creates a VM on RECAS-BARI resource provider (notice the disk.0.image.url
, that contails the URL of the OCCI endpoint followed by the VM image id) of type 7
(these ids can be obtained via AppDB) using the grycap.swarm module from Ansible Galaxy.
network public (outbound = 'yes' ) system master ( instance_type = '7' and net_interface.0.connection = 'public' and net_interface.0.dns_name = 'master' and disk.0.os.name = 'linux' and disk.0.image.url= ['http://cloud.recas.ba.infn.it:8787/occi/43aa9fe8-0cde-4a35-87b8-eda324f8f1b8'] and disk.0.os.credentials.username = 'cloudadm' and disk.0.applications contains (name='ansible.modules.grycap.swarm') ) configure master ( @begin --- - roles: - { role: 'grycap.swarm' } @end ) deploy master 1
For more information, check the RADL documentation
Getting information about available images/flavors
While OCCI had the possibility to query the mixins at the site, the returned information is limited and does not include the needed details to determine which mixing is relevant for a given application. IM does not provide a way to query the images or flavors at the site, instead you should use AppDB, either via the browser or using AppDB IS APIs, to discover site capabilities.
Creating a VM
Creation of a VM using OCCI requires a PUT
request to the /compute
URL of the service with the right templates as mixins and a title. With IM you need to define a RADL description of the VM that includes similar information. See the comparison below:
OCCI:
occi --endpoint <ENDPOINT> --auth x509 --user-cred $X509_USER_PROXY --voms \ --action create --resource compute --attribute occi.core.title="<TITLE>" \ --mixin <OS_TPL> --mixin <RES_TPL>
IM:
$ cat infra.radl network public (outbound = 'yes' ) system vm ( instance_type = '<RES_TPL>' and net_interface.0.connection = 'public' and net_interface.0.dns_name = '<TITLE>' and disk.0.os.name = 'linux' and disk.0.image.url= ['<ENDPOINT>/<OS_TPL>'] and disk.0.os.credentials.username = 'cloudadm' ) deploy vm 1; $ im_client.py -r https://appsgrycap.i3m.upv.es:31443/im/ -a auth.txt create infra.radl Secure connection with: https://appsgrycap.i3m.upv.es:31443/im/ Infrastructure successfully created with ID: 6a5f9a60-755e-11e8-b537-0a580af4031d
Contextualisation
OCCI allows two different contextualisation options:
- public key: this is added to the default user in the image by cloud init
- user data: blob interpreted by cloud-init to perform contextualisation actions
These are specified during creation of the VM with the --context
option to the rOCCI client.
With IM you can have two contextualisation methods, cloud-init and ansible, both options are described with the configure
section of your RADL. In any case IM creates a default user and a ssh-key pair for it. You can get the ssh-key with:
The following example uses cloud-init:
network private () system node ( cpu.count>=1 and ... ) configure node ( @begin runcmd: - [ wget, "http://slashdot.org", -O, /tmp/index.html ] @end ) deploy node 1 contextualize ( system node configure node with cloud_init )
jOCCI
Users of jOCCI can use the Java API of IM, available at https://github.com/indigo-dc/im-java-api. See the following working example that creates, modifies and destroy a given infrastructure with that library:
/* * (c) Copyright 2018 EGI Foundation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * */ package eu.egi.fedcloud; import java.util.concurrent.TimeUnit; import java.net.URL; import java.net.MalformedURLException; import java.nio.file.Paths; import es.upv.i3m.grycap.file.DoubleEscapeNewLinesFile; import es.upv.i3m.grycap.file.NoNullOrEmptyFile; import es.upv.i3m.grycap.file.Utf8File; import es.upv.i3m.grycap.im.InfrastructureManager; import es.upv.i3m.grycap.im.auth.credentials.AuthorizationHeader; import es.upv.i3m.grycap.im.auth.credentials.providers.OcciCredentials; import es.upv.i3m.grycap.im.auth.credentials.providers.ImCredentials; import es.upv.i3m.grycap.im.exceptions.ImClientException; import es.upv.i3m.grycap.im.pojo.InfrastructureUri; import es.upv.i3m.grycap.im.pojo.Property; import es.upv.i3m.grycap.im.rest.client.BodyContentType; import es.upv.i3m.grycap.im.States; import es.upv.i3m.grycap.im.VmProperties; public class ImExample { public static void waitForInfrastructure(InfrastructureManager im, InfrastructureUri infraUri) throws ImClientException { try { States st; do { TimeUnit.SECONDS.sleep(60); st = im.getInfrastructureState(infraUri.getInfrastructureId()).getEnumState(); System.out.println("STATE: " + st); } while (st != States.CONFIGURED && st != States.FAILED); // // show some info about the VM (e.g. IP) for (InfrastructureUri vmUri: im.getInfrastructureInfo(infraUri.getInfrastructureId()).getUris()) { String vmId = Paths.get((new URL(vmUri.getUri())).getPath()).getFileName().toString(); Property vmIP = im.getVmProperty(infraUri.getInfrastructureId(), vmId, VmProperties.NET_INTERFACE_0_IP); System.out.println("IP of VM " + vmId + ": " + vmIP.getValue()); } } catch (InterruptedException | MalformedURLException ex) { throw new RuntimeException(ex); } } public static void main (String[] args) { // IM Server endpoint String IM_ENDPOINT = "https://servproject.i3m.upv.es:8800"; // This is a sample code, do not hardcode credentials in your code! String IM_USER = "<your user>"; // <= Change here! String IM_PASSWD = "<your password>"; // <= Change here! String X509_USER_PROXY = "/tmp/x509up_u501"; // <= Change to match your proxy location (voms-proxy-info -path) // Change as needed String OCCI_ENDPOINT = "https://sbgcloud.in2p3.fr:8787/occi1.2"; try { // Prepare authentication header to IM String proxy = new DoubleEscapeNewLinesFile( new NoNullOrEmptyFile( new Utf8File(Paths.get(X509_USER_PROXY)) )).read(); AuthorizationHeader ah = new AuthorizationHeader(); ah.addCredential(ImCredentials.buildCredentials() .withUsername(IM_USER) .withPassword(IM_PASSWD)); // This can be changed to other type of credentials as needed ah.addCredential(OcciCredentials.buildCredentials() .withHost(OCCI_ENDPOINT) .withProxy(proxy)); InfrastructureManager im = new InfrastructureManager(IM_ENDPOINT, ah); // List existing infras, should be empty... System.out.println("Existing infrastructures:"); for (InfrastructureUri uri: im.getInfrastructureList().getUris()) { System.out.println(uri.getInfrastructureId()); } // create a simple VM // Check RADL specification at http://imdocs.readthedocs.io/en/latest/radl.html String vmRadl = "network public (outbound = 'yes' )\n" + "system vm (\n" // this is taken from AppDB IS + "instance_type = '2' and\n" + "net_interface.0.connection = 'public' and\n" + "net_interface.0.dns_name = 'master' and\n" + "disk.0.os.name = 'linux' and\n" // Again ids from AppDB IS + "disk.0.image.url= ['https://sbgcloud.in2p3.fr:8787/occi1.2/9e7c0d7c-84c6-4b1f-95a1-c541b4a8310d'] and\n" + "disk.0.os.credentials.username = 'cloudadm'\n" + ")\n" + "deploy vm 1\n"; InfrastructureUri infraUri = im.createInfrastructure(vmRadl, BodyContentType.RADL); waitForInfrastructure(im, infraUri); // modify the infrastructure, add one VM more String newVMRadl = "network public\n" + "system vm\n" + "deploy vm 1\n"; InfrastructureUri vm2Uri = im.addResource(infraUri.getInfrastructureId(), newVMRadl, BodyContentType.RADL).getUris().get(0); waitForInfrastructure(im, infraUri); // remove the second VM im.removeResource(infraUri.getInfrastructureId(), Paths.get((new URL(vm2Uri.getUri())).getPath()).getFileName().toString()); waitForInfrastructure(im, infraUri); // delete the whole infrastructure im.destroyInfrastructure(infraUri.getInfrastructureId()); } catch (ImClientException | MalformedURLException ex) { throw new RuntimeException(ex); } } }