diff --git a/.github/ISSUE_TEMPLATES/documentation-feedback.md b/.github/ISSUE_TEMPLATES/documentation-feedback.md new file mode 100644 index 0000000..76aa33a --- /dev/null +++ b/.github/ISSUE_TEMPLATES/documentation-feedback.md @@ -0,0 +1,35 @@ +--- +name: Documentation Feedback +about: Report errors, suggest improvements, or request clarification in documentation +title: '[DOCS] ' +labels: documentation +assignees: '' +--- + +## Documentation Issue + +**Page URL:** + + +**Issue Type:** +- [ ] Error/Typo +- [ ] Unclear explanation +- [ ] Missing information +- [ ] Outdated content +- [ ] Other + +**Description:** + + +**Suggested Fix:** + + +**User Context:** +- [ ] Chameleon PI +- [ ] Student user +- [ ] Researcher/Faculty +- [ ] External collaborator +- [ ] Other: ___________ + +**Additional Context:** + \ No newline at end of file diff --git a/source/_static/css/style.css b/source/_static/css/style.css index 97cf30d..d735320 100644 --- a/source/_static/css/style.css +++ b/source/_static/css/style.css @@ -3,14 +3,59 @@ font-weight: bold; } -.figure.screenshot { - max-width: 440px; - border: 1px solid #eeeeee; - box-shadow: 0 .1rem .5rem rgba(0, 0, 0, 0.1); - padding: .5rem; -} .figure.screenshot .caption { +/* Uniform styling for all documentation images */ +img { + max-width: 800px; + max-height: 500px; + width: auto; + height: auto; + object-fit: contain; + border: 1px solid #d1d5da; + border-radius: 4px; + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1); + margin: 1rem auto; + display: block; + background: #fff; + padding: 0.5rem; +} + +img:hover { + box-shadow: 0 4px 12px rgba(0, 0, 0, 0.15); +} + +/* Keep figure styling for captions when figures are used */ +.figure { + max-width: 820px; + margin: 2rem auto; + text-align: center; +} + +.figure img { + margin: 0; +} + +.figure .caption { text-align: left; - margin-top: .5rem; - font-size: 12px; - line-height: 1em; + margin-top: 0.75rem; + font-size: 0.85rem; + color: #586069; + line-height: 1.3; + font-style: normal; +} + +/* Responsive behavior */ +@media (max-width: 768px) { + img { + max-width: 95%; + padding: 0.25rem; + margin: 0.5rem auto; + } + + .figure { + max-width: 95%; + } + + .figure .caption { + font-size: 0.8rem; + } } diff --git a/source/_static/js/rate-the-docs-config.js b/source/_static/js/rate-the-docs-config.js new file mode 100644 index 0000000..697f691 --- /dev/null +++ b/source/_static/js/rate-the-docs-config.js @@ -0,0 +1,24 @@ +window.rateTheDocs = { + // Send events to your existing GA property + onRate: function(rating, page) { + if (typeof gtag !== 'undefined') { + gtag('event', 'rate_documentation', { + 'rating': rating, + 'page_title': document.title, + 'page_location': window.location.href, + 'custom_parameter': 'helpful_' + (rating > 3 ? 'yes' : 'no') + }); + } + }, + + onFeedback: function(feedback, rating, page) { + if (typeof gtag !== 'undefined') { + gtag('event', 'documentation_feedback', { + 'feedback_text': feedback.substring(0, 100), // Truncate for GA + 'rating': rating, + 'page_title': document.title, + 'page_location': window.location.href + }); + } + } +}; \ No newline at end of file diff --git a/source/_templates/footer.html b/source/_templates/footer.html new file mode 100644 index 0000000..4e53e31 --- /dev/null +++ b/source/_templates/footer.html @@ -0,0 +1,8 @@ +
+

+ Improve this page: + + Report an issue or suggest improvements + +

+
\ No newline at end of file diff --git a/source/conf.py b/source/conf.py index 980949d..4b3919d 100644 --- a/source/conf.py +++ b/source/conf.py @@ -52,7 +52,7 @@ # General information about the project. project = 'Chameleon Cloud Documentation' -copyright = '2018, Chameleon Cloud' +copyright = '2025, Chameleon Cloud' author = 'Chameleon Cloud' # The version info for the project you're documenting, acts as replacement for @@ -85,6 +85,9 @@ rst_prolog = """ .. |CHI@TACC| replace:: `CHI@TACC `__ .. |CHI@UC| replace:: `CHI@UC `__ +.. |CHI@NCAR| replace:: `CHI@NCAR `__ +.. |CHI@Edge| replace:: `CHI@Edge `__ +.. |CHI@NRP| replace:: `CHI@NRP `__ .. |KVM@TACC| replace:: `KVM@TACC `__ .. |Appliance Catalog| replace:: `Appliance Catalog `__ .. |Home| replace:: `chameleoncloud.org `__ @@ -105,9 +108,6 @@ #html_theme = 'sphinxdoc' html_theme = 'sphinx_rtd_theme' -def setup(app): - app.add_css_file("css/style.css") - # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. @@ -196,3 +196,18 @@ def setup(app): # author, 'ChameleonCloudDocumentation', 'One line description of project.', # 'Miscellaneous'), # ] + +html_context = { + "display_github": True, + "github_user": "ChameleonCloud", + "github_repo": "docs", + "github_version": "master", + "conf_py_path": "/source/", +} + + + +def setup(app): + app.add_css_file("css/style.css") + app.add_js_file("https://unpkg.com/rate-the-docs") + app.add_js_file("js/rate-the-docs-config.js") diff --git a/source/contents.rst b/source/contents.rst index b4d5914..888cab3 100644 --- a/source/contents.rst +++ b/source/contents.rst @@ -3,14 +3,14 @@ Welcome to Chameleon ===================== .. toctree:: - :maxdepth: 1 + :maxdepth: 2 :caption: Introduction index getting-started/index .. toctree:: - :maxdepth: 1 + :maxdepth: 2 :caption: Users and Projects user/federation @@ -20,29 +20,28 @@ Welcome to Chameleon user/help .. toctree:: - :maxdepth: 1 + :maxdepth: 2 :caption: Testbed interfaces - technical/gui - technical/cli - technical/jupyter + technical/gui/index + technical/cli/index + technical/jupyter/index .. toctree:: - :maxdepth: 1 + :maxdepth: 2 :caption: Technical guide technical/index - technical/discovery - technical/reservations - technical/baremetal - technical/images - technical/metrics - technical/complex - technical/swift - technical/shares - technical/networks - technical/fpga - technical/ep - technical/sharing - technical/daypass - technical/kvm + technical/discovery/index + technical/reservations/index + technical/baremetal/index + technical/images/index + technical/power_monitoring/index + technical/complex/index + technical/swift/index + technical/shares/index + technical/networks/index + technical/fpga/index + technical/sharing/index + technical/daypass/index + technical/kvm/index diff --git a/source/getting-started/index.rst b/source/getting-started/index.rst index 747cdce..7dfe162 100644 --- a/source/getting-started/index.rst +++ b/source/getting-started/index.rst @@ -23,7 +23,7 @@ .. _getting-started: ================ -Getting started +Getting Started ================ .. image:: ../_static/imgs/getting_started/chameleon-at-work.jpg @@ -57,126 +57,49 @@ Let's get started! Pre-Req: Creating a Chameleon Account ===================================== -Before you can get started using Chameleon, you will need to create a **user -account**. +Before you can use Chameleon, you need to create a free **user account**. -Don't worry! Creating an account is easy and free. We use Globus_, a platform -for secure and reliable data transfer, sharing, and management across various -storage systems and computing environments, to enable SSO on Chameleon. - -To create an account (and to log in for future sessions), go to the Chameleon_ -home page and click on the "Log In" button at the top right corner of the -window. +Creating an account is easy! Simply go to the Chameleon_ home page and click +**"Log In"** at the top right corner. You'll be redirected to our authentication +page where you can sign up using your institution credentials, Google account, +or other federated identity options. .. image:: ../_static/imgs/getting_started/chameleon-login.png -You will then be redirected to separate authentication page, with an option to -"Sign in via federated login." We recommend using this option to create your -account. - -Clicking on the federated login button will then take you to the Globus website -where you can enter your institution details. You will then be prompted to sign -in through your institution as you would normally. You can also create an -account directly with Globus using the `Globus ID -`_ service. This option is useful in the event you -don't want anything external. - -.. image:: ../_static/imgs/getting_started/globus-login.png +**We recommend using federated login** (sign in via your institution) as it's the +fastest way to get started and helps with PI eligibility verification if you plan +to create projects later. -.. warning:: - You may not find your institution on Globus. If so, you can still create an - account with another identity, such as GitHub, Google, or ORCID iD. However, - we encourage users to sign up using their institution, as it helps the - Chameleon operators verify user identity, which is an essential step - to getting Principal Investigator status on the testbed. More on this below! - (:ref:`Read more about logging into Chameleon via federated login `.) +.. note:: + **Need detailed login help?** See our comprehensive :doc:`federated authentication guide <../user/federation>` + for step-by-step instructions, authentication options, and troubleshooting tips. -.. attention:: - When creating an account, you will be asked to accept `terms and conditions - `_ of use. Please, - note that as part of those terms and conditions you are requested to - acknowledge Chameleon in publications produced using the testbed. See our FAQ - for information on `how to reference Chameleon in your publications - `_ - and the suggested `acknowledgement text - `_. - -Once you log in, you will be able to :ref:`edit your Chameleon profile -` and participate in our community. However, to actually use the -testbed you will first need to **join or create a project**. Let's learn how! +Once you log in, you can :ref:`edit your Chameleon profile ` and +participate in our community. However, to actually use the testbed you'll first +need to **join or create a project**. Let's learn how! .. _getting-started-project: Pre-Req: Create or Join a Project -================================ - -**Projects** are user-created workspaces on Chameleon that allow you to manage -testbed resources and project members, create hardware and network -reservations, and share project outcomes (like publications). All Chameleon -projects have an assigned **project ID** (``CHI-XXXXXX``), a project leader -(what we call a **Principal Investigator (PI)** 🕵️ on Chameleon), and an -**allocation** of compute resources. First-time projects are automatically -granted six months of compute (20,000 `service hours`_). - -.. note:: - Projects can request renewals after the first allocation to receive more - compute. Read more about renewals :ref:`here `. - - A PI can have multiple projects on Chameleon to isolate their research projects - and compute. We ask that users create a new project when starting a new - research endeavor, rather than reusing a previous one. This user behavior also - helps us report the magnificent impact and value that Chameleon provides and - helps keep the lights on so you can keep doing your research. - -There are two ways to join a project: - -1. Create a new project (requires PI status) -2. Join an existing project (requires project invitation from current project member) - -Creating a New Project ----------------------- +================================= -To create a new project on Chameleon, you will need to apply for and receive PI -status on Chameleon. To determine if you can obtain PI status, please see a -:ref:`list of PI eligibility criteria `. If you do not meet -these criteria (**graduate students often do not**), you will need to ask your -advisor or other scientist supervising your research to create the project for -you. +To use Chameleon resources, you need to be a member of an active **project**. Projects are +workspaces that provide compute allocations and manage team access to testbed resources. -You can request PI status by checking a box in `your Chameleon profile -`_. Once on your profile page, -click the "Edit Profile" action. You can then click on the checkbox "Request PI -Eligibility" and save your profile. Chameleon PI status requests are typically -reviewed within one business day. +There are two ways to get project access: -.. image:: ../_static/imgs/getting_started/new-project-form.png - :width: 80 % +**Option 1: Create a New Project** (if you're eligible to be a PI) + - Requires PI eligibility status on Chameleon + - See our :doc:`PI eligibility guide <../user/pi_eligibility>` for requirements + - Graduate students typically need their advisor to create the project -Once you have PI status, you may apply for a new project with an initial -allocation. Create a new project by going to the `Projects Dashboard`_ and -click the "Create a Project" in the right corner of the window. Complete the -form and click "Create Project." Once your project has been approved, you will -be able to utilize the testbed sites. +**Option 2: Join an Existing Project** (most common for new users) + - Ask a current project member to add you + - Provide your Chameleon username (found in your `profile `_) -Read more about :ref:`creating projects ` on Chameleon. - -Joining an Existing Project ---------------------------- - -.. image:: ../_static/imgs/getting_started/project-members-section.png - -If you want to join an existing Chameleon project, you will need to join an existing project. There are three ways to add a user to a project. - -#. The project PI/manager adds your username or email directly -#. The project PI/manager sends you an invitation (automatic if the email from above doesn't exist in our system yet) -#. the project PI/manager shares an invite link with you, which sends a join request for a project when you click it - -To find your username, go to `your Chameleon profile page -`_ - it is also displayed in the -top-right corner when you are logged in. Once you join a project, you will then -be able to use the project's compute allocation to make resource reservations. - -Read more about :ref:`user management ` on Chameleon. +.. note:: + **New to projects?** Read our comprehensive :doc:`project management guide <../user/project>` + for details on project concepts, user roles, allocations, and management. .. _start-using-chameleon: @@ -540,7 +463,7 @@ access your instance, you need to first assign a floating IP address - an IP address that is accessible over the public Internet. Step 1: Associate an IP -~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~ To associate an IP address with your instance, follow these steps. Note, it is best to wait until your instance is running before doing this step to ensure no @@ -581,7 +504,7 @@ issues. step 2. Step 2: Accessing Your Instance -~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Once your instance has launched with an associated floating IP address, it can be accessed via SSH using the private key that you added when creating an @@ -776,8 +699,6 @@ Click on the "**Launch on Chameleon**" button to start Jupyter. This loading pag should look familiar to the loading page when we launched the Jupyter Interface above. -.. image:: - Once Jupyter has loaded, we will have the artifact directory available in our workspace. Your directory should include the following files: diff --git a/source/index.rst b/source/index.rst index 93dea00..7ab694c 100644 --- a/source/index.rst +++ b/source/index.rst @@ -5,28 +5,104 @@ Welcome to Chameleon What is Chameleon? ================== -Chameleon is an NSF-funded testbed system for Computer Science experimentation. -It is designed to be deeply reconfigurable, with a wide variety of capabilities -for researching systems, networking, distributed and cluster computing and -security. Chameleon's features include: +Chameleon is an NSF-funded testbed system for Computer Science experimentation. +It provides researchers with deeply reconfigurable cloud infrastructure for systems, +networking, distributed computing, and security research. Unlike traditional cloud +services, Chameleon offers both bare metal access to physical hardware and traditional +virtual machines, giving you full control over the software stack and enabling +reproducible experimental research. -* Bare metal access to hardware, via the cloud -* A wide variety of hardware types, including: +Key Features +============ - * Infiniband - * NVMe - * GPUs/FPGAs - * Low-power Xeon and ARM processors +**Hardware Access** + * **Bare metal instances**: Full control over physical servers without virtualization overhead + * **Virtual machines**: Traditional OpenStack KVM instances for development and testing + * **Diverse hardware**: Intel/AMD CPUs (with ROCm support), ARM ThunderX2, GPUs, FPGAs, Atom processors, high-memory nodes + * **Storage options**: NVMe SSDs, traditional spinning disks, shared file systems + * **High-performance networking**: InfiniBand, 25/100 Gigabit Ethernet -* A powerful set of networking capabilities, including: +**Experimental Capabilities** + * **Resource isolation**: Dedicated hardware reservations for reproducible experiments + * **Custom images**: Create and share disk images with your experimental software + * **Power monitoring**: Measure energy consumption at the node and application level + * **Performance metrics**: Built-in monitoring and data collection tools - * Isolated layer-2 networks - * SDN/OpenFlow - * Dedicated WAN layer-2 links +**Advanced Networking** + * **Isolated networks**: Create private Layer-2 VLANs for multi-node experiments + * **Multi-site Layer-3**: Direct routing between Chameleon sites via FABRIC (FabnetV4) + * **WAN connectivity**: Connect to external networks and other testbeds via FABRIC + * **Flexible topologies**: Advanced routing and network configuration -Using Chameleon +**Collaboration & Reproducibility** + * **Trovi sharing portal**: Package and share complete experimental environments + * **Jupyter integration**: Interactive development and data analysis environment + * **Multi-site deployment**: Experiments across geographically distributed sites + +Getting Started =============== -Chameleon is available to Computer Science researchers and students. To access -the system, follow the instructions in :ref:`getting-started`. Find out more at -https://www.chameleoncloud.org. +**New to Chameleon?** + Start with our :doc:`getting-started guide ` to create an account, + join a project, and launch your first instance. + +**Need access to the testbed?** + Learn about :doc:`PI eligibility ` and :doc:`project management `. + +**Ready to use the testbed?** + Choose your interface: + + * :doc:`Web Interface ` - Point-and-click access to all features + * :doc:`Command Line ` - Programmatic access and automation + * :doc:`Jupyter Environment ` - Interactive notebooks and data analysis + +Quick Navigation +================ + +**Core Workflow** + 1. :doc:`Discover resources ` - Find the right hardware for your experiment + 2. :doc:`Make reservations ` - Reserve nodes and networks + 3. :doc:`Launch instances ` - Deploy your experimental environment + 4. :doc:`Monitor and collect data ` - Measure performance and energy usage + +**Advanced Features** + * :doc:`Custom images ` - Create reproducible software environments + * :doc:`Complex deployments ` - Multi-node orchestration with Heat + * :doc:`Networking ` - Advanced network topologies and isolation + * :doc:`FPGA programming ` - Hardware acceleration experiments + * :doc:`Share your work ` - Publish experiments via Trovi + +**Data & Storage** + * :doc:`Object storage ` - Scalable data storage and sharing + * :doc:`Shared file systems ` - NFS-mounted storage for instances + * :doc:`KVM instances ` - Traditional virtual machines when needed + +**Getting Help** + * :doc:`Help desk ` - Submit tickets and view system status + * :doc:`User profile ` - Manage your account settings + * :doc:`Daypass access ` - Temporary access for artifact reproduction + +About the Testbed +================= + +Chameleon operates multiple sites providing different capabilities: + +**Core Sites:** + +* **CHI@TACC** (Texas): Large-scale bare metal cloud with diverse Intel/AMD hardware including GigaIO nodes +* **CHI@UC** (Chicago): Networking-focused site with specialized hardware and GPU/FPGA resources +* **CHI@NCAR** (Colorado): ARM ThunderX2 nodes for edge computing and atmospheric science research +* **CHI@Edge**: Distributed edge computing with Raspberry Pi devices (including Raspberry Pi 5) +* **KVM@TACC** (Texas): Traditional OpenStack cloud + +**Associate Sites:** + +* **CHI@NRP**: National Research Platform integration +* **CHI@NU**: Northwestern University integration +* **CHI@EVL**: Electronic Visualization Laboratory (UIC) integration + +The testbed serves hundreds of research projects annually, supporting publications +in systems, networking, distributed computing, cybersecurity, edge computing, and +atmospheric sciences. + +Learn more about Chameleon and join the community at https://www.chameleoncloud.org. \ No newline at end of file diff --git a/source/technical/baremetal.rst b/source/technical/baremetal.rst index 0658966..37e5315 100644 --- a/source/technical/baremetal.rst +++ b/source/technical/baremetal.rst @@ -1,433 +1,11 @@ -===================== -Bare metal instances -===================== +:orphan: -Before launching an instance, make sure you own a lease. About how to create a -lease, please see :ref:`reservations`. Once your lease is started, you are -almost ready to start an instance. But first, you need to make sure that you -will be able to connect to it by setting up :ref:`gui-key-pairs`. +.. meta:: + :http-equiv=refresh: 0; url=./baremetal/index.html -Launching instances with the GUI -================================ +This page has moved +=================== -.. _baremetal-gui-launch: +This page has been reorganized. You will be redirected to the new location. -Launch an instance ------------------- - -To launch an instance with the GUI, follow the steps: - -#. In the navigation side bar, click *Project* > *Compute* > *Instances* to get - to the *Instances* page. - - .. figure:: baremetal/instancespage.png - :alt: The Instances page - - The Instances page - -#. Click the *Launch Instance* button in the upper right corner. This will open - the *Launch Instance* wizard with several configuration steps. Steps with - ``*`` are required. - - .. figure:: baremetal/launchinstance.png - :alt: The Launch Instance wizard. - - The Launch Instance wizard. - -#. In the *Details* step, enter a name for your instance that is unique within - your project and select a currently active reservation for the instance. - -#. In the *Source* step, select an image for your instance and click the "up" - arrow. The image should move to the *Allocated* list, and can be removed by - clicking the "Down" arrow if you wish to select a different image. - - .. figure:: baremetal/launchsource.png - :alt: The Source configuration step - - The Source configuration step - -#. In the *Flavor* step, select the *baremetal* flavor by clicking the "up" - arrow next to it. This is the only flavor available. - - .. figure:: baremetal/launchflavor.png - :alt: The Flavor configuration step - - The Flavor configuration step - - .. hint:: - - If you are familiar with Openstack, other implementations allow for the - selection of flavors based on machine disk size and RAM. On Chameleon, the - only flavor available is "baremetal" because hardware selection is - performed in reservations. - -#. In the *Networks* step, select a network by clicking the "up" arrow next to - it. To learn about the Chameleon default network and how to create your own - network, please see :ref:`networking`. - -#. In the *Key Pair* step, select one of your SSH key pairs. If you only have - one key pair associated with your account, then it is selected by default. - - .. figure:: baremetal/launchkeypair.png - :alt: The Key Pair configuration step - - The Key Pair configuration step - - .. important:: - - It is a good practice to make sure that the instance is launching with the - key pair of your choice, or you will not be able to access your instance. - - .. tip:: - You may import or create key pairs directly through this step. - -#. If you want to customize your instance after it has launched, you can add a - customization script in the *Configuration* step. - - - You can type in the script in *Customization Script*. - - Or you can upload your script via *Load script from a file*. - - .. figure:: baremetal/customizationscript.png - :alt: Adding a Customization Script - - Adding a Customization Script - - .. tip:: - You can :ref:`disable and turn off appliance agents - ` using a customization script. - -#. Finish configuring and start launching the instance by clicking on the - *Launch Instance* button. The instance will show up in the instance list, at - first in *Build* status. It takes a few minutes to deploy the instance on - bare metal hardware and reboot the machine. - - .. figure:: baremetal/instancesbuild.png - :alt: An Instance with the Build status - - An Instance with the Build status - -#. After a few minutes, the instance should become *Active*. The power state - will show as *Running*. You can now :ref:`baremetal-gui-associate-ip`. - - .. figure:: baremetal/instancesactive.png - :alt: An Instance with the Active status - - An Instance with the Active status - -#. To view instance details, click on the instance. - - .. figure:: baremetal/instancedetails.png - :alt: Instance details - - Instance details - -.. _baremetal-gui-associate-ip: - -Associate a Floating IP ------------------------ - -To make your instance publicly accessible over the Internet, you must associate -a *Floating IP Address* to it. - -#. On the *Floating IPs* page (under the *Network* section in the left-hand - sidebar), ensure that there is a free Floating IP available in your project. - If there is not, click the *Allocate IP to Project* button to bring up the - *Allocate Floating IP* dialog. In this dialog, you may simply click *Allocate - IP*. You can optionally specify a description for the IP for your - convenience. - - .. figure:: baremetal/associate_pool.png - :alt: the Allocate Floating IP dialog - - The Allocate Floating IP dialog - -#. Once a Floating IP is allocated to your project, it will display in the list - view, and you can click the *Associate* button for the Floating IP to assign - it to a running or spawning instance. This button will bring up the *Manage - Floating IP Associations* dialog. - - .. figure:: baremetal/floating_ip_overview.png - :alt: The Floating IP list view with a Floating IP available - - The Floating IP list view with a Floating IP available - -#. In the dialog, select an instance from the "Port to be associated" dropdown. - Your instance's display name will be displayed here. Click *Associate* to - complete the process of assigning the IP to your instance. - - .. figure:: baremetal/associate_ip.png - :alt: The Manage Floating IP Associations dialog with an IP selected - - The Manage Floating IP Associations dialog with an IP selected - -#. If you go back to the *Instances* page, you should now see the *floating - IP* attached to the instance. - - .. figure:: baremetal/instanceswithip.png - :alt: An instance with an allocated Floating IP - - An instance with an allocated Floating IP - -Launching Instances with the CLI -================================ - -.. tip:: - - Reading :ref:`cli` is highly recommanded before continuing on the following - sections. - -Creating an instance with the CLI ---------------------------------- - -To launch an instance inside a reservation, run: - -.. code-block:: bash - - openstack server create \ - --image CC-CentOS8 \ - --flavor baremetal \ - --key-name \ - --nic net-id= \ - --hint reservation= \ - my-instance - -The ID of the ``sharednet1`` network can be obtained using the command: - -.. code-block:: bash - - openstack network list - -Alternatively, you may look it up in the GUI in the *Network* > *Networks* page. -You can obtain your *reservation ID* via the web interface or by running: - -.. code-block:: bash - - openstack reservation lease show - -.. attention:: The **reservation ID** and the **lease ID** are different - -Running a script on boot -^^^^^^^^^^^^^^^^^^^^^^^^ - -You might want to automatically execute some code after launching an instance, -whether it is installing packages, changing configuration files, or running an -application. OpenStack provides a mechanism called `User Data -`_ to pass information -to instances. This information can be any data in any format, but if it is a -shell script it will be automatically executed after boot by `cloudinit -`_. You can provide this shell -script either via the web interface in the *Configuration* tab when launching an -instance, or by providing a file to the nova command line using the -``--user-data`` option. - -.. _turn-off-appliance-agents: -.. tip:: - - Chameleon supported images are configured with appliance agents, including - :ref:`collectd ` and :ref:`Heat agents `. - To turn off appliance agents on boot, in order to remove the potential impact - on experimental measurements, pass the following script as ``user-data``. - - .. code-block:: bash - - #!/bin/bash - systemctl stop collectd.service - systemctl disable collectd.service - systemctl stop os-collect-config.service - systemctl disable os-collect-config.service - - Turning off ``collectd`` will **stop** collecting :ref:`Gnocchi metrics - `, but you can :ref:`turn on and configure the daemon - ` anytime for monitoring your experiment. - -Customizing the Kernel ----------------------- - -It is easy to customize the operating system kernel or modify the kernel command -line. You now have the option of modifying the boot loader configuration (e.g., -``/boot/grub2/grub.cfg`` on CentOS 7 images) to point it to a new kernel on the -local disk, or specifying kernel parameters and then rebooting using this -modified configuration. - -To do this, you must be using a whole disk image rather than a partition image. -Whole disk images contain their own kernel and ramdisk files and do not have -``kernel_id`` and ``ramdisk_id`` properties in the image repository, unlike -partition images. Most Chameleon base images are whole disk images, giving you -a good place to start if you're interested in custom kernels. - -Running virtual machines on bare metal --------------------------------------- - -For cloud computing and virtualization experiments, you might want to run -virtual machines on bare hardware that you fully control rather than use the -shared OpenStack KVM cloud. There are many different ways to configure -networking for virtual machines. The configuration described below will enable -you to connect your virtual machines to the Internet using a `KVM Public Bridge -`_ which you must first -configure manually on your host on the default network interface. - -First, set up your environment for the OpenStack command line tools by following -:ref:`the instructions `. Install the `Neutron -`_ client in a virtualenv with: - -.. code-block:: bash - - pip install python-neutronclient - -Then, for each virtual machine you want to run, request a `Neutron -`_ port with: - -.. code-block:: bash - - openstack port-create sharednet1 - -This should display, among other information: - -- A fixed IP in the same private network as the physical nodes -- A MAC address - -Finally, start your virtual machine while assigning it the *MAC address* -provided by OpenStack. If your image is configured to use *DHCP*, the virtual -machine should receive the allocated IP. - -Neutron ports allocated this way are not automatically deleted, so please delete -them after your experiment is over using: - -.. code-block:: bash - - openstack port delete - -You may find the ID of your ports using: - -.. code-block:: bash - - openstack port list - - -Inspecting your node --------------------- - -From within an instance you have already launched, you can discover which node -it is running on by executing - -.. code-block:: bash - - curl http://169.254.169.254/openstack/latest/vendor_data.json - -This will return a JSON dictionary describing site, cluster, and node. - -Customizing networking ----------------------- - -In its default configuration, the bare metal deployment system used by Chameleon -(`OpenStack Ironic `_) is restricted -to using a single shared network per site. The network configuration features -available in the dashboard are not supported (Networks and Routers). On -|CHI@UC|, network layer 2 isolation is optionally available for compute nodes. -You may find more details on the documentation for :ref:`networking`. - -Interacting with instances -========================== - -Once your bare metal instance has launched, you may interact with it by using -SSH if you have associated a *Floating IP* to it or by using the *Serial -Console* from the GUI. - -.. _connecting-via-ssh: - -Connecting via SSH ------------------- - -If you have associated a *Floating IP* with the instance and you have the -private key in place, you should be able to connect to the instance via SSH -using the ``cc`` account. - -To access the instance using SSH, type the command in your terminal: - - .. code-block:: bash - - ssh cc@ - -.. error:: - If you get errors: - - .. code-block:: shell - - @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ - @ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ - @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ - IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! - ... - - It is likely that you have saved a previous entry for the instance's - *Floating IP* in your ``~/.ssh/known_hosts`` file on your computer. Simply - removing the entry from the file should solve the issue. - - You can remove the entry from the ``~/.ssh/known_hosts`` file by using the - command: - - .. code-block:: shell - - ssh-keygen -R - -You may receive the response below. Type ``yes`` and hit enter: - - .. code:: - - The authenticity of host '130.202.88.241 (130.202.88.241)' can't be established. - RSA key fingerprint is 5b:ca:f0:63:6f:22:c6:96:9f:c0:4a:d8:5e:dd:fd:eb. - Are you sure you want to continue connecting (yes/no)? - -When logged in, your prompt may appear like this: - - .. code:: - - [cc@my-first-instance ~]$ - -.. note:: - - If you notice SSH errors such as connection refused, password requests, or - failures to accept your key, it is likely that the physical node is still - going through the boot process. In that case, please wait before retrying. - Also make sure that you use the ``cc`` account. If after 10 minutes you still - cannot connect to the machine, please open a ticket with our |Help Desk|. - -You can now check whether the resource matches its known description in the -resource registry. For this, simply run: - - .. code-block:: bash - - sudo cc-checks -v - -The ``cc-checks`` program prints the result of each check in green if it is -successful and red if it failed. You can now run your experiment directly on the -machine via SSH. You can run commands with root privileges by prefixing them -with ``sudo``. To completely switch user and become root, use the ``sudo su - -root`` command. - -.. attention:: ``cc-checks`` is only available on legacy CentOS7 images! - -Connecting via serial console ------------------------------ - -Chameleon now allows you to connect to the serial console of your bare metal -nodes via the GUI. Once your instance is deployed, click on the *Console* button -in the instance contextual menu. - -.. figure:: baremetal/serialconsole.png - :alt: The Serial Console button - - The serial console button - -This should open a screen showing an interactive serial console (it could take -some time to show up, give it 30 seconds or so). - -.. figure:: baremetal/instanceconsole.png - :alt: An open Console - - An open console - -Our latest images are configured to auto-login into the ``cc`` account. Other -images may show you a login prompt. You can set a password on the ``cc`` account -by accessing it via SSH, using the command ``sudo passwd cc``, and then using -this password to connect to the console. +If you are not redirected automatically, please visit :doc:`baremetal/index`. \ No newline at end of file diff --git a/source/technical/baremetal/associate_ip.png b/source/technical/baremetal/baremetal/associate_ip.png similarity index 100% rename from source/technical/baremetal/associate_ip.png rename to source/technical/baremetal/baremetal/associate_ip.png diff --git a/source/technical/baremetal/associate_manage.png b/source/technical/baremetal/baremetal/associate_manage.png similarity index 100% rename from source/technical/baremetal/associate_manage.png rename to source/technical/baremetal/baremetal/associate_manage.png diff --git a/source/technical/baremetal/associate_pool.png b/source/technical/baremetal/baremetal/associate_pool.png similarity index 100% rename from source/technical/baremetal/associate_pool.png rename to source/technical/baremetal/baremetal/associate_pool.png diff --git a/source/technical/baremetal/customizationscript.png b/source/technical/baremetal/baremetal/customizationscript.png similarity index 100% rename from source/technical/baremetal/customizationscript.png rename to source/technical/baremetal/baremetal/customizationscript.png diff --git a/source/technical/baremetal/floating_ip_overview.png b/source/technical/baremetal/baremetal/floating_ip_overview.png similarity index 100% rename from source/technical/baremetal/floating_ip_overview.png rename to source/technical/baremetal/baremetal/floating_ip_overview.png diff --git a/source/technical/baremetal/instanceconsole.png b/source/technical/baremetal/baremetal/instanceconsole.png similarity index 100% rename from source/technical/baremetal/instanceconsole.png rename to source/technical/baremetal/baremetal/instanceconsole.png diff --git a/source/technical/baremetal/instancedetails.png b/source/technical/baremetal/baremetal/instancedetails.png similarity index 100% rename from source/technical/baremetal/instancedetails.png rename to source/technical/baremetal/baremetal/instancedetails.png diff --git a/source/technical/baremetal/instancesactive.png b/source/technical/baremetal/baremetal/instancesactive.png similarity index 100% rename from source/technical/baremetal/instancesactive.png rename to source/technical/baremetal/baremetal/instancesactive.png diff --git a/source/technical/baremetal/instancesbuild.png b/source/technical/baremetal/baremetal/instancesbuild.png similarity index 100% rename from source/technical/baremetal/instancesbuild.png rename to source/technical/baremetal/baremetal/instancesbuild.png diff --git a/source/technical/baremetal/instancespage.png b/source/technical/baremetal/baremetal/instancespage.png similarity index 100% rename from source/technical/baremetal/instancespage.png rename to source/technical/baremetal/baremetal/instancespage.png diff --git a/source/technical/baremetal/instanceswithip.png b/source/technical/baremetal/baremetal/instanceswithip.png similarity index 100% rename from source/technical/baremetal/instanceswithip.png rename to source/technical/baremetal/baremetal/instanceswithip.png diff --git a/source/technical/baremetal/launchflavor.png b/source/technical/baremetal/baremetal/launchflavor.png similarity index 100% rename from source/technical/baremetal/launchflavor.png rename to source/technical/baremetal/baremetal/launchflavor.png diff --git a/source/technical/baremetal/launchinstance.png b/source/technical/baremetal/baremetal/launchinstance.png similarity index 100% rename from source/technical/baremetal/launchinstance.png rename to source/technical/baremetal/baremetal/launchinstance.png diff --git a/source/technical/baremetal/launchkeypair.png b/source/technical/baremetal/baremetal/launchkeypair.png similarity index 100% rename from source/technical/baremetal/launchkeypair.png rename to source/technical/baremetal/baremetal/launchkeypair.png diff --git a/source/technical/baremetal/launchscheduler.png b/source/technical/baremetal/baremetal/launchscheduler.png similarity index 100% rename from source/technical/baremetal/launchscheduler.png rename to source/technical/baremetal/baremetal/launchscheduler.png diff --git a/source/technical/baremetal/launchsource.png b/source/technical/baremetal/baremetal/launchsource.png similarity index 100% rename from source/technical/baremetal/launchsource.png rename to source/technical/baremetal/baremetal/launchsource.png diff --git a/source/technical/baremetal/managefloatingip.png b/source/technical/baremetal/baremetal/managefloatingip.png similarity index 100% rename from source/technical/baremetal/managefloatingip.png rename to source/technical/baremetal/baremetal/managefloatingip.png diff --git a/source/technical/baremetal/serialconsole.png b/source/technical/baremetal/baremetal/serialconsole.png similarity index 100% rename from source/technical/baremetal/serialconsole.png rename to source/technical/baremetal/baremetal/serialconsole.png diff --git a/source/technical/baremetal/index.rst b/source/technical/baremetal/index.rst new file mode 100644 index 0000000..90b7b65 --- /dev/null +++ b/source/technical/baremetal/index.rst @@ -0,0 +1,20 @@ +.. _baremetal: + +==================== +Bare Metal Instances +==================== + +Bare metal instances on Chameleon provide you with exclusive access to physical hardware resources. This allows you to run experiments with full control over the compute environment, from the hypervisor level down to the bare metal. Bare metal instances are ideal for systems research, performance evaluation, and experiments that require specific hardware features or configurations. + +Before launching an instance, make sure you own a lease. About how to create a +lease, see :ref:`reservations`. Once your lease is started, you are +almost ready to start an instance. But first, you need to make sure that you +will be able to connect to it by setting up :ref:`gui-key-pairs`. + +.. toctree:: + :maxdepth: 1 + :caption: Bare Metal Topics + + launching_gui + launching_cli + interacting \ No newline at end of file diff --git a/source/technical/baremetal/interacting.rst b/source/technical/baremetal/interacting.rst new file mode 100644 index 0000000..7ccc057 --- /dev/null +++ b/source/technical/baremetal/interacting.rst @@ -0,0 +1,105 @@ +Interacting with Instances +========================== + +Once your bare metal instance has launched, you may interact with it by using +SSH if you have associated a *Floating IP* to it or by using the *Serial +Console* from the GUI. + +.. _connecting-via-ssh: + +Connecting via SSH +------------------ + +If you have associated a *Floating IP* with the instance and you have the +private key in place, you should be able to connect to the instance via SSH +using the ``cc`` account. + +To access the instance using SSH, type the command in your terminal: + + .. code-block:: bash + + ssh cc@ + +.. error:: + If you get errors: + + .. code-block:: shell + + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + @ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! + ... + + It is likely that you have saved a previous entry for the instance's + *Floating IP* in your ``~/.ssh/known_hosts`` file on your computer. Simply + removing the entry from the file should solve the issue. + + You can remove the entry from the ``~/.ssh/known_hosts`` file by using the + command: + + .. code-block:: shell + + ssh-keygen -R + +You may receive the response below. Type ``yes`` and hit enter: + + .. code:: + + The authenticity of host '130.202.88.241 (130.202.88.241)' can't be established. + RSA key fingerprint is 5b:ca:f0:63:6f:22:c6:96:9f:c0:4a:d8:5e:dd:fd:eb. + Are you sure you want to continue connecting (yes/no)? + +When logged in, your prompt may appear like this: + + .. code:: + + [cc@my-first-instance ~]$ + +.. note:: + + If you notice SSH errors such as connection refused, password requests, or + failures to accept your key, it is likely that the physical node is still + going through the boot process. In that case, wait before retrying. + Also make sure that you use the ``cc`` account. If after 10 minutes you still + cannot connect to the machine, open a ticket with our |Help Desk|. + +You can now check whether the resource matches its known description in the +resource registry. For this, simply run: + + .. code-block:: bash + + sudo cc-checks -v + +The ``cc-checks`` program prints the result of each check in green if it is +successful and red if it failed. You can now run your experiment directly on the +machine via SSH. You can run commands with root privileges by prefixing them +with ``sudo``. To completely switch user and become root, use the ``sudo su - +root`` command. + +.. attention:: ``cc-checks`` is only available on legacy CentOS7 images! + +Connecting via serial console +----------------------------- + +Chameleon now allows you to connect to the serial console of your bare metal +nodes via the GUI. Once your instance is deployed, click on the *Console* button +in the instance contextual menu. + +.. figure:: baremetal/serialconsole.png + :alt: The Serial Console button + + The serial console button + +This should open a screen showing an interactive serial console (it could take +some time to show up, give it 30 seconds or so). + +.. figure:: baremetal/instanceconsole.png + :alt: An open Console + + An open console + +Our latest images are configured to auto-login into the ``cc`` account. Other +images may show you a login prompt. You can set a password on the ``cc`` account +by accessing it via SSH, using the command ``sudo passwd cc``, and then using +this password to connect to the console. \ No newline at end of file diff --git a/source/technical/baremetal/launching_cli.rst b/source/technical/baremetal/launching_cli.rst new file mode 100644 index 0000000..9b38e1d --- /dev/null +++ b/source/technical/baremetal/launching_cli.rst @@ -0,0 +1,157 @@ +Launching Instances with the CLI +================================ + +.. tip:: + + Reading :ref:`cli` is highly recommended before continuing on the following + sections. + +Creating an instance with the CLI +--------------------------------- + +To launch an instance inside a reservation, run: + +.. code-block:: bash + + openstack server create \ + --image CC-CentOS8 \ + --flavor baremetal \ + --key-name \ + --nic net-id= \ + --hint reservation= \ + my-instance + +The ID of the ``sharednet1`` network can be obtained using the command: + +.. code-block:: bash + + openstack network list + +Alternatively, you may look it up in the GUI in the *Network* > *Networks* page. +You can obtain your *reservation ID* via the GUI or by running: + +.. code-block:: bash + + openstack reservation lease show + +.. attention:: The **reservation ID** and the **lease ID** are different + +Running a script on boot +^^^^^^^^^^^^^^^^^^^^^^^^ + +You might want to automatically execute some code after launching an instance, +whether it is installing packages, changing configuration files, or running an +application. OpenStack provides a mechanism called `User Data +`_ to pass information +to instances. This information can be any data in any format, but if it is a +shell script it will be automatically executed after boot by `cloudinit +`_. You can provide this shell +script either via the GUI in the *Configuration* tab when launching an +instance, or by providing a file to the nova command line using the +``--user-data`` option. + +.. _turn-off-appliance-agents: +.. tip:: + + Chameleon supported images are configured with appliance agents, including + ``collectd`` (for system monitoring) and :ref:`Heat agents `. + To turn off appliance agents on boot, in order to remove the potential impact + on experimental measurements, pass the following script as ``user-data``. + + .. code-block:: bash + + #!/bin/bash + systemctl stop collectd.service + systemctl disable collectd.service + systemctl stop os-collect-config.service + systemctl disable os-collect-config.service + + Turning off ``collectd`` will **stop** collecting system metrics, but you can + restart and configure the daemon anytime for monitoring your experiment. For power + monitoring capabilities, see :ref:`power-monitoring`. + +Customizing the Kernel +---------------------- + +It is easy to customize the operating system kernel or modify the kernel command +line. You now have the option of modifying the boot loader configuration (e.g., +``/boot/grub2/grub.cfg`` on CentOS 7 images) to point it to a new kernel on the +local disk, or specifying kernel parameters and then rebooting using this +modified configuration. + +To do this, you must be using a whole disk image rather than a partition image. +Whole disk images contain their own kernel and ramdisk files and do not have +``kernel_id`` and ``ramdisk_id`` properties in the image repository, unlike +partition images. Most Chameleon base images are whole disk images, giving you +a good place to start if you're interested in custom kernels. + +Running virtual machines on bare metal +-------------------------------------- + +For cloud computing and virtualization experiments, you might want to run +virtual machines on bare hardware that you fully control rather than use the +shared OpenStack KVM cloud. There are many different ways to configure +networking for virtual machines. The configuration described below will enable +you to connect your virtual machines to the Internet using a `KVM Public Bridge +`_ which you must first +configure manually on your host on the default network interface. + +First, set up your environment for the OpenStack command line tools by following +:ref:`the instructions `. Install the `Neutron +`_ client in a virtualenv with: + +.. code-block:: bash + + pip install python-neutronclient + +Then, for each virtual machine you want to run, request a `Neutron +`_ port with: + +.. code-block:: bash + + openstack port-create sharednet1 + +This should display, among other information: + +- A fixed IP in the same private network as the physical nodes +- A MAC address + +Finally, start your virtual machine while assigning it the *MAC address* +provided by OpenStack. If your image is configured to use *DHCP*, the virtual +machine should receive the allocated IP. + +Neutron ports allocated this way are not automatically deleted, so delete +them after your experiment is over using: + +.. code-block:: bash + + openstack port delete + +You may find the ID of your ports using: + +.. code-block:: bash + + openstack port list + + +Inspecting your node +-------------------- + +From within an instance you have already launched, you can discover which node +it is running on by executing + +.. code-block:: bash + + curl http://169.254.169.254/openstack/latest/vendor_data.json + +This will return a JSON dictionary describing site, cluster, and node. + +Customizing networking +---------------------- + +In its default configuration, the bare metal deployment system used by Chameleon +(OpenStack Ironic `_) is restricted +to using a single shared network per site. The network configuration features +available in the dashboard are not supported (Networks and Routers). On +|CHI@UC|, network layer 2 isolation is optionally available for compute nodes. +You may find more details on the documentation for :ref:`networking`. \ No newline at end of file diff --git a/source/technical/baremetal/launching_gui.rst b/source/technical/baremetal/launching_gui.rst new file mode 100644 index 0000000..74b980f --- /dev/null +++ b/source/technical/baremetal/launching_gui.rst @@ -0,0 +1,185 @@ +Launching Instances with the GUI +================================ + +.. _baremetal-gui-launch: + +Launch an instance +------------------ + +To launch an instance with the GUI, follow the steps: + +#. In the navigation side bar, click *Project* > *Compute* > *Instances* to get + to the *Instances* page. + + .. figure:: baremetal/instancespage.png + :alt: The Instances page + + The Instances page + +#. Click the *Launch Instance* button in the upper right corner. This will open + the *Launch Instance* wizard with several configuration steps. Steps with + ``*`` are required. + + .. figure:: baremetal/launchinstance.png + :alt: The Launch Instance wizard. + + The Launch Instance wizard. + +#. In the *Details* step, enter a name for your instance that is unique within + your project and select a currently active reservation for the instance. + +#. In the *Source* step, select an image for your instance and click the "up" + arrow. The image should move to the *Allocated* list, and can be removed by + clicking the "Down" arrow if you wish to select a different image. + + .. figure:: baremetal/launchsource.png + :alt: The Source configuration step + + The Source configuration step + +#. In the *Flavor* step, select the *baremetal* flavor by clicking the "up" + arrow next to it. This is the only flavor available. + + .. figure:: baremetal/launchflavor.png + :alt: The Flavor configuration step + + The Flavor configuration step + + .. hint:: + + If you are familiar with Openstack, other implementations allow for the + selection of flavors based on machine disk size and RAM. On Chameleon, the + only flavor available is "baremetal" because hardware selection is + performed in reservations. + +#. In the *Networks* step, select a network by clicking the "up" arrow next to + it. To learn about the Chameleon default network and how to create your own + network, see :ref:`networking`. + +#. In the *Key Pair* step, select one of your SSH key pairs. If you only have + one key pair associated with your account, then it is selected by default. + + .. figure:: baremetal/launchkeypair.png + :alt: The Key Pair configuration step + + The Key Pair configuration step + + .. important:: + + It is a good practice to make sure that the instance is launching with the + key pair of your choice, or you will not be able to access your instance. + + .. tip:: + You may import or create key pairs directly through this step. + + **Creating a New Key Pair** + + To create a key pair through the interface: + + 1. Click *+ Create Key Pair* button + 2. Provide a name for your new key pair and click *Create Key Pair* + 3. A ``.pem`` file containing the private key will be automatically downloaded + 4. The public key is saved automatically to Chameleon + 5. Save the private key to a secure location (your home directory is recommended for macOS/Linux) + + **Importing an Existing Key Pair** + + To import a key pair you've generated on your computer: + + 1. Click *Import Key Pair* button + 2. Provide a name for your imported key pair + 3. Paste your public key (typically found at ``~/.ssh/id_rsa.pub``) + + .. note:: + Chameleon **only** stores the public key for each SSH key pair. **Never** upload + your private key! Private keys begin with ``-----BEGIN RSA PRIVATE KEY-----`` + + .. tip:: + On macOS, you can copy your public key with: ``cat ~/.ssh/id_rsa.pub | pbcopy`` + +#. If you want to customize your instance after it has launched, you can add a + customization script in the *Configuration* step. + + - You can type in the script in *Customization Script*. + - Or you can upload your script via *Load script from a file*. + + .. figure:: baremetal/customizationscript.png + :alt: Adding a Customization Script + + Adding a Customization Script + + .. tip:: + You can :ref:`disable and turn off appliance agents + ` using a customization script. + +#. Finish configuring and start launching the instance by clicking on the + *Launch Instance* button. The instance will show up in the instance list, at + first in *Build* status. It takes a few minutes to deploy the instance on + bare metal hardware and reboot the machine. + + .. figure:: baremetal/instancesbuild.png + :alt: An Instance with the Build status + + An Instance with the Build status + +#. After a few minutes, the instance should become *Active*. The power state + will show as *Running*. You can now :ref:`baremetal-gui-associate-ip`. + + .. figure:: baremetal/instancesactive.png + :alt: An Instance with the Active status + + An Instance with the Active status + +#. To view instance details, click on the instance. + + .. figure:: baremetal/instancedetails.png + :alt: Instance details + + Instance details + +.. _baremetal-gui-associate-ip: + +Associate a Floating IP +----------------------- + +To make your instance publicly accessible over the Internet, you must associate +a *Floating IP Address* to it. + +#. On the *Floating IPs* page (under the *Network* section in the left-hand + sidebar), ensure that there is a free Floating IP available in your project. + If there is not, click the *Allocate IP to Project* button to bring up the + *Allocate Floating IP* dialog. In this dialog, you may simply click *Allocate + IP*. You can optionally specify a description for the IP for your + convenience. + + .. figure:: baremetal/associate_pool.png + :alt: the Allocate Floating IP dialog + + The Allocate Floating IP dialog + +#. Once a Floating IP is allocated to your project, it will display in the list + view, and you can click the *Associate* button for the Floating IP to assign + it to a running or spawning instance. This button will bring up the *Manage + Floating IP Associations* dialog. + + .. figure:: baremetal/floating_ip_overview.png + :alt: The Floating IP list view with a Floating IP available + + The Floating IP list view with a Floating IP available + +#. In the dialog, select an instance from the "Port to be associated" dropdown. + Your instance's display name will be displayed here. Click *Associate* to + complete the process of assigning the IP to your instance. + + .. figure:: baremetal/associate_ip.png + :alt: The Manage Floating IP Associations dialog with an IP selected + + The Manage Floating IP Associations dialog with an IP selected + +#. If you go back to the *Instances* page, you should now see the *floating + IP* attached to the instance. + + .. figure:: baremetal/instanceswithip.png + :alt: An instance with an allocated Floating IP + + An instance with an allocated Floating IP \ No newline at end of file diff --git a/source/technical/cli.rst b/source/technical/cli.rst index a54ccb3..695d6da 100644 --- a/source/technical/cli.rst +++ b/source/technical/cli.rst @@ -1,268 +1,11 @@ -.. _cli: +:orphan: -============================= -Command Line Interface (CLI) -============================= +.. meta:: + :http-equiv=refresh: 0; url=./cli/index.html -Introduction -============ +This page has moved +=================== -The Command Line Interface (CLI) provides a way to interact with Chameleon -resources using shell and scripting tools. Chameleon uses the `OpenStack Client -`_ to provide CLI -functionality. This documentation section provides an overview on how to install -the client and configure your shell environment to access Chameleon features. +This page has been reorganized. You will be redirected to the new location. -.. attention:: - - Some of the Chameleon features are **only** accessable via the CLI, such as - the Gnocchi metrics and the advanced networking features. - -.. note:: - - Chameleon Cloud is primarily designed to support Unix-like environments. - Threfore, it is highly recommended using CLI in a Unix-like system. For - Windows 10 users, you may want to enable `Windows Subsystem for Linux - `_ to get better - experience with the Chameleon CLI. - -.. _cli-installing: - -Installing the CLI -================== - -Prerequisites -------------- - -#. **Python** - Check if you have Python installed. - -#. **pip** - If you’re using Python 3.4 (or greater), then pip comes installed - with Python by default. - -OpenStack Client Installation ------------------------------ - -#. Install the CLI by typing ``pip install python-openstackclient`` in the - terminal. - -#. Verify that it has installed correctly by typing ``openstack``. You will - enter the Openstack Client in interactive mode and your prompt should change - to ``(openstack)``. - -#. Exit the client by typing ``exit``. - -#. There are some clients with new features or bugfixes not yet in the upstream - release branches, notably the Blazar CLI client. If you want to make - reservations via the CLI, you should install that here: - - .. code-block:: shell - - pip install git+https://github.com/chameleoncloud/python-blazarclient@chameleoncloud/2023.1 - -.. _cli-authentication: - -CLI authentication -================== - -When using the CLI, you have to provide some credentials so the system trusts -that the operations are really being executed by your user account. There are -two ways of doing this. - -Setting a CLI password ----------------------- - -You can set a CLI password via the `Chameleon Authentication Portal -`_. The -password you associate with your account can not be used to log in to the GUI or -Jupyter interfaces and can only be used to authenticate a command-line client. - -.. figure:: cli/set_cli_password.png - :alt: Setting a password in the Chameleon Authentication Portal - - Setting a password in the Chameleon Authentication Portal - -The benefit of this method is that this password will work on any Chameleon -site. - -.. note:: - - You should set a strong password for your CLI password, and it should not be - a password you use elsewhere. Otherwise, your account risks being compromised - by an attacker who has possibly obtained your password from another breached - service. We **highly** recommend using a password manager e.g., `BitWarden - `_, `LastPass - `, or `1Password - `_ to assist. - -.. _cli-application-credential: - -Creating an application credential ----------------------------------- - -You can also generate *application credentials*, which act as dedicated one-off -passwords that are authorized with the same permissions as your user account, -within a single project. If you work on multiple projects simultaneously, you -will need to generate one application credential for each project. - -To create an application credential, navigate to the "Identity" dashboard in the -:ref:`gui`, and go to the "Application Credentials" panel. Create a new -application credential and name it something meaningful (such as "CLI access for -project CH-XXX"). **You will also need to check the "unrestricted" checkbox in -order to use the CLI to make leases in Blazar**. If you do not need to make -reservations via the CLI, you can leave the box unchecked, as it is the safer -option. - -.. figure:: cli/applicationcredentials.png - :alt: Creating an application credential via the GUI - -Once the system generates the credential, you will be given the option to -download an :ref:`RC file ` that configures the CLI to use the -application credential for authentication. You will only see the secret -credentials once, so make sure to save the RC file or the secret somewhere, as -if it's lost, you will have to delete the credential and create a new one. - -.. _cli-rc-script: - -The OpenStack RC Script -======================= - -You must use the *OpenStack RC Scripts* to configure the environment variables -for accessing Chameleon features. You can downloaded the script from the -Chameleon GUI at the :ref:`gui-api-access`. - -.. hint:: - - If you use the Chameleon supported (CC) images, you'll find an ``openrc`` - file with a service token in the home directory for the ``cc`` user. The file - will be auto-sourced when you login, so you can use the - :ref:`openstack ` and the :ref:`swift ` CLI - directly, as well as the - :ref:`cc-snapshot ` tool. - -#. Log in to the GUI at |CHI@TACC| or |CHI@UC|. - - .. important:: - - Download the RC file from the site you would like to interact with. The - RC files are different for each site. - -#. Select the project you wish to access via :ref:`gui-project-menu`. - - .. figure:: gui/project_dropdown.png - :alt: The Project Dropdown - - The Project Dropdown - -#. Download *OpenStack RC Script* using :ref:`gui-user-menu` by clicking on - *Openstack RC File v3*. - - .. figure:: cli/userdropdown.png - :alt: The OpenStack RC File v3 link in the User Dropdown - - The OpenStack RC File v3 link in the User Dropdown - -#. Run the following command in the terminal: - - .. code-block:: shell - - source - - .. note:: - - The command **will not** work for Windows users. Skip this step and the - next step if you are using Windows system. - -#. Enter your password when prompted. - -#. For macOS/Linux users, your current terminal session has been configured to - access your project. Now type ``openstack`` in your terminal session. - - For Windows users, you have to provide the environment variables in the - *OpenStack RC* script as ``openstack`` command parameters. Run the following - command in your Windows prompt: - - .. code-block:: shell - - openstack --os-auth-url \ - --os-project-id \ - --os-project-name \ - --os-user-domain-name \ - --os-username \ - --os-password \ - --os-region-name \ - --os-interface \ - --os-identity-api-version - - Replace values of the parameters by reading from the *OpenStack RC* script. - - Another way to configure the OpenStack client for Windows users is to - add/edit environment variables manually via *System Properties* window. Then, - click on *Environment Variables...* button and manually add/edit the - environment variables in *OpenStack RC Script* to *Environment Variable* - window. - - .. figure:: cli/systemproperties.png - :alt: System Properties Window of Windows System - - System Properties Window of Windows System - - .. note:: - - For macOS/Linux users, every time when open a new terminal, you have to - run the ``source`` command to access the OpenStack client. - - .. error:: - - If you get authentication error, check if you input your password - correctly. - -#. Type ``project list`` at the ``(openstack)`` prompt. You should see a list of - the projects you belong to. - - .. error:: - - If you get permission error at this step, please check that: - - - the terminal session has been configured correctly with the environment - variables - - - the *OpenStack RC* script you ``source`` is **v3** - - - the OpenStack client version is the latest. To check the OpenStack - client version, use ``openstack --version`` command. Some older versions - may cause errors. - - .. error:: - - If you get the ``Missing value`` error when using a command, it is likely - that your terminal session has not been configured correctly and - completely with the environment variables. The error may be fixed by - re-running the ``source`` command over the OpenStack RC Script or using - the command line switches. - -.. _using-cli: - -Using the CLI -============= - -You can use the CLI in either Interactive Mode or Shell Mode. In either mode, -the OpenStack client has to be configured by using the *OpenStack RC Script* or -by providing the command line switches. For more information about the usage of -the OpenStack client, run ``openstack --help``. - -Interactive Mode ----------------- - -The Interactive Mode allows you to use the ``openstack`` commands through an -interactive prompt. To start the Interactive Mode, type ``openstack`` in the -configured terminal. Once entering the Interactive Mode, you will see a -``(openstack)`` prompt. Type the command you would like to run at the prompt. To -find out the commands, type ``help``. - -Shell Mode ----------- - -Each CLI command can be used in your terminal exactly the same way that it -appears in the Interactive Mode, simply by preceding the command with -``openstack``. For example, the command ``image list`` in the Interactive Mode -is equivalent to the command ``openstack image list`` in the Shell Mode. +If you are not redirected automatically, please visit :doc:`cli/index`. \ No newline at end of file diff --git a/source/technical/cli/authentication.rst b/source/technical/cli/authentication.rst new file mode 100644 index 0000000..3b12d79 --- /dev/null +++ b/source/technical/cli/authentication.rst @@ -0,0 +1,61 @@ +.. _cli-authentication: + +CLI authentication +================== + +When using the CLI, you have to provide some credentials so the system trusts +that the operations are really being executed by your user account. There are +two ways of doing this. + +Setting a CLI password +---------------------- + +You can set a CLI password via the `Chameleon Authentication Portal +`_. The +password you associate with your account can not be used to log in to the GUI or +Jupyter interfaces and can only be used to authenticate a command-line client. + +.. figure:: set_cli_password.png + :alt: Setting a password in the Chameleon Authentication Portal + + Setting a password in the Chameleon Authentication Portal + +The benefit of this method is that this password will work on any Chameleon +site. + +.. note:: + + You should set a strong password for your CLI password, and it should not be + a password you use elsewhere. Otherwise, your account risks being compromised + by an attacker who has possibly obtained your password from another breached + service. We **highly** recommend using a password manager e.g., `BitWarden + `_, `LastPass + `_, or `1Password + `_ to assist. + +.. _cli-application-credential: + +Creating an application credential +---------------------------------- + +You can also generate *application credentials*, which act as dedicated one-off +passwords that are authorized with the same permissions as your user account, +within a single project. If you work on multiple projects simultaneously, you +will need to generate one application credential for each project. + +To create an application credential, navigate to the "Identity" dashboard in the +:ref:`gui`, and go to the "Application Credentials" panel. Create a new +application credential and name it something meaningful (such as "CLI access for +project CH-XXX"). **You will also need to check the "unrestricted" checkbox in +order to use the CLI to make leases in Blazar**. If you do not need to make +reservations via the CLI, you can leave the box unchecked, as it is the safer +option. + +.. figure:: applicationcredentials.png + :alt: Creating an application credential via the GUI + +Once the system generates the credential, you will be given the option to +download an :ref:`RC file ` that configures the CLI to use the +application credential for authentication. You will only see the secret +credentials once, so make sure to save the RC file or the secret somewhere, as +if it's lost, you will have to delete the credential and create a new one. \ No newline at end of file diff --git a/source/technical/cli/index.rst b/source/technical/cli/index.rst new file mode 100644 index 0000000..02ac8a2 --- /dev/null +++ b/source/technical/cli/index.rst @@ -0,0 +1,33 @@ +.. _cli: + +============================= +Command Line Interface (CLI) +============================= + +The Command Line Interface (CLI) provides a way to interact with Chameleon +resources using shell and scripting tools. Chameleon uses the `OpenStack Client +`_ to provide CLI +functionality. This documentation section provides an overview on how to install +the client and configure your shell environment to access Chameleon features. + +.. attention:: + + Some of the Chameleon features are **only** accessible via the CLI, such as + power monitoring tools and the advanced networking features. + +.. note:: + + Chameleon Cloud is primarily designed to support Unix-like environments. + Therefore, it is highly recommended using CLI in a Unix-like system. For + Windows 10 users, you may want to enable `Windows Subsystem for Linux + `_ to get better + experience with the Chameleon CLI. + +.. toctree:: + :maxdepth: 1 + :caption: CLI Topics + + installation + authentication + rc_script + usage \ No newline at end of file diff --git a/source/technical/cli/installation.rst b/source/technical/cli/installation.rst new file mode 100644 index 0000000..350af7e --- /dev/null +++ b/source/technical/cli/installation.rst @@ -0,0 +1,32 @@ +.. _cli-installing: + +Installing the CLI +================== + +Prerequisites +------------- + +#. **Python** - Check if you have Python installed. + +#. **pip** - If you're using Python 3.4 (or greater), then pip comes installed + with Python by default. + +OpenStack Client Installation +----------------------------- + +#. Install the CLI by typing ``pip install python-openstackclient`` in the + terminal. + +#. Verify that it has installed correctly by typing ``openstack``. You will + enter the OpenStack Client in interactive mode and your prompt should change + to ``(openstack)``. + +#. Exit the client by typing ``exit``. + +#. There are some clients with new features or bugfixes not yet in the upstream + release branches, notably the Blazar CLI client. If you want to make + reservations via the CLI, you should install that here: + + .. code-block:: shell + + pip install git+https://github.com/chameleoncloud/python-blazarclient@chameleoncloud/2023.1 \ No newline at end of file diff --git a/source/technical/cli/rc_script.rst b/source/technical/cli/rc_script.rst new file mode 100644 index 0000000..e6d1b05 --- /dev/null +++ b/source/technical/cli/rc_script.rst @@ -0,0 +1,118 @@ +.. _cli-rc-script: + +The OpenStack RC Script +======================= + +You must use the *OpenStack RC Scripts* to configure the environment variables +for accessing Chameleon features. You can downloaded the script from the +Chameleon GUI at the :ref:`gui-api-access`. + +.. hint:: + + If you use the Chameleon supported (CC) images, you'll find an ``openrc`` + file with a service token in the home directory for the ``cc`` user. The file + will be auto-sourced when you login, so you can use the + :ref:`openstack ` and the :ref:`swift ` CLI + directly, as well as the + :ref:`cc-snapshot utility ` tool. + +#. Log in to the GUI at |CHI@TACC| or |CHI@UC|. + + .. important:: + + Download the RC file from the site you would like to interact with. The + RC files are different for each site. + +#. Select the project you wish to access via :ref:`gui-project-menu`. + + .. figure:: ../gui/project_dropdown.png + :alt: The Project Dropdown + + The Project Dropdown + +#. Download *OpenStack RC Script* using :ref:`gui-user-menu` by clicking on + *Openstack RC File v3*. + + .. figure:: userdropdown.png + :alt: The OpenStack RC File v3 link in the User Dropdown + + The OpenStack RC File v3 link in the User Dropdown + +#. Run the following command in the terminal: + + .. code-block:: shell + + source + + .. note:: + + The command **will not** work for Windows users. Skip this step and the + next step if you are using Windows system. + +#. Enter your password when prompted. + +#. For macOS/Linux users, your current terminal session has been configured to + access your project. Now type ``openstack`` in your terminal session. + + For Windows users, you have to provide the environment variables in the + *OpenStack RC* script as ``openstack`` command parameters. Run the following + command in your Windows prompt: + + .. code-block:: shell + + openstack --os-auth-url \ + --os-project-id \ + --os-project-name \ + --os-user-domain-name \ + --os-username \ + --os-password \ + --os-region-name \ + --os-interface \ + --os-identity-api-version + + Replace values of the parameters by reading from the *OpenStack RC* script. + + Another way to configure the OpenStack client for Windows users is to + add/edit environment variables manually via *System Properties* window. Then, + click on *Environment Variables...* button and manually add/edit the + environment variables in *OpenStack RC Script* to *Environment Variable* + window. + + .. figure:: systemproperties.png + :alt: System Properties Window of Windows System + + System Properties Window of Windows System + + .. note:: + + For macOS/Linux users, every time when open a new terminal, you have to + run the ``source`` command to access the OpenStack client. + + .. error:: + + If you get authentication error, check if you input your password + correctly. + +#. Type ``project list`` at the ``(openstack)`` prompt. You should see a list of + the projects you belong to. + + .. error:: + + If you get permission error at this step, check that: + + - the terminal session has been configured correctly with the environment + variables + + - the *OpenStack RC* script you ``source`` is **v3** + + - the OpenStack client version is the latest. To check the OpenStack + client version, use ``openstack --version`` command. Some older versions + may cause errors. + + .. error:: + + If you get the ``Missing value`` error when using a command, it is likely + that your terminal session has not been configured correctly and + completely with the environment variables. The error may be fixed by + re-running the ``source`` command over the OpenStack RC Script or using + the command line switches. \ No newline at end of file diff --git a/source/technical/cli/usage.rst b/source/technical/cli/usage.rst new file mode 100644 index 0000000..cb24e63 --- /dev/null +++ b/source/technical/cli/usage.rst @@ -0,0 +1,26 @@ +.. _using-cli: + +Using the CLI +============= + +You can use the CLI in either Interactive Mode or Shell Mode. In either mode, +the OpenStack client has to be configured by using the *OpenStack RC Script* or +by providing the command line switches. For more information about the usage of +the OpenStack client, run ``openstack --help``. + +Interactive Mode +---------------- + +The Interactive Mode allows you to use the ``openstack`` commands through an +interactive prompt. To start the Interactive Mode, type ``openstack`` in the +configured terminal. Once entering the Interactive Mode, you will see a +``(openstack)`` prompt. Type the command you would like to run at the prompt. To +find out the commands, type ``help``. + +Shell Mode +---------- + +Each CLI command can be used in your terminal exactly the same way that it +appears in the Interactive Mode, simply by preceding the command with +``openstack``. For example, the command ``image list`` in the Interactive Mode +is equivalent to the command ``openstack image list`` in the Shell Mode. \ No newline at end of file diff --git a/source/technical/complex.rst b/source/technical/complex.rst index 78a00e1..02255ee 100644 --- a/source/technical/complex.rst +++ b/source/technical/complex.rst @@ -1,841 +1,11 @@ -.. _complex: +:orphan: -Complex appliances -================== +.. meta:: + :http-equiv=refresh: 0; url=./complex/index.html -Introduction ------------- +This page has moved +=================== -Deploying an MPI cluster, an OpenStack installation, or any other type of cluster in which nodes can take on multiple roles can be complex: you have to provision potentially hundreds of nodes, configure them to take on various roles, and make them share information that is generated or assigned only at deployment time, such as hostnames, IP addresses, or security keys. When you want to run a different experiment later you have to redo all this work. When you want to reproduce the experiment, or allow somebody else to reproduce it, you have to take very precise notes and pay great attention to their execution. +This page has been reorganized. You will be redirected to the new location. -To help solve this problem and facilitate reproducibility and sharing, the Chameleon team configured a tool that allows you to deploy complex clusters with “one click”. This tool requires not just a simple *image* (i.e., appliance) but also a document, called a *template*, that contains the information needed to orchestrate the deployment and configuration of such clusters. We call this *image + template* combination **Complex Appliances** because it consists of more than just the image (i.e., appliance). - -In a nutshell, *Complex Appliances* allow you to specify not only what image you want to deploy but also on how many nodes you want to deploy that image, what roles the deployed instances should boot into (such as e.g., head node and worker node in a cluster), what information from a specific instance should be passed to another instance in that *Complex Appliance*, and what scripts should be executed on boot so that this information is properly used for configuring the “one click” cluster. - -This guide will tell you all you need to know in order to use and configure *Complex Appliances* on Chameleon. - -.. hint:: - Since *Complex Appliances* in Chameleon are currently implemented using the `OpenStack Heat `_ orchestration service, we will be using OpenStack terminology and features to work with them. The templates described above are YAML files using the `Heat Orchestration Template (HOT) `_ format (Heat also supports the AWS CloudFormation template format, but this is not covered here). A deployed complex appliance is referred to as a “stack” – just as a deployed single appliance is typically referred to as an “instance”. - -Complex Appliances in the Catalog ---------------------------------- - -The `Chameleon Appliance Catalog `_ has several *Complex Appliances* for popular technologies that people want to deploy such as OpenStack or MPI or even more advanced deployments such as efficient SR-IOV enabled MPI in KVM virtual machines. We also provide common building blocks for cluster architectures, such as an NFS share. *Complex Appliances* are identified by a badge in their top-right corner representing a group of machines, as in the screenshot below: - -.. figure:: complex/nfsappliance.png - :alt: A Complex Appliance with a badge in the upper right - - A Complex Appliance with a badge in the upper right - -To view the details of a *Complex Appliance*, simply click on it. - -.. figure:: complex/nfsappliancedetail.png - :alt: A Complex Appliance page - - A Complex Appliance page - -.. tip:: You may download the *Template* file or copy the *Template* file URL to clipboard by clicking the *Get Template* button. The *Template* file or it's URL is required when launching a *Complex Appliance*. - -Managing Complex Appliances using the GUI ------------------------------------------ - -Before launching a *Complex Appliance*, please make sure that you have a reservation for the appropriate node types and a key pair configured. Since most *Complex Appliances* will consist of multiple nodes, make sure you have set the *Minimum Number of Hosts* in your Lease. You will also need a *Template* file or the URL for a *Template* file from the `Appliance Catalog `_. At |CHI@TACC| site or |CHI@UC| site, go to *Project* > *Orchestration* > *Stacks* use the navigation side bar. - -.. figure:: complex/stacks.png - :alt: The Stacks page - - The Stacks page - -.. tip:: - You can go to *Stacks* page directly from the `Appliance Catalog `_. - - #. Go to the `Appliance Catalog `_ and identify the appliance you want to launch. Click on it to open its details page. - - #. Click on the "Launch Complex Appliance at ``CHI\@TACC``" or "Launch Complex Appliance at ``CHI\@UC``" button depending on where your reservation is created. - - -Launching a Complex Appliance -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -To launch a stack, click the *Launch Stack* button in the upper right of the *Stacks* page. Then follow the steps: - -#. Start setting up a *Template* by choosing a *Template Source* in the dropdown. You may either select the *File* option as *Template Source* and upload the *Template* file, or select the *URL* option and provide the URL of the *Template* file. - - .. figure:: complex/selecttemplate.png - :alt: The Select Template step - - The Select Template step - - .. important:: **Do not** change the environment source settings! - -#. Once you have provided a Template, click the *Next* button. Chameleon will validate the Template file and proceed to the *Launch Stack* step. - - .. figure:: complex/launchstack.png - :alt: The Launch Stack step - - The Launch Stack step - -#. Choose a name for your stack. Ignore the “Creation Timeout” and “Rollback On Failure” settings. You also need to enter your Chameleon password. Then, you need to select a value for the parameters of the template. Finally, click the *Launch* button. -#. Your stack should be in status "Create In Progress" for several minutes while it first launches the server instance, followed by the client instances. It will then move to the status "Create Complete". - -.. figure:: complex/createinprogress.png - :alt: A Complex Appliance with the Create in Progress status - - A Complex Appliance with the Create in Progress status - -Monitoring a Complex Appliance -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -To monitor and get more details about your *Complex Appliance*, click on it in the *Stacks* page. - -- The *Topology* tab displays a topology graph of the stack. The rack of machine represents the client instance group. The server’s floating IP (the public IP assigned to a resource) is represented by an IP in a circle; while an IP in a circle is also used to represent the association of the IP with the server instance (not the greatest idea to use the same symbol for both the IP and the association -- we agree but can’t do much about it at the moment). Blow off some steam by dragging the visualization across the screen, it can be rather fun! - - .. note:: Blinking nodes indicates that they are still provisioning. - - .. figure:: complex/topology.png - :alt: The Topology tab - - The Topology tab - -- The *Overview* tab displays various parameters, including the *ID* of the stack and *Outputs* such as IP addresses assigned to each node. If you have a floating IP associated to the server, you can now ``ssh`` to the server using the floating IP just as you do with regular instances. The client may not have a floating IP attached to it, but you can connect to it via the server node with the client’s private IP. - - .. tip:: To talk to the client without an associated floating IP, connect to the server with ``ssh -A`` to enable the SSH agent forwarding after loading your key to your SSH agent with ``ssh-add ``. - - .. figure:: complex/overview.png - :alt: The Overview tab - - The Overview tab - -- Under the *Resources* tab you will see the resources of the stack (the server, clients, server’s public/floating IP, and its the association) and information about them. - - .. figure:: complex/resources.png - :alt: The Resources tab - - The Resources tab - -- In the *Events* tab you will see information about the history of the deployment so far. - - .. figure:: complex/events.png - :alt: The Events tab - - The Events tab - -- In *Template* tab, you will see the template that was used to deploy this stack. - - .. figure:: complex/template.png - :alt: The Template tab - - The Template tab - -Deleting a Complex Appliance -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -To delete a *Complex Appliance*, select it in the *Stacks* page and click the *Delete Stacks* button. This will delete all resources of the stack, such as nodes and floating IP addresses. - -Managing Complex Appliances using the CLI ------------------------------------------ - -.. tip:: Reading :ref:`cli` is highly recommanded before continuing on the following sections. - -In addition to :ref:`cli-installing`, you will need to install the ``python-heatclient`` package using the command: - -.. code-block:: bash - - pip install python-heatclient - -Then, set up your environment for OpenStack command line usage, as described in :ref:`cli-rc-script`. You can get a list of your *Complex Appliances* in your project using the command: - -.. code-block:: bash - - openstack stack list - -The output should look like the following: - -.. code:: - - +--------------------------------------+---------------+-------------------+----------------------+----------------------+ - | ID | Stack Name | Stack Status | Creation Time | Updated Time | - +--------------------------------------+---------------+-------------------+----------------------+----------------------+ - | e5df33b5-5282-4935-8097-973328ca71e5 | my_stack | CREATE_COMPLETE | 2018-01-23T22:45:12Z | None | - +--------------------------------------+---------------+-------------------+----------------------+----------------------+ - -Launching a Complex Appliance -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -To launch a *Complex Appliance* using *Template*, run the command on your local machine: - -.. code-block:: bash - - openstack stack create --template --parameter = - -Provide the path to and the name of the *Template* file in your local file system via the ``template`` switch. The ```` is the name of the *Complex Appliance*. In addition, you may provide the parameters required in the *Template* file with their values by ``parameter`` switch. For example, the `NFS Server Template `_ lists the following ``parameters`` section: - -.. code:: - - parameters: - nfs_client_count: - type: number - description: Number of NFS client instances - default: 1 - constraints: - - range: { min: 1 } - description: There must be at least one client. - key_name: - type: string - description: Name of a KeyPair to enable SSH access to the instance - default: default - constraints: - - custom_constraint: nova.keypair - reservation_id: - type: string - description: ID of the Blazar reservation to use for launching instances. - constraints: - - custom_constraint: blazar.reservation - -Therefore, in order to use this *Template*, you must provide values for ``nfs_client_count``, ``key_name`` and ``reservation_id``. - -Monitoring a Complex Appliance -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -You can get details about your *Complex Appliance*, such as *Outputs*, *Events* and *Resources*, via the CLI. You will need the *UUID* of the *Complex Appliance*. - -.. tip:: To get the *UUID* of your *Complex Appliance*, use the *Stacks* page on the GUI or retrieve it by ``openstack stack list`` command. - -- To view the *Outputs*, run: - - .. code-block:: bash - - openstack stack output list - - For example, the list of the outputs for the `NFS Share `_ stack is: - - .. code:: - - +------------+-----------------------------------------+ - | output_key | description | - +------------+-----------------------------------------+ - | client_ips | Private IP addresses of the NFS clients | - | server_ip | Public IP address of the NFS server | - +------------+-----------------------------------------+ - - You can get more details about the outputs by using the following command: - - .. code-block:: bash - - openstack stack output show --all - -- To view the *Events*, run: - - .. code-block:: bash - - openstack stack event list - -- To view the *Resources*, run: - - .. code-block:: bash - - openstack stack resource list - - Your output may look like this: - - .. code:: - - +---------------------------+--------------------------------------+---------------------------------+-----------------+----------------------+ - | resource_name | physical_resource_id | resource_type | resource_status | updated_time | - +---------------------------+--------------------------------------+---------------------------------+-----------------+----------------------+ - | nfs_server_ip_association | | OS::Neutron::FloatingIPAssociation | INIT_COMPLETE | 2018-03-19T18:38:05Z | - | nfs_server | 0ab0169c-f762-4d27-8724-b359caa50f1f | OS::Nova::Server | CREATE_FAILED | 2018-03-19T18:38:05Z | - | nfs_server_floating_ip | ecb391f8-4653-43a6-b2c6-bb93a6d89115 | OS::Nova::FloatingIP | CREATE_COMPLETE | 2018-03-19T18:38:05Z | - | nfs_clients | | OS::Heat::ResourceGroup | INIT_COMPLETE | 2018-03-19T18:38:05Z | - +---------------------------+--------------------------------------+---------------------------------+-----------------+----------------------+ - - Then, you may retrieve information about a specific resource using the command: - - .. code-block:: bash - - openstack stack resource show - -Deleting a Complex Appliance -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Use the following command to delete a stack: - -.. code-block:: bash - - openstack stack delete - -It will remove all the resources attached to the stack. - -Heat Orchestration Templates ----------------------------- - -A *Heat Orchestration Template* is a YAML file that specifies how resources are used and configured in a *Complex Appliance*. - -A Case Example: NFS Share -~~~~~~~~~~~~~~~~~~~~~~~~~ - -Let's look at the `NFS Share Template `_. The NFS share appliance deploys: - -- An NFS server instance, that exports the directory ``/exports/example`` to any instance running on Chameleon bare metal, -- One or several NFS client instances, which configure ``/etc/fstab`` to mount this NFS share to ``/mnt`` (and can subsequently read from and write to it). - -This template is reproduced further below, and includes inline comments starting with the ``#`` character. There are three main sections: - -- resources -- parameters -- outputs - -The ``resources`` section is the most important part of the template: it defines which OpenStack *Resources* to create and configure. Inside this section you can see four resources defined: - -- ``nfs_server_floating_ip``: creates a *Floating IP* on the ``ext-net`` public network. It is not attached to any instance yet. -- ``nfs_server``: creates the NFS server instance (an instance is defined with the type ``OS::Nova::Server`` in *Heat*). It is a bare metal instance (``flavor: baremetal``) using the ``CC-CentOS7`` image and connected to the private network named ``sharednet1``. We set the key pair to use the value of the parameter defined earlier, using the ``get_param`` function. Similarly, the reservation to use is passed to the scheduler. Finally, a ``user_data`` script is given to the instance, which configures it as an NFS server exporting ``/exports/example`` to Chameleon instances. -- ``nfs_server_ip_association``: associates the floating IP created earlier with the NFS server instance. -- ``nfs_clients``: defines a resource group containing instance configured to be NFS clients and mount the directory exported by the NFS server defined earlier. The IP of the NFS server is gathered using the ``get_attr`` function, and placed into ``user_data`` using the ``str_replace`` function. - -Once a Resource has been specified, you may provide it as a value for another Resource's property using the ``get_resource`` function. - -The ``parameters`` section defines inputs to be used on *Complex Appliance* launch. Parameters all have the same data structure: each one has a name (``key_name`` or ``reservation_id`` in this case), a data type (``number`` or ``string``), a comment field called ``description``, optionally a ``default value``, and a list of ``constraints`` (in this case only one per parameter). Constraints tell *Heat* to match a parameter to a specific type of OpenStack resource. *Complex appliances* on Chameleon require users to customize at least the key pair name and reservation ID, and will generally provide additional parameters to customize other properties of the cluster, such as its size, as in this example. The values of Parameters can be used in the ``resources`` section using the ``get_param`` function. - -The ``outputs`` section defines what values are returned to the user. *Outputs* are declared similarly to *Parameters*: they each have a name, an optional description, and a value. They allow to return information from the stack to the user. You may use the ``get_attr`` function to retrieve a resource's attribute for output. - -Heat Template Customization -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Customizing an existing template is a good way to start developing your own. We will use a simpler template than the previous example to start with: it is the `Hello World complex appliance `_. - -First, delete the stack you launched, because we will need all three nodes to be free. To do this, go back to the *Project* > *Orchestration* > *Stacks* page, select your stack, and then click on the *Delete Stacks* button. You will be asked to confirm, so click on the *Delete Stacks* button. - - .. figure:: complex/deletestacks.png - :alt: Confirm deleting stack dialog - - Confirm deleting stack dialog - -The template for the `Hello World complex appliance `_ is reproduced below. It is similar to the NFS share appliance, except that it deploys only a single client. You can see that it has four resources defined: - -- ``nfs_server_floating_ip`` -- ``nfs_server`` -- ``nfs_server_ip_association`` -- ``nfs_client`` - -The ``nfs_client`` instance mounts the NFS directory shared by the ``nfs_server`` instance, just like in our earlier example. - -:: - - # This describes what is deployed by this template. - description: NFS server and client deployed with Heat on Chameleon - - # This defines the minimum Heat version required by this template. - heat_template_version: 2015-10-15 - - # The resources section defines what OpenStack resources are to be deployed and - # how they should be configured. - resources: - nfs_server_floating_ip: - type: OS::Nova::FloatingIP - properties: - pool: ext-net - - nfs_server: - type: OS::Nova::Server - properties: - flavor: baremetal - image: CC-CentOS7 - key_name: { get_param: key_name } - networks: - - network: sharednet1 - scheduler_hints: { reservation: { get_param: reservation_id } } - user_data: | - #!/bin/bash - yum install -y nfs-utils - mkdir -p /exports/example - chown -R cc:cc /exports - echo '/exports/example 10.140.80.0/22(rw,async) 10.40.0.0/23(rw,async)' >> /etc/exports - systemctl enable rpcbind && systemctl start rpcbind - systemctl enable nfs-server && systemctl start nfs-server - - nfs_server_ip_association: - type: OS::Neutron::FloatingIPAssociation - properties: - floatingip_id: {get_resource: nfs_server_floating_ip} - port_id: {get_attr: [nfs_server, addresses, sharednet1, 0, port]} - - nfs_client: - type: OS::Nova::Server - properties: - flavor: baremetal - image: CC-CentOS7 - key_name: { get_param: key_name } - networks: - - network: sharednet1 - scheduler_hints: { reservation: { get_param: reservation_id } } - user_data: - str_replace: - template: | - #!/bin/bash - yum install -y nfs-utils - echo "$nfs_server_ip:/exports/example /mnt/ nfs" > /etc/fstab - mount -a - params: - $nfs_server_ip: { get_attr: [nfs_server, first_address] } - - # The parameters section gathers configuration from the user. - parameters: - key_name: - type: string - description: Name of a KeyPair to enable SSH access to the instance - default: default - constraints: - - custom_constraint: nova.keypair - reservation_id: - type: string - description: ID of the Blazar reservation to use for launching instances. - constraints: - - custom_constraint: blazar.reservation - -Download `this template `_ to your local machine, and open it in your favorite text editor. - -We will customize the template to add a second NFS client by creating a new resource called ``another_nfs_client``. Add the following text to your template inside the resources section. Make sure to respect the level of indentation, which is important in YAML. - -:: - - another_nfs_client: - type: OS::Nova::Server - properties: - flavor: baremetal - image: CC-CentOS7 - key_name: { get_param: key_name } - networks: - - network: sharednet1 - scheduler_hints: { reservation: { get_param: reservation_id } } - user_data: - str_replace: - template: | - #!/bin/bash - yum install -y nfs-utils - echo "$nfs_server_ip:/exports/example /mnt/ nfs" > /etc/fstab - mount -a - params: - $nfs_server_ip: { get_attr: [nfs_server, first_address] } - -Now, launch a new stack with this template. Since the customized template is only on your computer and cannot be addressed by a URL, use the *Direct Input* method instead and copy/paste the content of the customized template. The resulting topology view is shown below: as you can see, the two client instances are shown separately since each one is defined as a separate resource in the template. - - .. figure:: complex/topologycustomhelloworld.png - :alt: Topology of the customized Hello World Appliance - - Topology of the customized Hello World Appliance - -You may have realized already that while adding just one additional client instance was easy, launching more of them would require to copy / paste blocks of YAML many times while ensuring that the total count is correct. This would be easy to get wrong, especially when dealing with tens or hundreds of instances. - -So instead, we leverage another construct from *Heat*: resource groups. Resource groups allow to define one kind of resource and request it to be created any number of times. - -Remove the ``nfs_client`` and ``another_client`` resources from your customized template, and replace them with the following: - -:: - - nfs_clients: - type: OS::Heat::ResourceGroup - properties: - count: 2 - resource_def: - type: OS::Nova::Server - properties: - flavor: baremetal - image: CC-CentOS7 - key_name: { get_param: key_name } - networks: - - network: sharednet1 - scheduler_hints: { reservation: { get_param: reservation_id } } - user_data: - str_replace: - template: | - #!/bin/bash - yum install -y nfs-utils - echo "$nfs_server_ip:/exports/example /mnt/ nfs" > /etc/fstab - mount -a - params: - $nfs_server_ip: { get_attr: [nfs_server, first_address] } - -A resource group is configured with a properties field, containing the definition of the resource to launch (``resource_def``) and the number of resources to launch (``count``). Once launched, you will notice that the topology view groups all client instances under a single *Resource Group* icon. We use the same ``resource_def`` than when defining separate instances earlier. - -Another way we can customize this template is by adding outputs to the template. Outputs allow a *Heat* template to return data to the user. This can be useful to return values like IP addresses or credentials that the user must know to use the system. - -We will create an output returning the floating IP address used by the NFS server. We define an outputs section, and one output with the name ``server_ip`` and a description. The value of the output is gathered using the ``get_attr`` function which obtains the IP address of the server instance. - -:: - - outputs: - server_ip: - description: Public IP address of the NFS server - value: { get_attr: [nfs_server_floating_ip, ip] } - -You can get outputs in the *Overview* tab of the *Stack Details* page. If you want to use the command line, install ``python-heatclient`` and use the ``heat output-list`` and ``heat output-show`` commands, or get a full list in the information returned by ``heat stack-show``. - -Multiple outputs can be defined in the outputs section. Each of them needs to have a unique name. For example, we can add another output to list the private IPs assigned to client instances: - -:: - - client_ips: - description: Private IP addresses of the NFS clients - value: { get_attr: [nfs_clients, first_address] } - -The image below shows the resulting outputs as viewed from the web interface. Of course IP addresses will be specific to each deployment. - - .. figure:: complex/helloworldoutputs.png - :alt: The Outputs of customized Hello World appliance - - The Outputs of customized Hello World appliance - -Finally, we can add a new parameter to replace the hard-coded number of client instances by a value passed to the template. Add the following text to the parameters section: - -:: - - nfs_client_count: - type: number - description: Number of NFS client instances - default: 1 - constraints: - - range: { min: 1 } - description: There must be at least one client. - -Inside the resource group definition, change ``count: 2`` to ``count: { get_param: nfs_client_count }`` to retrieve and use the parameter we just defined. When you launch this template, you will see that an additional parameter allows you to define the number of client instances, like in the NFS share appliance. - -At this stage, we have fully recreated the *NFS share* appliance starting from the *Hello World* one! The next section will explain how to write a new template from scratch. - -Writing a New Template -~~~~~~~~~~~~~~~~~~~~~~ - -You may want to write a whole new template, rather than customizing an existing one. Each template should follow the same layout and be composed of the following sections: - -- Heat template version -- Description -- Resources -- Parameters -- Outputs - -Heat template version -^^^^^^^^^^^^^^^^^^^^^ - -Each Heat template has to include the ``heat_template_version`` key with a valid version of `HOT (Heat Orchestration Template) `_. Chameleon bare metal supports any HOT version up to **2015-10-15**, which corresponds to OpenStack Liberty. -The `Heat documentation `_ lists all available versions and their features. We recommended that you always use the latest Chameleon supported version to have access to all supported features: - -``heat_template_version: 2015-10-15`` - -Description -^^^^^^^^^^^ - -While not mandatory, it is good practice to describe what is deployed and configured by your template. It can be on a single line: - -:: - - description: This describes what this Heat template deploys on Chameleon. - -If a longer description is needed, you can provide multi-line text in YAML, for example: - -:: - - description: > - This describes what this Heat - template deploys on Chameleon. - -Resources -^^^^^^^^^ - -The resources section is required and must contain at least one resource definition. A `complete list of resources types known to Heat `_ is -available. - -However, only a subset of them are supported by Chameleon, and some are limited to administrative use. We recommend that you only use: - -- OS::Glance::Image -- OS::Heat::ResourceGroup -- OS::Heat::SoftwareConfig -- OS::Heat::SoftwareDeployment -- OS::Heat::SoftwareDeploymentGroup -- OS::Neutron::FloatingIP -- OS::Neutron::FloatingIPAssociation -- OS::Neutron::Port (advanced users only) -- OS::Nova::Keypair -- OS::Nova::Server - -If you know of another resource that you would like to use and think it should be supported by the OpenStack services on Chameleon bare metal, please let us know via our |Help Desk|. - -Parameters -^^^^^^^^^^ - -Parameters allow users to customize the template with necessary or optional values. -For example, they can customize which Chameleon appliance they want to deploy, or which key pair to install. -Default values can be provided with the ``default`` key, as well as constraints to ensure that only valid OpenStack resources can be selected. -For example, ``custom_constraint: glance.image`` restricts the image selection to an available OpenStack image, while providing a pre-filled selection box in the web interface. -`More details about constraints `_ are available in the *Heat* documentation. - -Outputs -^^^^^^^ - -Outputs allow template to give information from the deployment to users. This can include usernames, passwords, IP addresses, hostnames, paths, etc. The outputs declaration is using the following format: - -:: - - outputs: - first_output_name: - description: Description of the first output - value: first_output_value - second_output_name: - description: Description of the second output - value: second_output_value - -Generally values will be calls to ``get_attr``, ``get_param``, or some other function to get information from parameters or resources deployed by the -template and return them in the proper format to the user. - - -Reserved Networks and Floating IPs -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Chameleon's reservation service allows users to reserve VLAN segments and floating ips. In order to make use of these -reserved resources in a (HOT) template, follow the guidelines below. For more information on VLAN and floating ip reservations, -see documentaiton on :ref:`reservation-cli-vlan` and :ref:`reservation-cli-fip` - -When you reserve a VLAN segment via blazar, it will automatically create a network for you. However, this network -is not usable in your template unless a subnet and router have been associated with the network. Once this is done, you can simply -add the network name as the network parameter for your server as you would ``sharednet1``. The below cli commands -provides an example of how to complete the setup for your reserved network. - -:: - - openstack subnet create --subnet-range 192.168.100.0/24 \ - --allocation-pool start=192.168.100.100,end=192.168.100.108 \ - --dns-nameserver 8.8.8.8 --dhcp \ - --network \ - my_subnet_name - openstack router create my_router_name - openstack router add subnet my_router_name my_subnet_name - openstack router set --external-gateway public my_router_name - -For reserved floating ips, you need to associate the floating ip with a server using the ``OS::Neutron::FloatingIPAssociation`` object type. -Many of our older complex appliance templates use the ``OS::Nova::FloatingIPAssociation`` object, but this has since been deprecated. See example below -for proper usage: - -:: - - my_server_ip_association: - type: OS::Neutron::FloatingIPAssociation - properties: - floatingip_id: - port_id: {get_attr: [my_server, addresses, , 0, port]} - - -If you are having trouble finding the ``uuid`` of the floating ip address then the below command will help you. - -:: - - openstack floating ip list -c ID -c "Floating IP Address" -c Tags --long - -The output should look like the sample output below with the `uuid` listed under the `ID` column. You can check your lease in -the reservation section of the GUI to find the `reservation id` associated with the floating ip in the `Tags` section of the output. - -:: - - +--------------------------------------+---------------------+------------------------------------------------------------------+ - | ID | Floating IP Address | Tags | - +--------------------------------------+---------------------+------------------------------------------------------------------+ - | 0fe31fad-60ac-462f-bb6c-4d40c1506621 | 192.5.87.206 | [u'reservation:d90ad917-300a-4cf7-a836-083534244f56', u'blazar'] | - | 92a347a9-31a5-43c1-80e2-9cdb38ebf66f | 192.5.87.224 | [u'reservation:5f470c97-0166-4934-a813-509b743e2d62', u'blazar'] | - | c8480d67-533d-4f55-a197-8271da6d9344 | 192.5.87.71 | [] | - +--------------------------------------+---------------------+------------------------------------------------------------------+ - - -Sharing Complex Appliances --------------------------- - -If you have written your own *Complex Appliance* or substantially customized an existing one, we would love if you shared them with our user community! The process is very similar to regular appliances: log into the Chameleon portal, go to the appliance catalog, and click on the button in the top-right corner: *Add an appliance* (you need to be logged in to see it). - -.. figure:: complex/addappliance.png - :alt: The Add an Appliance button - - The Add an Appliance button - -You will be prompted to enter a name, description, and documentation. Instead of providing appliance IDs, copy your template to the dedicated field. Finally, share your contact information and assign a version string to your appliance. Once submitted, your appliance will be reviewed. We will get in touch if a change is needed, but if it's all good we will publish it right away! - -Advanced Topics ---------------- - -.. _all-to-all-info-exchange: - -All-to-All Information Exchange -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The previous examples have all used ``user_data`` scripts to provide instances with contextualization information. While it is easy to use, this contextualization method has a major drawback: because it is given to the instance as part of its launch request, it cannot use any context information that is not yet known at this time. In practice, this means that in a client-server deployment, only one of these pattern will be possible: - -- The server has to be deployed first, and once it is deployed, the clients can be launched and contextualized with information from the server. The server won’t know about the clients unless there is a mechanism (not managed by *Heat*) for the client to contact the server. -- The clients have to be deployed first, and once they are deployed, the server can be launched and contextualized with information from the clients. The clients won’t know about the server unless there is a mechanism (not managed by *Heat*) for the server to contact the clients. - -This limitation was already apparent in our `NFS share `_ appliance: this is why the server instance exports the file system to all bare metal instances on Chameleon, because it doesn’t know which specific IP addresses are allocated to the clients. - -This limitation is even more important if the deployment is not hierarchical, i.e. all instances need to know about all others. For example, a cluster with IP and hostnames populated in ``/etc/hosts`` required each instance to be known by every other instance. - -This section presents a more advanced form of contextualization that can perform this kind of information exchange. -This is implemented by *Heat* agents running inside instances and communicating with the *Heat* service to send and receive information. -This means you will need to use an image bundling these agents. -Currently, all Chameleon-supported images (CC) are supporting this mode of contextualization. -If you build your own images using the `CC-CentOS7 `_ builder, `CC-CentOS `_ builder or `CC-Ubuntu `_ builder, you will automatically have these agents installed. This contextualization is performed with several Heat resources: - -- ``OS::Heat::SoftwareConfig``: This resource describes code to run on an instance. It can be configured with inputs and provide outputs. -- ``OS::Heat::SoftwareDeployment``: This resource applies a SoftwareConfig to a specific instance. -- ``OS::Heat::SoftwareDeploymentGroup``: This resource applies a SoftwareConfig to a specific group of instances. - - -The template below illustrates how it works. It launches a group of instances that will automatically populates their ``/etc/hosts`` file with IP and hostnames from other instances in the deployment. - -.. code:: - - heat_template_version: 2015-10-15 - - description: > - This template demonstrates how to exchange hostnames and IP addresses to populate /etc/hosts. - - parameters: - flavor: - type: string - default: baremetal - constraints: - - custom_constraint: nova.flavor - image: - type: string - default: CC-CentOS8 - constraints: - - custom_constraint: glance.image - key_name: - type: string - default: default - constraints: - - custom_constraint: nova.keypair - instance_count: - type: number - default: 2 - reservation_id: - type: string - description: ID of the Blazar reservation to use for launching instances. - constraints: - - custom_constraint: blazar.reservation - - resources: - export_hosts: - type: OS::Heat::SoftwareConfig - properties: - outputs: - - name: hosts - group: script - config: | - #!/bin/sh - (echo -n $(facter ipaddress); echo -n ' '; echo $(facter hostname)) > ${heat_outputs_path}.hosts - - export_hosts_sdg: - type: OS::Heat::SoftwareDeploymentGroup - properties: - config: { get_resource: export_hosts } - servers: { get_attr: [server_group, refs_map] } - signal_transport: HEAT_SIGNAL - - populate_hosts: - type: OS::Heat::SoftwareConfig - properties: - inputs: - - name: hosts - group: script - config: | - #!/usr/bin/env python - import ast - import os - import string - import subprocess - hosts = os.getenv('hosts') - if hosts is not None: - hosts = ast.literal_eval(string.replace(hosts, '\n', '\\n')) - with open('/etc/hosts', 'a') as hosts_file: - for ip_host in hosts.values(): - hosts_file.write(ip_host.rstrip() + '\n') - - populate_hosts_sdg: - type: OS::Heat::SoftwareDeploymentGroup - depends_on: export_hosts_sdg - properties: - config: { get_resource: populate_hosts } - servers: { get_attr: [server_group, refs_map] } - signal_transport: HEAT_SIGNAL - input_values: - hosts: { get_attr: [ export_hosts_sdg, hosts ] } - - server_group: - type: OS::Heat::ResourceGroup - properties: - count: { get_param: instance_count } - resource_def: - type: OS::Nova::Server - properties: - flavor: { get_param: flavor } - image: { get_param: image } - key_name: { get_param: key_name } - networks: - - network: sharednet1 - scheduler_hints: { reservation: { get_param: reservation_id } } - user_data_format: SOFTWARE_CONFIG - software_config_transport: POLL_SERVER_HEAT - - outputs: - deployment_results: - value: { get_attr: [export_hosts_sdg, hosts] } - -There are two ``SoftwareConfig`` resources: - -- The first ``SoftwareConfig``, ``export_hosts``, uses the ``facter`` tool to extract IP address and hostname into a single line (in the format expected for ``/etc/hosts``) and writes it to a special path (``${heat_outputs_path}.hosts``). This prompts Heat to assign the content of this file to the output with the name hosts. -- The second ``SoftwareConfig``, ``populate_hosts``, takes as input a variable named hosts, and applies a script that reads the variable from the environment, parses it with ``ast.literal_eval`` (as it is formatted as a Python dict), and writes each value of the dictionary to ``/etc/hosts``. - -The ``SoftwareDeploymentGroup`` resources ``export_hosts_sdg`` and ``populate_hosts_sdg`` apply each ``SoftwareConfig`` to the instance ``ResourceGroup`` with the correct configuration. - -Finally, the instance ``ResourceGroup`` is configured so that each instance uses the following contextualization method instead of a ``user_data`` script: - -.. code:: - - user_data_format: SOFTWARE_CONFIG - software_config_transport: POLL_SERVER_HEAT - -You can follow the same template pattern to configure your own deployment requiring all-to-all information exchange. - -.. _automated-deployemnt: - -Automated Deployment -~~~~~~~~~~~~~~~~~~~~ - -On Chameleon you can configure a Heat Stack to launch as soon as your lease begins. Whether your experiments require a large cluster or a single node, automated deployment saves you time configuring your environment and even allows you to run your entire experiment automatically when the necessary resources become available. - -At present, you will need to use our customized versions of the Heat and Blazar CLI tools to implement this feature. - -Install Custom CLI -^^^^^^^^^^^^^^^^^^ - -You can install Chameleon's ``python-heatclient`` and ``python-blazarclient`` packages via ``pip`` by running the following commands: - -.. code:: - - pip install git+https://github.com/ChameleonCloud/python-heatclient.git - pip install git+https://github.com/ChameleonCloud/python-blazarclient.git - - -Initialize Stack -^^^^^^^^^^^^^^^^ - -Next you will need to configure a Heat stack with the ``--initialize`` flag on the CLI and a dummy ``reservation_id`` parameter. The ``dummy`` id can be anything (even an empty string) so long as the ``reservation_id`` parameter is specified so that Blazar can overwrite it once your advanced reservation is scheduled and the stack is ready to launch. Once your stack is initialized, the status should read ``INIT_COMPLETE``. This indicates that your template was validated and all the data required to launch a stack has been stored. See example command below: - -.. code:: - - openstack stack create -t --initialize --parameter reservation_id=dummy - - -Create Reservation with Stack_ID -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Finally, for a stack to launch when your reservation begins, we need to let Blazar know which stack to notify Heat to update. This is done via the command line by specifying ``orchestration`` as an ``on_start`` action with a stack_id (e.g. ``on_start=orchestration:``) under the ``--reservation`` flag. Under the hood, Blazar will update your initialized Heat stack with the reservation_id assigned to the lease. See example below: - -.. code:: - - openstack reservation lease create --start-date "" --end-date "" \ - --reservation min=,max=,resource_type=physical:host,on_start=orchestration: \ - +If you are not redirected automatically, please visit :doc:`complex/index`. \ No newline at end of file diff --git a/source/technical/complex/advanced_topics.rst b/source/technical/complex/advanced_topics.rst new file mode 100644 index 0000000..60cadee --- /dev/null +++ b/source/technical/complex/advanced_topics.rst @@ -0,0 +1,187 @@ +.. _complex-advanced: + +Advanced Topics +=============== + +.. _all-to-all-info-exchange: + +All-to-All Information Exchange +------------------------------- + +The previous examples have all used ``user_data`` scripts to provide instances with contextualization information. While it is easy to use, this contextualization method has a major drawback: because it is given to the instance as part of its launch request, it cannot use any context information that is not yet known at this time. In practice, this means that in a client-server deployment, only one of these pattern will be possible: + +- The server has to be deployed first, and once it is deployed, the clients can be launched and contextualized with information from the server. The server won't know about the clients unless there is a mechanism (not managed by *Heat*) for the client to contact the server. +- The clients have to be deployed first, and once they are deployed, the server can be launched and contextualized with information from the clients. The clients won't know about the server unless there is a mechanism (not managed by *Heat*) for the server to contact the clients. + +This limitation was already apparent in our `NFS share `_ appliance: this is why the server instance exports the file system to all bare metal instances on Chameleon, because it doesn't know which specific IP addresses are allocated to the clients. + +This limitation is even more important if the deployment is not hierarchical, i.e. all instances need to know about all others. For example, a cluster with IP and hostnames populated in ``/etc/hosts`` required each instance to be known by every other instance. + +This section presents a more advanced form of contextualization that can perform this kind of information exchange. +This is implemented by *Heat* agents running inside instances and communicating with the *Heat* service to send and receive information. +This means you will need to use an image bundling these agents. +Currently, all Chameleon-supported images (CC) are supporting this mode of contextualization. +If you build your own images using the `CC-CentOS7 `_ builder, `CC-CentOS `_ builder or `CC-Ubuntu `_ builder, you will automatically have these agents installed. This contextualization is performed with several Heat resources: + +- ``OS::Heat::SoftwareConfig``: This resource describes code to run on an instance. It can be configured with inputs and provide outputs. +- ``OS::Heat::SoftwareDeployment``: This resource applies a SoftwareConfig to a specific instance. +- ``OS::Heat::SoftwareDeploymentGroup``: This resource applies a SoftwareConfig to a specific group of instances. + + +The template below illustrates how it works. It launches a group of instances that will automatically populates their ``/etc/hosts`` file with IP and hostnames from other instances in the deployment. + +.. code:: + + heat_template_version: 2015-10-15 + + description: > + This template demonstrates how to exchange hostnames and IP addresses to populate /etc/hosts. + + parameters: + flavor: + type: string + default: baremetal + constraints: + - custom_constraint: nova.flavor + image: + type: string + default: CC-CentOS8 + constraints: + - custom_constraint: glance.image + key_name: + type: string + default: default + constraints: + - custom_constraint: nova.keypair + instance_count: + type: number + default: 2 + reservation_id: + type: string + description: ID of the Blazar reservation to use for launching instances. + constraints: + - custom_constraint: blazar.reservation + + resources: + export_hosts: + type: OS::Heat::SoftwareConfig + properties: + outputs: + - name: hosts + group: script + config: | + #!/bin/sh + (echo -n $(facter ipaddress); echo -n ' '; echo $(facter hostname)) > ${heat_outputs_path}.hosts + + export_hosts_sdg: + type: OS::Heat::SoftwareDeploymentGroup + properties: + config: { get_resource: export_hosts } + servers: { get_attr: [server_group, refs_map] } + signal_transport: HEAT_SIGNAL + + populate_hosts: + type: OS::Heat::SoftwareConfig + properties: + inputs: + - name: hosts + group: script + config: | + #!/usr/bin/env python + import ast + import os + import string + import subprocess + hosts = os.getenv('hosts') + if hosts is not None: + hosts = ast.literal_eval(string.replace(hosts, '\n', '\\n')) + with open('/etc/hosts', 'a') as hosts_file: + for ip_host in hosts.values(): + hosts_file.write(ip_host.rstrip() + '\n') + + populate_hosts_sdg: + type: OS::Heat::SoftwareDeploymentGroup + depends_on: export_hosts_sdg + properties: + config: { get_resource: populate_hosts } + servers: { get_attr: [server_group, refs_map] } + signal_transport: HEAT_SIGNAL + input_values: + hosts: { get_attr: [ export_hosts_sdg, hosts ] } + + server_group: + type: OS::Heat::ResourceGroup + properties: + count: { get_param: instance_count } + resource_def: + type: OS::Nova::Server + properties: + flavor: { get_param: flavor } + image: { get_param: image } + key_name: { get_param: key_name } + networks: + - network: sharednet1 + scheduler_hints: { reservation: { get_param: reservation_id } } + user_data_format: SOFTWARE_CONFIG + software_config_transport: POLL_SERVER_HEAT + + outputs: + deployment_results: + value: { get_attr: [export_hosts_sdg, hosts] } + +There are two ``SoftwareConfig`` resources: + +- The first ``SoftwareConfig``, ``export_hosts``, uses the ``facter`` tool to extract IP address and hostname into a single line (in the format expected for ``/etc/hosts``) and writes it to a special path (``${heat_outputs_path}.hosts``). This prompts Heat to assign the content of this file to the output with the name hosts. +- The second ``SoftwareConfig``, ``populate_hosts``, takes as input a variable named hosts, and applies a script that reads the variable from the environment, parses it with ``ast.literal_eval`` (as it is formatted as a Python dict), and writes each value of the dictionary to ``/etc/hosts``. + +The ``SoftwareDeploymentGroup`` resources ``export_hosts_sdg`` and ``populate_hosts_sdg`` apply each ``SoftwareConfig`` to the instance ``ResourceGroup`` with the correct configuration. + +Finally, the instance ``ResourceGroup`` is configured so that each instance uses the following contextualization method instead of a ``user_data`` script: + +.. code:: + + user_data_format: SOFTWARE_CONFIG + software_config_transport: POLL_SERVER_HEAT + +You can follow the same template pattern to configure your own deployment requiring all-to-all information exchange. + +.. _automated-deployemnt: + +Automated Deployment +-------------------- + +On Chameleon you can configure a Heat Stack to launch as soon as your lease begins. Whether your experiments require a large cluster or a single node, automated deployment saves you time configuring your environment and even allows you to run your entire experiment automatically when the necessary resources become available. + +At present, you will need to use our customized versions of the Heat and Blazar CLI tools to implement this feature. + +Install Custom CLI +~~~~~~~~~~~~~~~~~~ + +You can install Chameleon's ``python-heatclient`` and ``python-blazarclient`` packages via ``pip`` by running the following commands: + +.. code:: + + pip install git+https://github.com/ChameleonCloud/python-heatclient.git + pip install git+https://github.com/ChameleonCloud/python-blazarclient.git + + +Initialize Stack +~~~~~~~~~~~~~~~~ + +Next you will need to configure a Heat stack with the ``--initialize`` flag on the CLI and a dummy ``reservation_id`` parameter. The ``dummy`` id can be anything (even an empty string) so long as the ``reservation_id`` parameter is specified so that Blazar can overwrite it once your advanced reservation is scheduled and the stack is ready to launch. Once your stack is initialized, the status should read ``INIT_COMPLETE``. This indicates that your template was validated and all the data required to launch a stack has been stored. See example command below: + +.. code:: + + openstack stack create -t --initialize --parameter reservation_id=dummy + + +Create Reservation with Stack_ID +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Finally, for a stack to launch when your reservation begins, we need to let Blazar know which stack to notify Heat to update. This is done via the command line by specifying ``orchestration`` as an ``on_start`` action with a stack_id (e.g. ``on_start=orchestration:``) under the ``--reservation`` flag. Under the hood, Blazar will update your initialized Heat stack with the reservation_id assigned to the lease. See example below: + +.. code:: + + openstack reservation lease create --start-date "" --end-date "" \ + --reservation min=,max=,resource_type=physical:host,on_start=orchestration: \ + \ No newline at end of file diff --git a/source/technical/complex/catalog.rst b/source/technical/complex/catalog.rst new file mode 100644 index 0000000..f5b5450 --- /dev/null +++ b/source/technical/complex/catalog.rst @@ -0,0 +1,20 @@ +.. _complex-catalog: + +Complex Appliances in the Catalog +================================= + +The `Chameleon Appliance Catalog `_ has several *Complex Appliances* for popular technologies that people want to deploy such as OpenStack or MPI or even more advanced deployments such as efficient SR-IOV enabled MPI in KVM virtual machines. We also provide common building blocks for cluster architectures, such as an NFS share. *Complex Appliances* are identified by a badge in their top-right corner representing a group of machines, as in the screenshot below: + +.. figure:: ../complex/nfsappliance.png + :alt: A Complex Appliance with a badge in the upper right + + A Complex Appliance with a badge in the upper right + +To view the details of a *Complex Appliance*, simply click on it. + +.. figure:: ../complex/nfsappliancedetail.png + :alt: A Complex Appliance page + + A Complex Appliance page + +.. tip:: You may download the *Template* file or copy the *Template* file URL to clipboard by clicking the *Get Template* button. The *Template* file or its URL is required when launching a *Complex Appliance*. \ No newline at end of file diff --git a/source/technical/complex/cli_management.rst b/source/technical/complex/cli_management.rst new file mode 100644 index 0000000..93c2cf3 --- /dev/null +++ b/source/technical/complex/cli_management.rst @@ -0,0 +1,135 @@ +.. _complex-cli: + +Managing Complex Appliances using the CLI +========================================= + +.. tip:: Reading :ref:`cli` is highly recommended before continuing on the following sections. + +In addition to :ref:`cli-installing`, you will need to install the ``python-heatclient`` package using the command: + +.. code-block:: bash + + pip install python-heatclient + +Then, set up your environment for OpenStack command line usage, as described in :ref:`cli-rc-script`. You can get a list of your *Complex Appliances* in your project using the command: + +.. code-block:: bash + + openstack stack list + +The output should look like the following: + +.. code:: + + +--------------------------------------+---------------+-------------------+----------------------+----------------------+ + | ID | Stack Name | Stack Status | Creation Time | Updated Time | + +--------------------------------------+---------------+-------------------+----------------------+----------------------+ + | e5df33b5-5282-4935-8097-973328ca71e5 | my_stack | CREATE_COMPLETE | 2018-01-23T22:45:12Z | None | + +--------------------------------------+---------------+-------------------+----------------------+----------------------+ + +Launching a Complex Appliance +----------------------------- + +To launch a *Complex Appliance* using *Template*, run the command on your local machine: + +.. code-block:: bash + + openstack stack create --template --parameter = + +Provide the path to and the name of the *Template* file in your local file system via the ``template`` switch. The ```` is the name of the *Complex Appliance*. In addition, you may provide the parameters required in the *Template* file with their values by ``parameter`` switch. For example, the `NFS Server Template `_ lists the following ``parameters`` section: + +.. code:: + + parameters: + nfs_client_count: + type: number + description: Number of NFS client instances + default: 1 + constraints: + - range: { min: 1 } + description: There must be at least one client. + key_name: + type: string + description: Name of a KeyPair to enable SSH access to the instance + default: default + constraints: + - custom_constraint: nova.keypair + reservation_id: + type: string + description: ID of the Blazar reservation to use for launching instances. + constraints: + - custom_constraint: blazar.reservation + +Therefore, in order to use this *Template*, you must provide values for ``nfs_client_count``, ``key_name`` and ``reservation_id``. + +Monitoring a Complex Appliance +------------------------------ + +You can get details about your *Complex Appliance*, such as *Outputs*, *Events* and *Resources*, via the CLI. You will need the *UUID* of the *Complex Appliance*. + +.. tip:: To get the *UUID* of your *Complex Appliance*, use the *Stacks* page on the GUI or retrieve it by ``openstack stack list`` command. + +- To view the *Outputs*, run: + + .. code-block:: bash + + openstack stack output list + + For example, the list of the outputs for the `NFS Share `_ stack is: + + .. code:: + + +------------+-----------------------------------------+ + | output_key | description | + +------------+-----------------------------------------+ + | client_ips | Private IP addresses of the NFS clients | + | server_ip | Public IP address of the NFS server | + +------------+-----------------------------------------+ + + You can get more details about the outputs by using the following command: + + .. code-block:: bash + + openstack stack output show --all + +- To view the *Events*, run: + + .. code-block:: bash + + openstack stack event list + +- To view the *Resources*, run: + + .. code-block:: bash + + openstack stack resource list + + Your output may look like this: + + .. code:: + + +---------------------------+--------------------------------------+---------------------------------+-----------------+----------------------+ + | resource_name | physical_resource_id | resource_type | resource_status | updated_time | + +---------------------------+--------------------------------------+---------------------------------+-----------------+----------------------+ + | nfs_server_ip_association | | OS::Neutron::FloatingIPAssociation | INIT_COMPLETE | 2018-03-19T18:38:05Z | + | nfs_server | 0ab0169c-f762-4d27-8724-b359caa50f1f | OS::Nova::Server | CREATE_FAILED | 2018-03-19T18:38:05Z | + | nfs_server_floating_ip | ecb391f8-4653-43a6-b2c6-bb93a6d89115 | OS::Nova::FloatingIP | CREATE_COMPLETE | 2018-03-19T18:38:05Z | + | nfs_clients | | OS::Heat::ResourceGroup | INIT_COMPLETE | 2018-03-19T18:38:05Z | + +---------------------------+--------------------------------------+---------------------------------+-----------------+----------------------+ + + Then, you may retrieve information about a specific resource using the command: + + .. code-block:: bash + + openstack stack resource show + +Deleting a Complex Appliance +---------------------------- + +Use the following command to delete a stack: + +.. code-block:: bash + + openstack stack delete + +It will remove all the resources attached to the stack. \ No newline at end of file diff --git a/source/technical/complex/gui_management.rst b/source/technical/complex/gui_management.rst new file mode 100644 index 0000000..00f8ec0 --- /dev/null +++ b/source/technical/complex/gui_management.rst @@ -0,0 +1,97 @@ +.. _complex-gui: + +Managing Complex Appliances using the GUI +========================================= + +Before launching a *Complex Appliance*, make sure that you have a reservation for the appropriate node types and a key pair configured. Since most *Complex Appliances* will consist of multiple nodes, make sure you have set the *Minimum Number of Hosts* in your Lease. You will also need a *Template* file or the URL for a *Template* file from the `Appliance Catalog `_. At |CHI@TACC| site or |CHI@UC| site, go to *Project* > *Orchestration* > *Stacks* use the navigation side bar. + +.. figure:: ../complex/stacks.png + :alt: The Stacks page + + The Stacks page + +.. tip:: + You can go to *Stacks* page directly from the `Appliance Catalog `_. + + #. Go to the `Appliance Catalog `_ and identify the appliance you want to launch. Click on it to open its details page. + + #. Click on the "Launch Complex Appliance at ``CHI@TACC``" or "Launch Complex Appliance at ``CHI\@UC``" button depending on where your reservation is created. + + +Launching a Complex Appliance +----------------------------- + +To launch a stack, click the *Launch Stack* button in the upper right of the *Stacks* page. Then follow the steps: + +#. Start setting up a *Template* by choosing a *Template Source* in the dropdown. You may either select the *File* option as *Template Source* and upload the *Template* file, or select the *URL* option and provide the URL of the *Template* file. + + .. figure:: ../complex/selecttemplate.png + :alt: The Select Template step + + The Select Template step + + .. important:: **Do not** change the environment source settings! + +#. Once you have provided a Template, click the *Next* button. Chameleon will validate the Template file and proceed to the *Launch Stack* step. + + .. figure:: ../complex/launchstack.png + :alt: The Launch Stack step + + The Launch Stack step + +#. Choose a name for your stack. Ignore the "Creation Timeout" and "Rollback On Failure" settings. You also need to enter your Chameleon password. Then, you need to select a value for the parameters of the template. Finally, click the *Launch* button. +#. Your stack should be in status "Create In Progress" for several minutes while it first launches the server instance, followed by the client instances. It will then move to the status "Create Complete". + +.. figure:: ../complex/createinprogress.png + :alt: A Complex Appliance with the Create in Progress status + + A Complex Appliance with the Create in Progress status + +Monitoring a Complex Appliance +------------------------------ + +To monitor and get more details about your *Complex Appliance*, click on it in the *Stacks* page. + +- The *Topology* tab displays a topology graph of the stack. The rack of machine represents the client instance group. The server's floating IP (the public IP assigned to a resource) is represented by an IP in a circle; while an IP in a circle is also used to represent the association of the IP with the server instance (not the greatest idea to use the same symbol for both the IP and the association -- we agree but can't do much about it at the moment). Blow off some steam by dragging the visualization across the screen, it can be rather fun! + + .. note:: Blinking nodes indicates that they are still provisioning. + + .. figure:: ../complex/topology.png + :alt: The Topology tab + + The Topology tab + +- The *Overview* tab displays various parameters, including the *ID* of the stack and *Outputs* such as IP addresses assigned to each node. If you have a floating IP associated to the server, you can now ``ssh`` to the server using the floating IP just as you do with regular instances. The client may not have a floating IP attached to it, but you can connect to it via the server node with the client's private IP. + + .. tip:: To talk to the client without an associated floating IP, connect to the server with ``ssh -A`` to enable the SSH agent forwarding after loading your key to your SSH agent with ``ssh-add ``. + + .. figure:: ../complex/overview.png + :alt: The Overview tab + + The Overview tab + +- Under the *Resources* tab you will see the resources of the stack (the server, clients, server's public/floating IP, and its the association) and information about them. + + .. figure:: ../complex/resources.png + :alt: The Resources tab + + The Resources tab + +- In the *Events* tab you will see information about the history of the deployment so far. + + .. figure:: ../complex/events.png + :alt: The Events tab + + The Events tab + +- In *Template* tab, you will see the template that was used to deploy this stack. + + .. figure:: ../complex/template.png + :alt: The Template tab + + The Template tab + +Deleting a Complex Appliance +---------------------------- + +To delete a *Complex Appliance*, select it in the *Stacks* page and click the *Delete Stacks* button. This will delete all resources of the stack, such as nodes and floating IP addresses. \ No newline at end of file diff --git a/source/technical/complex/heat_templates.rst b/source/technical/complex/heat_templates.rst new file mode 100644 index 0000000..8df3fd1 --- /dev/null +++ b/source/technical/complex/heat_templates.rst @@ -0,0 +1,376 @@ +.. _heat-templates: + +Heat Orchestration Templates +============================ + +A *Heat Orchestration Template* is a YAML file that specifies how resources are used and configured in a *Complex Appliance*. + +A Case Example: NFS Share +------------------------- + +Let's look at the `NFS Share Template `_. The NFS share appliance deploys: + +- An NFS server instance, that exports the directory ``/exports/example`` to any instance running on Chameleon bare metal, +- One or several NFS client instances, which configure ``/etc/fstab`` to mount this NFS share to ``/mnt`` (and can subsequently read from and write to it). + +This template is reproduced further below, and includes inline comments starting with the ``#`` character. There are three main sections: + +- resources +- parameters +- outputs + +The ``resources`` section is the most important part of the template: it defines which OpenStack *Resources* to create and configure. Inside this section you can see four resources defined: + +- ``nfs_server_floating_ip``: creates a *Floating IP* on the ``ext-net`` public network. It is not attached to any instance yet. +- ``nfs_server``: creates the NFS server instance (an instance is defined with the type ``OS::Nova::Server`` in *Heat*). It is a bare metal instance (``flavor: baremetal``) using the ``CC-CentOS7`` image and connected to the private network named ``sharednet1``. We set the key pair to use the value of the parameter defined earlier, using the ``get_param`` function. Similarly, the reservation to use is passed to the scheduler. Finally, a ``user_data`` script is given to the instance, which configures it as an NFS server exporting ``/exports/example`` to Chameleon instances. +- ``nfs_server_ip_association``: associates the floating IP created earlier with the NFS server instance. +- ``nfs_clients``: defines a resource group containing instance configured to be NFS clients and mount the directory exported by the NFS server defined earlier. The IP of the NFS server is gathered using the ``get_attr`` function, and placed into ``user_data`` using the ``str_replace`` function. + +Once a Resource has been specified, you may provide it as a value for another Resource's property using the ``get_resource`` function. + +The ``parameters`` section defines inputs to be used on *Complex Appliance* launch. Parameters all have the same data structure: each one has a name (``key_name`` or ``reservation_id`` in this case), a data type (``number`` or ``string``), a comment field called ``description``, optionally a ``default value``, and a list of ``constraints`` (in this case only one per parameter). Constraints tell *Heat* to match a parameter to a specific type of OpenStack resource. *Complex appliances* on Chameleon require users to customize at least the key pair name and reservation ID, and will generally provide additional parameters to customize other properties of the cluster, such as its size, as in this example. The values of Parameters can be used in the ``resources`` section using the ``get_param`` function. + +The ``outputs`` section defines what values are returned to the user. *Outputs* are declared similarly to *Parameters*: they each have a name, an optional description, and a value. They allow to return information from the stack to the user. You may use the ``get_attr`` function to retrieve a resource's attribute for output. + +Heat Template Customization +--------------------------- + +Customizing an existing template is a good way to start developing your own. We will use a simpler template than the previous example to start with: it is the `Hello World complex appliance `_. + +First, delete the stack you launched, because we will need all three nodes to be free. To do this, go back to the *Project* > *Orchestration* > *Stacks* page, select your stack, and then click on the *Delete Stacks* button. You will be asked to confirm, so click on the *Delete Stacks* button. + + .. figure:: ../complex/deletestacks.png + :alt: Confirm deleting stack dialog + + Confirm deleting stack dialog + +The template for the `Hello World complex appliance `_ is reproduced below. It is similar to the NFS share appliance, except that it deploys only a single client. You can see that it has four resources defined: + +- ``nfs_server_floating_ip`` +- ``nfs_server`` +- ``nfs_server_ip_association`` +- ``nfs_client`` + +The ``nfs_client`` instance mounts the NFS directory shared by the ``nfs_server`` instance, just like in our earlier example. + +:: + + # This describes what is deployed by this template. + description: NFS server and client deployed with Heat on Chameleon + + # This defines the minimum Heat version required by this template. + heat_template_version: 2015-10-15 + + # The resources section defines what OpenStack resources are to be deployed and + # how they should be configured. + resources: + nfs_server_floating_ip: + type: OS::Nova::FloatingIP + properties: + pool: ext-net + + nfs_server: + type: OS::Nova::Server + properties: + flavor: baremetal + image: CC-CentOS7 + key_name: { get_param: key_name } + networks: + - network: sharednet1 + scheduler_hints: { reservation: { get_param: reservation_id } } + user_data: | + #!/bin/bash + yum install -y nfs-utils + mkdir -p /exports/example + chown -R cc:cc /exports + echo '/exports/example 10.140.80.0/22(rw,async) 10.40.0.0/23(rw,async)' >> /etc/exports + systemctl enable rpcbind && systemctl start rpcbind + systemctl enable nfs-server && systemctl start nfs-server + + nfs_server_ip_association: + type: OS::Neutron::FloatingIPAssociation + properties: + floatingip_id: {get_resource: nfs_server_floating_ip} + port_id: {get_attr: [nfs_server, addresses, sharednet1, 0, port]} + + nfs_client: + type: OS::Nova::Server + properties: + flavor: baremetal + image: CC-CentOS7 + key_name: { get_param: key_name } + networks: + - network: sharednet1 + scheduler_hints: { reservation: { get_param: reservation_id } } + user_data: + str_replace: + template: | + #!/bin/bash + yum install -y nfs-utils + echo "$nfs_server_ip:/exports/example /mnt/ nfs" > /etc/fstab + mount -a + params: + $nfs_server_ip: { get_attr: [nfs_server, first_address] } + + # The parameters section gathers configuration from the user. + parameters: + key_name: + type: string + description: Name of a KeyPair to enable SSH access to the instance + default: default + constraints: + - custom_constraint: nova.keypair + reservation_id: + type: string + description: ID of the Blazar reservation to use for launching instances. + constraints: + - custom_constraint: blazar.reservation + +Download `this template `_ to your local machine, and open it in your favorite text editor. + +We will customize the template to add a second NFS client by creating a new resource called ``another_nfs_client``. Add the following text to your template inside the resources section. Make sure to respect the level of indentation, which is important in YAML. + +:: + + another_nfs_client: + type: OS::Nova::Server + properties: + flavor: baremetal + image: CC-CentOS7 + key_name: { get_param: key_name } + networks: + - network: sharednet1 + scheduler_hints: { reservation: { get_param: reservation_id } } + user_data: + str_replace: + template: | + #!/bin/bash + yum install -y nfs-utils + echo "$nfs_server_ip:/exports/example /mnt/ nfs" > /etc/fstab + mount -a + params: + $nfs_server_ip: { get_attr: [nfs_server, first_address] } + +Now, launch a new stack with this template. Since the customized template is only on your computer and cannot be addressed by a URL, use the *Direct Input* method instead and copy/paste the content of the customized template. The resulting topology view is shown below: as you can see, the two client instances are shown separately since each one is defined as a separate resource in the template. + + .. figure:: ../complex/topologycustomhelloworld.png + :alt: Topology of the customized Hello World Appliance + + Topology of the customized Hello World Appliance + +You may have realized already that while adding just one additional client instance was easy, launching more of them would require to copy / paste blocks of YAML many times while ensuring that the total count is correct. This would be easy to get wrong, especially when dealing with tens or hundreds of instances. + +So instead, we leverage another construct from *Heat*: resource groups. Resource groups allow to define one kind of resource and request it to be created any number of times. + +Remove the ``nfs_client`` and ``another_client`` resources from your customized template, and replace them with the following: + +:: + + nfs_clients: + type: OS::Heat::ResourceGroup + properties: + count: 2 + resource_def: + type: OS::Nova::Server + properties: + flavor: baremetal + image: CC-CentOS7 + key_name: { get_param: key_name } + networks: + - network: sharednet1 + scheduler_hints: { reservation: { get_param: reservation_id } } + user_data: + str_replace: + template: | + #!/bin/bash + yum install -y nfs-utils + echo "$nfs_server_ip:/exports/example /mnt/ nfs" > /etc/fstab + mount -a + params: + $nfs_server_ip: { get_attr: [nfs_server, first_address] } + +A resource group is configured with a properties field, containing the definition of the resource to launch (``resource_def``) and the number of resources to launch (``count``). Once launched, you will notice that the topology view groups all client instances under a single *Resource Group* icon. We use the same ``resource_def`` than when defining separate instances earlier. + +Another way we can customize this template is by adding outputs to the template. Outputs allow a *Heat* template to return data to the user. This can be useful to return values like IP addresses or credentials that the user must know to use the system. + +We will create an output returning the floating IP address used by the NFS server. We define an outputs section, and one output with the name ``server_ip`` and a description. The value of the output is gathered using the ``get_attr`` function which obtains the IP address of the server instance. + +:: + + outputs: + server_ip: + description: Public IP address of the NFS server + value: { get_attr: [nfs_server_floating_ip, ip] } + +You can get outputs in the *Overview* tab of the *Stack Details* page. If you want to use the command line, install ``python-heatclient`` and use the ``heat output-list`` and ``heat output-show`` commands, or get a full list in the information returned by ``heat stack-show``. + +Multiple outputs can be defined in the outputs section. Each of them needs to have a unique name. For example, we can add another output to list the private IPs assigned to client instances: + +:: + + client_ips: + description: Private IP addresses of the NFS clients + value: { get_attr: [nfs_clients, first_address] } + +The image below shows the resulting outputs as viewed from the GUI. Of course IP addresses will be specific to each deployment. + + .. figure:: ../complex/helloworldoutputs.png + :alt: The Outputs of customized Hello World appliance + + The Outputs of customized Hello World appliance + +Finally, we can add a new parameter to replace the hard-coded number of client instances by a value passed to the template. Add the following text to the parameters section: + +:: + + nfs_client_count: + type: number + description: Number of NFS client instances + default: 1 + constraints: + - range: { min: 1 } + description: There must be at least one client. + +Inside the resource group definition, change ``count: 2`` to ``count: { get_param: nfs_client_count }`` to retrieve and use the parameter we just defined. When you launch this template, you will see that an additional parameter allows you to define the number of client instances, like in the NFS share appliance. + +At this stage, we have fully recreated the *NFS share* appliance starting from the *Hello World* one! The next section will explain how to write a new template from scratch. + +Writing a New Template +---------------------- + +You may want to write a whole new template, rather than customizing an existing one. Each template should follow the same layout and be composed of the following sections: + +- Heat template version +- Description +- Resources +- Parameters +- Outputs + +Heat template version +~~~~~~~~~~~~~~~~~~~~~ + +Each Heat template has to include the ``heat_template_version`` key with a valid version of `HOT (Heat Orchestration Template) `_. Chameleon bare metal supports any HOT version up to **2015-10-15**, which corresponds to OpenStack Liberty. +The `Heat documentation `_ lists all available versions and their features. We recommended that you always use the latest Chameleon supported version to have access to all supported features: + +``heat_template_version: 2015-10-15`` + +Description +~~~~~~~~~~~ + +While not mandatory, it is good practice to describe what is deployed and configured by your template. It can be on a single line: + +:: + + description: This describes what this Heat template deploys on Chameleon. + +If a longer description is needed, you can provide multi-line text in YAML, for example: + +:: + + description: > + This describes what this Heat + template deploys on Chameleon. + +Resources +~~~~~~~~~ + +The resources section is required and must contain at least one resource definition. A `complete list of resources types known to Heat `_ is +available. + +However, only a subset of them are supported by Chameleon, and some are limited to administrative use. We recommend that you only use: + +- OS::Glance::Image +- OS::Heat::ResourceGroup +- OS::Heat::SoftwareConfig +- OS::Heat::SoftwareDeployment +- OS::Heat::SoftwareDeploymentGroup +- OS::Neutron::FloatingIP +- OS::Neutron::FloatingIPAssociation +- OS::Neutron::Port (advanced users only) +- OS::Nova::Keypair +- OS::Nova::Server + +If you know of another resource that you would like to use and think it should be supported by the OpenStack services on Chameleon bare metal, let us know via our |Help Desk|. + +Parameters +~~~~~~~~~~ + +Parameters allow users to customize the template with necessary or optional values. +For example, they can customize which Chameleon appliance they want to deploy, or which key pair to install. +Default values can be provided with the ``default`` key, as well as constraints to ensure that only valid OpenStack resources can be selected. +For example, ``custom_constraint: glance.image`` restricts the image selection to an available OpenStack image, while providing a pre-filled selection box in the GUI. +`More details about constraints `_ are available in the *Heat* documentation. + +Outputs +~~~~~~~ + +Outputs allow template to give information from the deployment to users. This can include usernames, passwords, IP addresses, hostnames, paths, etc. The outputs declaration is using the following format: + +:: + + outputs: + first_output_name: + description: Description of the first output + value: first_output_value + second_output_name: + description: Description of the second output + value: second_output_value + +Generally values will be calls to ``get_attr``, ``get_param``, or some other function to get information from parameters or resources deployed by the +template and return them in the proper format to the user. + + +Reserved Networks and Floating IPs +---------------------------------- + +Chameleon's reservation service allows users to reserve VLAN segments and floating ips. In order to make use of these +reserved resources in a (HOT) template, follow the guidelines below. For more information on VLAN and floating ip reservations, +see documentaiton on :ref:`reservation-cli-vlan` and :ref:`reservation-cli-fip` + +When you reserve a VLAN segment via blazar, it will automatically create a network for you. However, this network +is not usable in your template unless a subnet and router have been associated with the network. Once this is done, you can simply +add the network name as the network parameter for your server as you would ``sharednet1``. The below cli commands +provides an example of how to complete the configuration for your reserved network. + +:: + + openstack subnet create --subnet-range 192.168.100.0/24 \ + --allocation-pool start=192.168.100.100,end=192.168.100.108 \ + --dns-nameserver 8.8.8.8 --dhcp \ + --network \ + my_subnet_name + openstack router create my_router_name + openstack router add subnet my_router_name my_subnet_name + openstack router set --external-gateway public my_router_name + +For reserved floating ips, you need to associate the floating ip with a server using the ``OS::Neutron::FloatingIPAssociation`` object type. +Many of our older complex appliance templates use the ``OS::Nova::FloatingIPAssociation`` object, but this has since been deprecated. See example below +for proper usage: + +:: + + my_server_ip_association: + type: OS::Neutron::FloatingIPAssociation + properties: + floatingip_id: + port_id: {get_attr: [my_server, addresses, , 0, port]} + + +If you are having trouble finding the ``uuid`` of the floating ip address then the below command will help you. + +:: + + openstack floating ip list -c ID -c "Floating IP Address" -c Tags --long + +The output should look like the sample output below with the `uuid` listed under the `ID` column. You can check your lease in +the reservation section of the GUI to find the `reservation id` associated with the floating ip in the `Tags` section of the output. + +:: + + +--------------------------------------+---------------------+------------------------------------------------------------------+ + | ID | Floating IP Address | Tags | + +--------------------------------------+---------------------+------------------------------------------------------------------+ + | 0fe31fad-60ac-462f-bb6c-4d40c1506621 | 192.5.87.206 | [u'reservation:d90ad917-300a-4cf7-a836-083534244f56', u'blazar'] | + | 92a347a9-31a5-43c1-80e2-9cdb38ebf66f | 192.5.87.224 | [u'reservation:5f470c97-0166-4934-a813-509b743e2d62', u'blazar'] | + | c8480d67-533d-4f55-a197-8271da6d9344 | 192.5.87.71 | [] | + +--------------------------------------+---------------------+------------------------------------------------------------------+ \ No newline at end of file diff --git a/source/technical/complex/index.rst b/source/technical/complex/index.rst new file mode 100644 index 0000000..aac53cc --- /dev/null +++ b/source/technical/complex/index.rst @@ -0,0 +1,27 @@ +.. _complex: + +================== +Complex Appliances +================== + +Deploying an MPI cluster, an OpenStack installation, or any other type of cluster in which nodes can take on multiple roles can be complex: you have to provision potentially hundreds of nodes, configure them to take on various roles, and make them share information that is generated or assigned only at deployment time, such as hostnames, IP addresses, or security keys. When you want to run a different experiment later you have to redo all this work. When you want to reproduce the experiment, or allow somebody else to reproduce it, you have to take very precise notes and pay great attention to their execution. + +To help solve this problem and facilitate reproducibility and sharing, the Chameleon team configured a tool that allows you to deploy complex clusters with "one click". This tool requires not just a simple *image* (i.e., appliance) but also a document, called a *template*, that contains the information needed to orchestrate the deployment and configuration of such clusters. We call this *image + template* combination **Complex Appliances** because it consists of more than just the image (i.e., appliance). + +In a nutshell, *Complex Appliances* allow you to specify not only what image you want to deploy but also on how many nodes you want to deploy that image, what roles the deployed instances should boot into (such as e.g., head node and worker node in a cluster), what information from a specific instance should be passed to another instance in that *Complex Appliance*, and what scripts should be executed on boot so that this information is properly used for configuring the "one click" cluster. + +This guide will tell you all you need to know in order to use and configure *Complex Appliances* on Chameleon. + +.. hint:: + Since *Complex Appliances* in Chameleon are currently implemented using the `OpenStack Heat `_ orchestration service, we will be using OpenStack terminology and features to work with them. The templates described above are YAML files using the `Heat Orchestration Template (HOT) `_ format (Heat also supports the AWS CloudFormation template format, but this is not covered here). A deployed complex appliance is referred to as a "stack" – just as a deployed single appliance is typically referred to as an "instance". + +.. toctree:: + :maxdepth: 2 + :caption: Complex Appliances Topics + + catalog + gui_management + cli_management + heat_templates + sharing + advanced_topics \ No newline at end of file diff --git a/source/technical/complex/sharing.rst b/source/technical/complex/sharing.rst new file mode 100644 index 0000000..6fbd332 --- /dev/null +++ b/source/technical/complex/sharing.rst @@ -0,0 +1,13 @@ +.. _complex-sharing: + +Sharing Complex Appliances +========================== + +If you have written your own *Complex Appliance* or substantially customized an existing one, we would love if you shared them with our user community! The process is very similar to regular appliances: log into the Chameleon portal, go to the appliance catalog, and click on the button in the top-right corner: *Add an appliance* (you need to be logged in to see it). + +.. figure:: ../complex/addappliance.png + :alt: The Add an Appliance button + + The Add an Appliance button + +You will be prompted to enter a name, description, and documentation. Instead of providing appliance IDs, copy your template to the dedicated field. Finally, share your contact information and assign a version string to your appliance. Once submitted, your appliance will be reviewed. We will get in touch if a change is needed, but if it's all good we will publish it right away! \ No newline at end of file diff --git a/source/technical/daypass.rst b/source/technical/daypass.rst index e55b91f..7f7d25a 100644 --- a/source/technical/daypass.rst +++ b/source/technical/daypass.rst @@ -1,94 +1,11 @@ -.. _daypass: +:orphan: -======= -Daypass -======= -Normally, only Chameleon users with active allocations are able to launch and -view Trovi artifacts. To allow anyone to launch an artifact, we also provide -daypass. This allows for a non-Chameleon user to have access to Chameleon -for a limited amount of time, using a small, separate allocation. -People interested in reproducing your project will send requests -to the managers of a project. If approved, the requesting user will receive an -email invitation to join a reproducibility project. When they accept, they -can use this project to run your artifact. After the specified time limit, -they will be automatically removed from this project. -Daypass can be enabled on an artifact-by-artifact basis in Trovi. +.. meta:: + :http-equiv=refresh: 0; url=./daypass/index.html -.. _enable-daypass: +This page has moved +=================== -Allowing Reproducibility Requests -================================= - -First, the owner of an artifact must permit reproducibility requests. This can -be revoked at any time, preventing future requests. Additionally, you must also -give your artifact a value for "Hours a user has to reproduce." This value -specifies how long a user will have access to Chameleon for. Consider how -long it takes to run your experiment from start to finish as a lower bound for -this value. The artifact owner must also assign their artifact to a project via -the dropdown selector. As these requests are granting access to Chameleon -resources, this is needed to tie granted requests to a PI. - -These fields can be accessed by navigating to an artifact's detail page, and -then selecting "Share." At the bottom of the share page, you will see the -below forms, which are the project assignment, the enabling of reproducibility -requests, and the hours to reproduce. - -.. figure:: daypass/sharing_reproducibility.png - :alt: An image showing the sharing fields for reproducibility requests - :figclass: screenshot - -After these items are saved, a reproducibility allocation request is -automatically made under your PI's name. Your artifact should now appear with a -"Request Daypass" button below the "Launch" button. The "Launch" button will -not appear for users that are not a member of an active Chameleon project. - -Requesting a Daypass --------------------- - -When another researcher wishes to reproduce your artifact (or when you wish to -do so for another's artifact), selecting "Request Daypass" will display a form -asking for a name, institution, and the reason for reproducing the artifact. -For artifacts under your project/allocation, this gives you oversight and -discretion as to who you grant access to, as ultimately you're responsible for -usage under your allocation (including reproducibility allocations). For -artifacts you wish to explore, it gives you the chance to reach out to the -project owners and explain why you are interested in their work. - -.. figure:: daypass/request_daypass_button.png - :alt: An image showing the "Request Daypass" button - :figclass: screenshot - -After submitting the form, the managers (and PI) of the project associated with -the artifact will receive an email informing them of the request. - -Reviewing a Daypass Request ---------------------------- - -After receiving an email with the daypass request, PIs and project managers -can navigate to the review page by clicking the link in the email. Here, they -will see all of the details submitted with the request. A decision can be made -by choosing "approved" or "rejected" in the selector, and then clicking submit. - -.. figure:: daypass/review_daypass_request.png - :alt: An image showing the "Review Daypass" screen - :figclass: screenshot - -After this decision is made, an email is sent to the requestor with the result. -If the request is approved, an invitation is sent to the user. - -Using an Invitation -------------------- - -If your daypass request is approved, an email will be sent to you with an -invite link. After clicking this link, you will be automatically added to the -project. The email will also mention how long the invitation is for. When the -invite is accepted, you will be taken to the project page for the -reproducibility project. Please note the ID of the project (CHI-XXXXX), which -may be needed to configure an artifact. - -Next, you can navigate back to the original artifact URL you were given. The -"Launch" button can be used now to start running the artifact. - -After the duration for the invite has passed, you will be automatically removed -from the project. +This page has been reorganized. You will be redirected to the new location. +If you are not redirected automatically, please visit :doc:`daypass/index`. \ No newline at end of file diff --git a/source/technical/daypass/index.rst b/source/technical/daypass/index.rst new file mode 100644 index 0000000..b938ffa --- /dev/null +++ b/source/technical/daypass/index.rst @@ -0,0 +1,94 @@ +.. _daypass: + +======= +Daypass +======= +Normally, only Chameleon users with active allocations are able to launch and +view Trovi artifacts. To allow anyone to launch an artifact, we also provide +daypass. This allows for a non-Chameleon user to have access to Chameleon +for a limited amount of time, using a small, separate allocation. +People interested in reproducing your project will send requests +to the managers of a project. If approved, the requesting user will receive an +email invitation to join a reproducibility project. When they accept, they +can use this project to run your artifact. After the specified time limit, +they will be automatically removed from this project. +Daypass can be enabled on an artifact-by-artifact basis in Trovi. + +.. _enable-daypass: + +Allowing Reproducibility Requests +================================= + +First, the owner of an artifact must permit reproducibility requests. This can +be revoked at any time, preventing future requests. Additionally, you must also +give your artifact a value for "Hours a user has to reproduce." This value +specifies how long a user will have access to Chameleon for. Consider how +long it takes to run your experiment from start to finish as a lower bound for +this value. The artifact owner must also assign their artifact to a project via +the dropdown selector. As these requests are granting access to Chameleon +resources, this is needed to tie granted requests to a PI. + +These fields can be accessed by navigating to an artifact's detail page, and +then selecting "Share." At the bottom of the share page, you will see the +below forms, which are the project assignment, the enabling of reproducibility +requests, and the hours to reproduce. + +.. figure:: sharing_reproducibility.png + :alt: An image showing the sharing fields for reproducibility requests + :figclass: screenshot + +After these items are saved, a reproducibility allocation request is +automatically made under your PI's name. Your artifact should now appear with a +"Request Daypass" button below the "Launch" button. The "Launch" button will +not appear for users that are not a member of an active Chameleon project. + +Requesting a Daypass +-------------------- + +When another researcher wishes to reproduce your artifact (or when you wish to +do so for another's artifact), selecting "Request Daypass" will display a form +asking for a name, institution, and the reason for reproducing the artifact. +For artifacts under your project/allocation, this gives you oversight and +discretion as to who you grant access to, as ultimately you're responsible for +usage under your allocation (including reproducibility allocations). For +artifacts you wish to explore, it gives you the chance to reach out to the +project owners and explain why you are interested in their work. + +.. figure:: request_daypass_button.png + :alt: An image showing the "Request Daypass" button + :figclass: screenshot + +After submitting the form, the managers (and PI) of the project associated with +the artifact will receive an email informing them of the request. + +Reviewing a Daypass Request +--------------------------- + +After receiving an email with the daypass request, PIs and project managers +can navigate to the review page by clicking the link in the email. Here, they +will see all of the details submitted with the request. A decision can be made +by choosing "approved" or "rejected" in the selector, and then clicking submit. + +.. figure:: review_daypass_request.png + :alt: An image showing the "Review Daypass" screen + :figclass: screenshot + +After this decision is made, an email is sent to the requestor with the result. +If the request is approved, an invitation is sent to the user. + +Using an Invitation +------------------- + +If your daypass request is approved, an email will be sent to you with an +invite link. After clicking this link, you will be automatically added to the +project. The email will also mention how long the invitation is for. When the +invite is accepted, you will be taken to the project page for the +reproducibility project. Note the ID of the project (CHI-XXXXX), which +may be needed to configure an artifact. + +Next, you can navigate back to the original artifact URL you were given. The +"Launch" button can be used now to start running the artifact. + +After the duration for the invite has passed, you will be automatically removed +from the project. + diff --git a/source/technical/discovery.rst b/source/technical/discovery.rst index 2b4bb9a..7a1cec3 100644 --- a/source/technical/discovery.rst +++ b/source/technical/discovery.rst @@ -1,348 +1,11 @@ -.. _resource-discovery: +:orphan: -=================== -Resource discovery -=================== - -Introduction -============ - -Chameleon supports fine-grained resource discovery for experimentation, which means that you can identify a specific node, view the node's hardware maintenance history and reserve it for repeated use. - -All physical resources available in Chameleon are described in the Chameleon resource registry. The resource registry is based on the `Reference API from the Grid'5000 project `_. Users can consult the registry via the resource discovery web interface or directly via REST APIs. - -.. note:: Some resource discovery features are available through the `Chameleon Portal `_, while others are available **only** through the REST APIs. - -The Hardware Catalog on the Chameleon Portal -============================================ - -You may use the `Hardware `_ page at the `Chameleon Portal `_ to see the different hardware resource types available at each Chameleon site. - -Availability -____________ - -The *CHI\@TACC* and *CHI\@UC* buttons in the *Availability* section of the Resource Browser allow you to open the Lease Calendars at the Chameleon sites. You must login using your Chameleon account to view these lease calendars. - -.. figure:: discovery/availability.png - :alt: Resource availability links to the lease calendars - - Resource availability links to the lease calendars - -Chameleon Resource Browser -__________________________ - -The Chameleon Resource Browser allows you to filter Chameleon resources by node type and view details of each node. - -.. figure:: discovery/resourcebrowser.png - :alt: Chameleon Resource Browser - - The Chameleon Resource Browser - -You may filter for specific node types by selecting the checkboxes that match your filter criteria or by clicking the buttons such as *Compute* and *Infiniband Support*. The numbers printed next to the node types indicate the total number of nodes of that certain type. After you have selected filter criteria, you can click the *View* button to see details of individual nodes that match your filtering criteria. - -.. figure:: discovery/nodedetails.png - :alt: Node details - - Node details - -.. tip:: To get more precise characteristics of the selected node, search the node at `Intel's CPU database `_. - -.. note:: - All the nodes in Chameleon is identified by their *UUIDs*. You will need the *UUID* of a node for making reservations and identifying metrics collected from the node using Gnocchi. In addition, each node also has a *Version UUID*, which is used for retrieving its maintenance history. - -.. attention:: - When we replace faulty hardware on a node, the replacement part typically has the same hardware characteristics. For example, a node with a faulty 250 GB hard drive would be replaced with the same 250 GB hard drive model. However, it may be important for your experimental reproducibility to know about those hardware replacement events, in case it affects your metrics. - -Generating a Reservation Script -_______________________________ - -The `Chameleon Portal `_ does not support a direct reservation from the `Hardware `_ page. However, you may generate a script for reserving the selected nodes by clicking on the *Reserve* button and use the auto-generated script later for the reservation. - -.. figure:: discovery/reserve.png - :alt: Generating a reservation script - - Generating a reservation script - -After the form is submitted by clicking the *Generate Script* button, a new dialog that contains the auto-generated command line will show. - -.. figure:: discovery/reservationscript.png - :alt: An auto-generated reservation script - - An auto-generated reservation script - -For node reservation using auto-generated command, please see :ref:`reservation-cli`. - -Using the REST APIs for Resource Discovery -=================================================== - -The API is designed for users who want to programmatically discover Chameleon resources. It uses a REST architecture on top of the HTTP protocol. As a consequence, any HTTP client can be used to query the API: command-line tools (cURL), browsers, and the numerous HTTP libraries available in your favorite programming language. - -It also implements the concept of "Hypermedia as the Engine of Application State" (HATEOAS), by specifying a set of hyperlinks in all responses returned by the API, which allow a user agent to discover at runtime the set of available resources as well as their semantics and content types, and transition from one resource to another. - -Prerequisites -___________________________ - -Chameleon uses `cURL `_ to interact with the API. The User-Agent `cURL `_ is a command line tool for transferring data with URL syntax, supporting many protocols including HTTP and HTTPS. - -To install `cURL `_, follow the instructions below: - -**OS X** - -cURL is installed by default on OS X. Nothing to do for you! - -**Linux** - -Use your package manager to install cURL. Either (Debian/Ubuntu-based distributions): - -.. code-block:: shell - - $ sudo apt-get install curl +.. meta:: + :http-equiv=refresh: 0; url=./discovery/index.html -or (RedHat-based distributions): - -.. code-block:: shell - - $ sudo yum install curl - -**Windows** - -Download and install the cURL package from `the website `_. - -Your First Requests -___________________________ - -The API entry-point for the resource discovery API is located at https://api.chameleoncloud.org/. Open your Terminal program (or the cURL executable if you're on Windows), and use cURL to fetch the resource located at that URL: - -.. code-block:: shell - - curl -i https://api.chameleoncloud.org/ - -.. tip:: The ``-i`` flag tells cURL to display the HTTP header in addition to the HTTP body. - -Below is what you should see in response: - -.. code-block:: javascript - - HTTP/1.1 200 OK - Server: nginx/1.6.2 - Date: Thu, 19 Apr 2018 14:34:01 GMT - Content-Type: application/vnd.grid5000.item+json; charset=utf-8 - Content-Length: 757 - Connection: keep-alive - Allow: GET - Vary: accept - Last-Modified: Wed, 14 Mar 2018 15:05:58 GMT - ETag: "cc990a75afbc3aed5979c5cad2358b14" - Cache-Control: max-age=60, public, must-revalidate=true, proxy-revalidate=true, s-maxage=60 - X-Info: Use `?pretty=yes` or add the HTTP header `X-Rack-PrettyJSON: yes` if you want pretty output. - X-UA-Compatible: IE=Edge,chrome=1 - X-Runtime: 0.034541 - - {"type":"grid","uid":"chameleoncloud","version":"ee0253a05223dd0f5b88df7f78fb988e67f7b039","release":"3.5.7","timestamp":1524148441,"links":[{"rel":"sites","href":"/sites","type":"application/vnd.grid5000.collection+json"},{"rel":"self","type":"application/vnd.grid5000.item+json","href":"/"},{"rel":"parent","type":"application/vnd.grid5000.item+json","href":"/"},{"rel":"version","type":"application/vnd.grid5000.item+json","href":"/versions/ee0253a05223dd0f5b88df7f78fb988e67f7b039"},{"rel":"versions","type":"application/vnd.grid5000.collection+json","href":"/versions"},{"rel":"users","type":"application/vnd.grid5000.collection+json","href":"/users"},{"rel":"notifications","type":"application/vnd.grid5000.collection+json","href":"/notifications"}]} - -.. note:: The HTTP status of ``200 OK`` indicates that the server is able to process your request and that everything is fine. - -.. tip:: By default the response body is not displayed in a pretty format. You must add the pretty query parameter to the end of the URI if you want the API to display it in a prettier way. ``curl -i https://api.chameleoncloud.org/?pretty`` - -.. attention:: **Do not** use the pretty query parameter in your scripts, since it requires a bit more processing power to generate. - -You may notice that the response contains a number of link elements, which advertise other resources that you can access. For example, let's fetch the ``/sites`` resource. - -.. code-block:: shell - - curl https://api.chameleoncloud.org/sites?pretty - -The response should look like: - -.. code-block:: json - - { - "total": 2, - "offset": 0, - "items": [ - { - "description": "Texas Advanced Computing Center", - "email_contact": "help@chameleoncloud.org", - "latitude": 30.390223, - "location": "Austin, Texas, USA", - "longitude": -97.72563, - "name": "TACC", - "security_contact": "help@chameleoncloud.org", - "sys_admin_contact": "help@chameleoncloud.org", - "type": "site", - "uid": "tacc", - "user_support_contact": "help@chameleoncloud.org", - "web": "https://www.chameleoncloud.org", - "version": "ee0253a05223dd0f5b88df7f78fb988e67f7b039", - "links": [ - { - "rel": "clusters", - "href": "/sites/tacc/clusters", - "type": "application/vnd.grid5000.collection+json" - }, - { - "rel": "self", - "type": "application/vnd.grid5000.item+json", - "href": "/sites/tacc" - }, - { - "rel": "parent", - "type": "application/vnd.grid5000.item+json", - "href": "/" - }, - { - "rel": "version", - "type": "application/vnd.grid5000.item+json", - "href": "/sites/tacc/versions/ee0253a05223dd0f5b88df7f78fb988e67f7b039" - }, - { - "rel": "versions", - "type": "application/vnd.grid5000.collection+json", - "href": "/sites/tacc/versions" - }, - { - "rel": "jobs", - "type": "application/vnd.grid5000.collection+json", - "href": "/sites/tacc/jobs" - }, - { - "rel": "deployments", - "type": "application/vnd.grid5000.collection+json", - "href": "/sites/tacc/deployments" - }, - { - "rel": "vlans", - "type": "application/vnd.grid5000.collection+json", - "href": "/sites/tacc/vlans" - }, - { - "rel": "metrics", - "type": "application/vnd.grid5000.collection+json", - "href": "/sites/tacc/metrics" - }, - { - "rel": "status", - "type": "application/vnd.grid5000.item+json", - "href": "/sites/tacc/status" - } - ] - }, - { - "description": "University of Chicago", - "email_contact": "help@chameleoncloud.org", - "latitude": 41.718002, - "location": "Argonne National Laboratory, Lemont, Illinois, USA", - "longitude": -87.982952, - "name": "UC", - "security_contact": "help@chameleoncloud.org", - "sys_admin_contact": "help@chameleoncloud.org", - "type": "site", - "uid": "uc", - "user_support_contact": "help@chameleoncloud.org", - "web": "https://www.chameleoncloud.org", - "version": "ee0253a05223dd0f5b88df7f78fb988e67f7b039", - "links": [ - { - "rel": "clusters", - "href": "/sites/uc/clusters", - "type": "application/vnd.grid5000.collection+json" - }, - { - "rel": "self", - "type": "application/vnd.grid5000.item+json", - "href": "/sites/uc" - }, - { - "rel": "parent", - "type": "application/vnd.grid5000.item+json", - "href": "/" - }, - { - "rel": "version", - "type": "application/vnd.grid5000.item+json", - "href": "/sites/uc/versions/ee0253a05223dd0f5b88df7f78fb988e67f7b039" - }, - { - "rel": "versions", - "type": "application/vnd.grid5000.collection+json", - "href": "/sites/uc/versions" - }, - { - "rel": "jobs", - "type": "application/vnd.grid5000.collection+json", - "href": "/sites/uc/jobs" - }, - { - "rel": "deployments", - "type": "application/vnd.grid5000.collection+json", - "href": "/sites/uc/deployments" - }, - { - "rel": "vlans", - "type": "application/vnd.grid5000.collection+json", - "href": "/sites/uc/vlans" - }, - { - "rel": "metrics", - "type": "application/vnd.grid5000.collection+json", - "href": "/sites/uc/metrics" - }, - { - "rel": "status", - "type": "application/vnd.grid5000.item+json", - "href": "/sites/uc/status" - } - ] - } - ], - "version": "ee0253a05223dd0f5b88df7f78fb988e67f7b039", - "links": [ - { - "rel": "self", - "type": "application/vnd.grid5000.collection+json", - "href": "/sites" - } - ] - } - -Discover Resources -___________________________ - -It is easy to discover resources using REST APIs when you chase down the ``links`` in the responses. - -As seen in the previous section, when you fetch the API root resource, you can find the link to the collection of sites. If you look at the site description, you will find a list of links to other resources. For example, each site has a link named ``clusters``. When you fetch this link, it returns the list of clusters on that site. - -For example, to get clusters at *TACC*: - -.. code-block:: shell - - curl https://api.chameleoncloud.org/sites/tacc/clusters/?pretty - -Again, you will find ``links`` in each cluster description. There is a link named ``nodes`` for each cluster, which as its name indicates, returns the list of nodes for the specific cluster. - -For example, to get nodes on the *Alamo* cluster at *TACC* site: - -.. code-block:: shell - - curl https://api.chameleoncloud.org/sites/tacc/clusters/alamo/nodes/?pretty - -You should get back a big collection of nodes. Each node is described in great details, so that you can programmatically find the cluster and nodes that are most suitable for your experiments. - -The following command examples allow you to see that some of the nodes on the *Alamo* cluster at *TACC* have a different disk configuration: - -.. code-block:: shell - - curl https://api.chameleoncloud.org/sites/tacc/clusters/alamo/nodes/45f0fc6a-a21b-4461-8414-ebf765143aad?pretty | grep -A 10 storage_devices - curl -s https://api.chameleoncloud.org/sites/tacc/clusters/alamo/nodes/0a5b61b2-dc1c-4bee-86f7-247c9689ea88?pretty | grep -A 10 storage_devices - - -Fetch the Latest Changes -___________________________ - -Let's go back to the site's description. In Chameleon, resources are added, updated, or removed over time. If you want to keep an eye on those changes, you can fetch the latest changes that occurred on a specific site: - -.. code-block:: shell +This page has moved +=================== - curl https://api.chameleoncloud.org/sites/tacc/versions/?pretty +This page has been reorganized. You will be redirected to the new location. -Each version listed in the response represents a change to some resources of the Chameleon testbed. +If you are not redirected automatically, please visit :doc:`discovery/index`. \ No newline at end of file diff --git a/source/technical/discovery/hardware_catalog.rst b/source/technical/discovery/hardware_catalog.rst new file mode 100644 index 0000000..aa3b661 --- /dev/null +++ b/source/technical/discovery/hardware_catalog.rst @@ -0,0 +1,58 @@ +The Hardware Catalog on the Chameleon Portal +============================================ + +You may use the `Hardware `_ page at the `Chameleon Portal `_ to see the different hardware resource types available at each Chameleon site. + +Availability +____________ + +The *CHI\@TACC* and *CHI\@UC* buttons in the *Availability* section of the Resource Browser allow you to open the Lease Calendars at the Chameleon sites. You must login using your Chameleon account to view these lease calendars. + +.. figure:: availability.png + :alt: Resource availability links to the lease calendars + + Resource availability links to the lease calendars + +Chameleon Resource Browser +__________________________ + +The Chameleon Resource Browser allows you to filter Chameleon resources by node type and view details of each node. + +.. figure:: resourcebrowser.png + :alt: Chameleon Resource Browser + + The Chameleon Resource Browser + +You may filter for specific node types by selecting the checkboxes that match your filter criteria or by clicking the buttons such as *Compute* and *Infiniband Support*. The numbers printed next to the node types indicate the total number of nodes of that certain type. After you have selected filter criteria, you can click the *View* button to see details of individual nodes that match your filtering criteria. + +.. figure:: nodedetails.png + :alt: Node details + + Node details + +.. tip:: To get more precise characteristics of the selected node, search the node at `Intel's CPU database `_. + +.. note:: + All the nodes in Chameleon is identified by their *UUIDs*. You will need the *UUID* of a node for making reservations and for power monitoring. In addition, each node also has a *Version UUID*, which is used for retrieving its maintenance history. + +.. attention:: + When we replace faulty hardware on a node, the replacement part typically has the same hardware characteristics. For example, a node with a faulty 250 GB hard drive would be replaced with the same 250 GB hard drive model. However, it may be important for your experimental reproducibility to know about those hardware replacement events, in case it affects your metrics. + +Generating a Reservation Script +_______________________________ + +The `Chameleon Portal `_ does not support a direct reservation from the `Hardware `_ page. However, you may generate a script for reserving the selected nodes by clicking on the *Reserve* button and use the auto-generated script later for the reservation. + +.. figure:: reserve.png + :alt: Generating a reservation script + + Generating a reservation script + +After the form is submitted by clicking the *Generate Script* button, a new dialog that contains the auto-generated command line will show. + +.. figure:: reservationscript.png + :alt: An auto-generated reservation script + + An auto-generated reservation script + +For node reservation using auto-generated command, see :ref:`reservation-cli`. \ No newline at end of file diff --git a/source/technical/discovery/index.rst b/source/technical/discovery/index.rst new file mode 100644 index 0000000..6248f78 --- /dev/null +++ b/source/technical/discovery/index.rst @@ -0,0 +1,20 @@ +.. _resource-discovery: + +================== +Resource Discovery +================== + +Resource discovery on Chameleon allows you to explore and identify the specific hardware resources available for your experiments. You can discover nodes by their hardware characteristics, view detailed specifications, check maintenance history, and reserve specific resources that meet your experimental requirements. + +Chameleon supports fine-grained resource discovery for experimentation, which means that you can identify a specific node, view the node's hardware maintenance history and reserve it for repeated use. + +All physical resources available in Chameleon are described in the Chameleon resource registry. The resource registry is based on the `Reference API from the Grid'5000 project `_. Users can consult the registry via the resource discovery GUI or directly via REST APIs. + +.. note:: Some resource discovery features are available through the `Chameleon Portal `_, while others are available **only** through the REST APIs. + +.. toctree:: + :maxdepth: 1 + :caption: Resource Discovery Topics + + hardware_catalog + rest_api \ No newline at end of file diff --git a/source/technical/discovery/rest_api.rst b/source/technical/discovery/rest_api.rst new file mode 100644 index 0000000..fc767e0 --- /dev/null +++ b/source/technical/discovery/rest_api.rst @@ -0,0 +1,274 @@ +Using the REST APIs for Resource Discovery +=================================================== + +The API is designed for users who want to programmatically discover Chameleon resources. It uses a REST architecture on top of the HTTP protocol. As a consequence, any HTTP client can be used to query the API: command-line tools (cURL), browsers, and the numerous HTTP libraries available in your favorite programming language. + +It also implements the concept of "Hypermedia as the Engine of Application State" (HATEOAS), by specifying a set of hyperlinks in all responses returned by the API, which allow a user agent to discover at runtime the set of available resources as well as their semantics and content types, and transition from one resource to another. + +Prerequisites +___________________________ + +Chameleon uses `cURL `_ to interact with the API. The User-Agent `cURL `_ is a command line tool for transferring data with URL syntax, supporting many protocols including HTTP and HTTPS. + +To install `cURL `_, follow the instructions below: + +**OS X** + +cURL is installed by default on OS X. Nothing to do for you! + +**Linux** + +Use your package manager to install cURL. Either (Debian/Ubuntu-based distributions): + +.. code-block:: shell + + $ sudo apt-get install curl + +or (RedHat-based distributions): + +.. code-block:: shell + + $ sudo yum install curl + +**Windows** + +Download and install the cURL package from `the website `_. + +Your First Requests +___________________________ + +The API entry-point for the resource discovery API is located at https://api.chameleoncloud.org/. Open your Terminal program (or the cURL executable if you're on Windows), and use cURL to fetch the resource located at that URL: + +.. code-block:: shell + + curl -i https://api.chameleoncloud.org/ + +.. tip:: The ``-i`` flag tells cURL to display the HTTP header in addition to the HTTP body. + +Below is what you should see in response: + +.. code-block:: javascript + + HTTP/1.1 200 OK + Server: nginx/1.6.2 + Date: Thu, 19 Apr 2018 14:34:01 GMT + Content-Type: application/vnd.grid5000.item+json; charset=utf-8 + Content-Length: 757 + Connection: keep-alive + Allow: GET + Vary: accept + Last-Modified: Wed, 14 Mar 2018 15:05:58 GMT + ETag: "cc990a75afbc3aed5979c5cad2358b14" + Cache-Control: max-age=60, public, must-revalidate=true, proxy-revalidate=true, s-maxage=60 + X-Info: Use `?pretty=yes` or add the HTTP header `X-Rack-PrettyJSON: yes` if you want pretty output. + X-UA-Compatible: IE=Edge,chrome=1 + X-Runtime: 0.034541 + + {"type":"grid","uid":"chameleoncloud","version":"ee0253a05223dd0f5b88df7f78fb988e67f7b039","release":"3.5.7","timestamp":1524148441,"links":[{"rel":"sites","href":"/sites","type":"application/vnd.grid5000.collection+json"},{"rel":"self","type":"application/vnd.grid5000.item+json","href":"/"},{"rel":"parent","type":"application/vnd.grid5000.item+json","href":"/"},{"rel":"version","type":"application/vnd.grid5000.item+json","href":"/versions/ee0253a05223dd0f5b88df7f78fb988e67f7b039"},{"rel":"versions","type":"application/vnd.grid5000.collection+json","href":"/versions"},{"rel":"users","type":"application/vnd.grid5000.collection+json","href":"/users"},{"rel":"notifications","type":"application/vnd.grid5000.collection+json","href":"/notifications"}]} + +.. note:: The HTTP status of ``200 OK`` indicates that the server is able to process your request and that everything is fine. + +.. tip:: By default the response body is not displayed in a pretty format. You must add the pretty query parameter to the end of the URI if you want the API to display it in a prettier way. ``curl -i https://api.chameleoncloud.org/?pretty`` + +.. attention:: **Do not** use the pretty query parameter in your scripts, since it requires a bit more processing power to generate. + +You may notice that the response contains a number of link elements, which advertise other resources that you can access. For example, let's fetch the ``/sites`` resource. + +.. code-block:: shell + + curl https://api.chameleoncloud.org/sites?pretty + +The response should look like: + +.. code-block:: json + + { + "total": 2, + "offset": 0, + "items": [ + { + "description": "Texas Advanced Computing Center", + "email_contact": "help@chameleoncloud.org", + "latitude": 30.390223, + "location": "Austin, Texas, USA", + "longitude": -97.72563, + "name": "TACC", + "security_contact": "help@chameleoncloud.org", + "sys_admin_contact": "help@chameleoncloud.org", + "type": "site", + "uid": "tacc", + "user_support_contact": "help@chameleoncloud.org", + "web": "https://www.chameleoncloud.org", + "version": "ee0253a05223dd0f5b88df7f78fb988e67f7b039", + "links": [ + { + "rel": "clusters", + "href": "/sites/tacc/clusters", + "type": "application/vnd.grid5000.collection+json" + }, + { + "rel": "self", + "type": "application/vnd.grid5000.item+json", + "href": "/sites/tacc" + }, + { + "rel": "parent", + "type": "application/vnd.grid5000.item+json", + "href": "/" + }, + { + "rel": "version", + "type": "application/vnd.grid5000.item+json", + "href": "/sites/tacc/versions/ee0253a05223dd0f5b88df7f78fb988e67f7b039" + }, + { + "rel": "versions", + "type": "application/vnd.grid5000.collection+json", + "href": "/sites/tacc/versions" + }, + { + "rel": "jobs", + "type": "application/vnd.grid5000.collection+json", + "href": "/sites/tacc/jobs" + }, + { + "rel": "deployments", + "type": "application/vnd.grid5000.collection+json", + "href": "/sites/tacc/deployments" + }, + { + "rel": "vlans", + "type": "application/vnd.grid5000.collection+json", + "href": "/sites/tacc/vlans" + }, + { + "rel": "metrics", + "type": "application/vnd.grid5000.collection+json", + "href": "/sites/tacc/metrics" + }, + { + "rel": "status", + "type": "application/vnd.grid5000.item+json", + "href": "/sites/tacc/status" + } + ] + }, + { + "description": "University of Chicago", + "email_contact": "help@chameleoncloud.org", + "latitude": 41.718002, + "location": "Argonne National Laboratory, Lemont, Illinois, USA", + "longitude": -87.982952, + "name": "UC", + "security_contact": "help@chameleoncloud.org", + "sys_admin_contact": "help@chameleoncloud.org", + "type": "site", + "uid": "uc", + "user_support_contact": "help@chameleoncloud.org", + "web": "https://www.chameleoncloud.org", + "version": "ee0253a05223dd0f5b88df7f78fb988e67f7b039", + "links": [ + { + "rel": "clusters", + "href": "/sites/uc/clusters", + "type": "application/vnd.grid5000.collection+json" + }, + { + "rel": "self", + "type": "application/vnd.grid5000.item+json", + "href": "/sites/uc" + }, + { + "rel": "parent", + "type": "application/vnd.grid5000.item+json", + "href": "/" + }, + { + "rel": "version", + "type": "application/vnd.grid5000.item+json", + "href": "/sites/uc/versions/ee0253a05223dd0f5b88df7f78fb988e67f7b039" + }, + { + "rel": "versions", + "type": "application/vnd.grid5000.collection+json", + "href": "/sites/uc/versions" + }, + { + "rel": "jobs", + "type": "application/vnd.grid5000.collection+json", + "href": "/sites/uc/jobs" + }, + { + "rel": "deployments", + "type": "application/vnd.grid5000.collection+json", + "href": "/sites/uc/deployments" + }, + { + "rel": "vlans", + "type": "application/vnd.grid5000.collection+json", + "href": "/sites/uc/vlans" + }, + { + "rel": "metrics", + "type": "application/vnd.grid5000.collection+json", + "href": "/sites/uc/metrics" + }, + { + "rel": "status", + "type": "application/vnd.grid5000.item+json", + "href": "/sites/uc/status" + } + ] + } + ], + "version": "ee0253a05223dd0f5b88df7f78fb988e67f7b039", + "links": [ + { + "rel": "self", + "type": "application/vnd.grid5000.collection+json", + "href": "/sites" + } + ] + } + +Discover Resources +___________________________ + +It is easy to discover resources using REST APIs when you chase down the ``links`` in the responses. + +As seen in the previous section, when you fetch the API root resource, you can find the link to the collection of sites. If you look at the site description, you will find a list of links to other resources. For example, each site has a link named ``clusters``. When you fetch this link, it returns the list of clusters on that site. + +For example, to get clusters at *TACC*: + +.. code-block:: shell + + curl https://api.chameleoncloud.org/sites/tacc/clusters/?pretty + +Again, you will find ``links`` in each cluster description. There is a link named ``nodes`` for each cluster, which as its name indicates, returns the list of nodes for the specific cluster. + +For example, to get nodes on the *Alamo* cluster at *TACC* site: + +.. code-block:: shell + + curl https://api.chameleoncloud.org/sites/tacc/clusters/alamo/nodes/?pretty + +You should get back a big collection of nodes. Each node is described in great details, so that you can programmatically find the cluster and nodes that are most suitable for your experiments. + +The following command examples allow you to see that some of the nodes on the *Alamo* cluster at *TACC* have a different disk configuration: + +.. code-block:: shell + + curl https://api.chameleoncloud.org/sites/tacc/clusters/alamo/nodes/45f0fc6a-a21b-4461-8414-ebf765143aad?pretty | grep -A 10 storage_devices + curl -s https://api.chameleoncloud.org/sites/tacc/clusters/alamo/nodes/0a5b61b2-dc1c-4bee-86f7-247c9689ea88?pretty | grep -A 10 storage_devices + + +Fetch the Latest Changes +___________________________ + +Let's go back to the site's description. In Chameleon, resources are added, updated, or removed over time. If you want to keep an eye on those changes, you can fetch the latest changes that occurred on a specific site: + +.. code-block:: shell + + curl https://api.chameleoncloud.org/sites/tacc/versions/?pretty + +Each version listed in the response represents a change to some resources of the Chameleon testbed. \ No newline at end of file diff --git a/source/technical/ep.rst b/source/technical/ep.rst deleted file mode 100644 index 7d30a6e..0000000 --- a/source/technical/ep.rst +++ /dev/null @@ -1,435 +0,0 @@ -.. _experiment-precis: - -============================== -Experiment Precis -============================== - -.. warning:: - Precis is no longer supported on the testbed! - - -.. _ep-introduction: - -Introduction -____________ -Chameleon records experiment setup (OpenStack) events that users performed on the testbed, such as creating leases, creating instances, and setting up networks. -Users can request their experiment records from Chameleon using their Chameleon credentials. A report on those experiment records is known as the *Experiment Precis*. -An *Experiment Precis* is bounded to a lease. Chameleon defines an *experiment* as a series of testbed setup (OpenStack) events a user performed under a lease of a project. -Using *Experiment Precis*, users will be able to analyze, understand and even replay their experiments. - -Currently, *Experiment Precis* is provided in JSON format. The following contents are included: - -.. code-block:: json - - { - "experiment_precis_name": "The name of the experiment precis", - "experiment_precis_id": "The id of the experiment precis", - "site": "Which Chameleon site the experiment was performed", - "testbed_version": "The testbed version at the time the experiment started", - "report_time_in_utc": "The datetime when the experiment precis was requested (in UTC)", - "report_time_in_ct": "The datetime when the experiment precis was requested (in CT)", - "events": "A list of events performed", - "event_page": "The page number of the events", - "event_page_size": "The page size of the events", - "hardware": "Hardwares that were used during the experiment", - "hardware_page": "The page number of the hardwares", - "hardware_page_size": "The page size of the harwareds", - "metric": "Metrics saved in Gnocchi", - "metric_page": "The page number of the metrics", - "metric_page_size": "The page size of the metrics" - } - -.. _ep-install: - -Installation -_____________ - -To request your experiment precis from Chameleon, you need to install the *Chameleon Experiment Precis (cep) Client*. -To install ``cepclient`` on your local machine, run the following command: - -.. code-block:: bash - - pip install cepclient - -To test if ``cepclient`` is properly installed, run: - -.. code-block:: bash - - cep --help - -.. _ep-setup: - -Setting up cep client -___________________________________________ - -Before using the *cep client*, you must configure the authentication-related environment variables via ``source`` :ref:`the OpenStack RC script ` -or provide the authentication values as command parameters. If you choose to pass command parameters, use ``cep --help`` and look for ``Authentication Options`` for more information. - -.. note:: - When using *rc script* for setting up *cep client*, please download and use the **v3** file. - -.. _ep-list: - -List experiment precis -_______________________ - -You can use ``list`` command to find all the experiments you have run. - -.. code-block:: bash - - cep list - -The output looks like the following: - -.. code:: - - +---------------------+---------------------+--------------------------------------+--------------------------------------+--------------------------------------+ - | created_at | updated_at | id | name | lease_id | - +---------------------+---------------------+--------------------------------------+--------------------------------------+--------------------------------------+ - | 2018-10-18 09:21:02 | 2018-10-18 09:21:02 | 0fa88391-6b14-465d-a62c-91ad8f6eb920 | 0fa88391-6b14-465d-a62c-91ad8f6eb920 | 972b70aa-33ca-42fc-9d4e-e07b2b9df3c3 | - | 2018-10-18 10:05:50 | 2018-10-18 10:05:50 | 93ffbb79-e732-4046-a49d-b223ff8f1bd5 | 93ffbb79-e732-4046-a49d-b223ff8f1bd5 | 9f91c7ac-212b-4d46-8f88-1e9db341f41a | - +---------------------+---------------------+--------------------------------------+--------------------------------------+--------------------------------------+ - -The *Experiment Precis* will be listed in the reverse order of *creation datetime*, i.e. the latest *Experiment Precis* is listed the first. - -For more information, run: - -.. code-block:: bash - - cep list --help - -.. _ep-rename: - -Rename experiment precis -_________________________ - -Initially, Chameleon sets the name of an *Experiment Precis* the same as its id. However, you can rename it for the convenience of future retrieving. -To rename an *Experiment Precis*, run the following command: - -.. code-block:: bash - - cep rename --name - -.. tip:: - Renaming your experiment precis to a meaningful name will help you 1) mark your *special* experiment; 2) understand what the experiment is about; 3) retrieve your experiment precis. - -For more information, run: - -.. code-block:: bash - - cep rename --help - -.. _ep-print: - -Print experiment precis -________________________ - -Finally, you can retrieve all the details about your experiment by using the ``print`` command. - -.. code-block:: bash - - cep print - -The above command will print the requested experiment precis on your terminal in a compact format. To pretty-print the experiment precis, add ``--pretty`` to the command. -To print the experiment precis to a file, add ``--output `` to the command. - -The following is an example of ``cep print`` output: - -.. code-block:: javascript - - { - "event_page": 0, - "event_page_size": -1, - "events": [ - { - "event_time": "2018-10-18 15:05:50", - "event_type": "lease.create", - "metadata": { - "end_date": "2018-10-19T15:05:00.000000", - "start_date": "2018-10-18T15:06:00.000000" - }, - "resource_id": "9f91c7ac-212b-4d46-8f88-1e9db341f41a", - "service": "blazar" - }, - { - "event_time": "2018-10-18 15:06:05", - "event_type": "lease.event.start_lease", - "metadata": { - "end_date": "2018-10-19T15:05:00.000000", - "start_date": "2018-10-18T15:06:00.000000" - }, - "resource_id": "9f91c7ac-212b-4d46-8f88-1e9db341f41a", - "service": "blazar" - }, - - ... - - { - "event_time": "2018-10-19 15:05:11", - "event_type": "lease.event.end_lease", - "metadata": { - "end_date": "2018-10-19T15:05:00.000000", - "start_date": "2018-10-18T15:06:00.000000" - }, - "resource_id": "9f91c7ac-212b-4d46-8f88-1e9db341f41a", - "service": "blazar" - } - ], - "experiment_precis_id": "93ffbb79-e732-4046-a49d-b223ff8f1bd5", - "experiment_precis_name": "zhenz-test-2", - "hardware": [ - { - "architecture": { - "platform_type": "x86_64", - "smp_size": 2, - "smt_size": 48 - }, - "bios": { - "release_date": "03/09/2015", - "vendor": "Dell Inc.", - "version": 1.2 - }, - "chassis": { - "manufacturer": "Dell Inc.", - "name": "PowerEdge R630", - "serial": "8Q28C42" - }, - "gpu": { - "gpu": false - }, - "links": [ - { - "href": "/sites/uc/clusters/chameleon/nodes/b0525159-5c95-4b71-83f2-b8d6bdd2acd2", - "rel": "self", - "type": "application/vnd.grid5000.item+json" - }, - { - "href": "/sites/uc/clusters/chameleon", - "rel": "parent", - "type": "application/vnd.grid5000.item+json" - }, - { - "href": "/sites/uc/clusters/chameleon/nodes/b0525159-5c95-4b71-83f2-b8d6bdd2acd2/versions/53c90ef0512d5013ee30d431cd62e68bfd34d4ca", - "rel": "version", - "type": "application/vnd.grid5000.item+json" - }, - { - "href": "/sites/uc/clusters/chameleon/nodes/b0525159-5c95-4b71-83f2-b8d6bdd2acd2/versions", - "rel": "versions", - "type": "application/vnd.grid5000.collection+json" - } - ], - "main_memory": { - "humanized_ram_size": "128 GiB", - "ram_size": 134956859392 - }, - "monitoring": { - "wattmeter": false - }, - "network_adapters": [ - { - "bridged": false, - "device": "eno1", - "driver": "bnx2x", - "interface": "Ethernet", - "mac": "44:a8:42:15:c4:dd", - "management": false, - "model": "NetXtreme II BCM57800 1/10 Gigabit Ethernet", - "mounted": true, - "rate": 10000000000, - "vendor": "Broadcom Corporation" - }, - { - "bridged": false, - "device": "eno2", - "driver": "bnx2x", - "interface": "Ethernet", - "mac": "44:a8:42:15:c4:df", - "management": false, - "model": "NetXtreme II BCM57800 1/10 Gigabit Ethernet", - "mounted": false, - "rate": 10000000000, - "vendor": "Broadcom Corporation" - }, - { - "bridged": false, - "device": "eno3", - "driver": "bnx2x", - "interface": "Ethernet", - "mac": "44:a8:42:15:c4:e1", - "management": false, - "model": "NetXtreme II BCM57800 1/10 Gigabit Ethernet", - "mounted": false, - "rate": 1000000000, - "vendor": "Broadcom Corporation" - }, - { - "bridged": false, - "device": "eno4", - "driver": "bnx2x", - "interface": "Ethernet", - "mac": "44:a8:42:15:c4:e3", - "management": false, - "model": "NetXtreme II BCM57800 1/10 Gigabit Ethernet", - "mounted": false, - "rate": 1000000000, - "vendor": "Broadcom Corporation" - } - ], - "node_type": "compute_skylake", - "placement": { - "node": 14, - "rack": 1 - }, - "processor": { - "cache_l1": null, - "cache_l1d": 32768, - "cache_l1i": 32768, - "cache_l2": 262144, - "cache_l3": 31457280, - "clock_speed": 3100000000, - "instruction_set": "x86-64", - "model": "Intel Xeon", - "other_description": "Intel(R) Xeon(R) CPU E5-2670 v3 @ 2.30GHz", - "vendor": "Intel", - "version": "E5-2670 v3" - }, - "storage_devices": [ - { - "device": "sda", - "driver": "megaraid_sas", - "humanized_size": "250 GB", - "interface": "SATA", - "model": "ST9250610NS", - "rev": "AA63", - "size": 250059350016, - "vendor": "Seagate" - } - ], - "supported_job_types": { - "besteffort": false, - "deploy": true, - "virtual": "ivt" - }, - "type": "node", - "uid": "b0525159-5c95-4b71-83f2-b8d6bdd2acd2", - "version": "53c90ef0512d5013ee30d431cd62e68bfd34d4ca" - } - ], - "hardware_page": 0, - "hardware_page_size": 25, - "metric_page": 0, - "metric_page_size": 25, - "metrics": [ - { - "instance_id": "44ad06ee-41d7-48f9-a52a-179030754707", - "metric_id": "dd22e02386714516a913d966659617eb", - "metric_name": "interface-eno1@if_dropped" - }, - { - "instance_id": "44ad06ee-41d7-48f9-a52a-179030754707", - "metric_id": "512ac94754b64906a12960d1f0a929c9", - "metric_name": "interface-eno1@if_errors" - }, - - ... - - { - "instance_id": "44ad06ee-41d7-48f9-a52a-179030754707", - "metric_id": "89cec271c0fb477b9e7bb37ad3df1331", - "metric_name": "memory@memory.slab_recl" - } - ], - "report_time_in_ct": "2018-10-19 11:06:51", - "report_time_in_utc": "2018-10-19 16:06:51", - "site": "CHI_DEV_UC", - "testbed_version": "702d4d47ab21c890c0bb146f4e0256f618264487" - } - -The ``events`` section is a list of testbed events ordered by event timestamp. The ``hardware`` section contains information of all the nodes that were used in the experiment. -The hardware information is retrieved by using the same method as :ref:`the Resource Discovery `. The ``metrics`` section is a list of the metrics captured during the experiment. -The *Experiment Precis* only contains the ``instance_id``, ``metric_id``, and ``metric_name`` in the ``metrics`` list. You can use :ref:`the openstack metric command line ` -to get all the measurements of a particular metric over time for an instance. - -For more information, run: - -.. code-block:: bash - - cep print --help - -.. important:: - Chameleon only keeps an experiment precis for **180 days**. - Please make sure to save your experiment precis you'd like to keep for a longer time by using ``cep print`` command. - You can output it to a file and keep it as a record. - -__________________ -Pagination -__________________ - -In the case of "large" experiment with large number of nodes and metrics, ``events``, ``hardwares``, and ``metrics`` are printed in pages. By default, the page number is set to 0 and the page size is set to 25. -However, you can tune the pagination by specifying the following parameters: - -.. code-block:: bash - - --event-page-size EVENT_PAGE_SIZE - Page size for event; ignored if event is excluded; set - to negative value to show all - --event-page EVENT_PAGE - Page number for event; ignored if event is excluded - --metric-page-size METRIC_PAGE_SIZE - Page size for metric; ignored if metric is excluded; - set to negative value to show all - --metric-page METRIC_PAGE - Page number for metric; ignored if metric is excluded - --hardware-page-size HARDWARE_PAGE_SIZE - Page size for hardware; ignored if hardware is - excluded; set to negative value to show all - --hardware-page HARDWARE_PAGE - Page number for hardware; ignored if hardware is - excluded - -.. tip:: - - To show all, set page size to a negative value. If page size is negative, ``page`` parameter will be ignored. Negative value for ``page`` is not allowed. - -.. _ep-filters: - -_____________________ -Filters -_____________________ - -The ``cep`` tool provides multiple filters to help you focus on the contents you care. - -**Event Filters** - -- To exclude all the events from the *Experiment Precis*, use ``--exclude-event``. -- To include or exclude services, use ``--include-services`` and/or ``--exclude-services``. - For example, if you only want to print ``blazar`` (reservation) and ``nova`` (instance) related events, run the following command: - - .. code-block:: bash - - cep print --pretty --include-services blazar,nova -- You can exclude event metadata by passing ``--exclude-event-metadata``. -- You can apply datetime filters to your events. For example, to print events up to ``2018-10-05 00:00:00``, run: - - .. code-block:: bash - - cep print --pretty --end-datetime "2018-10-05 00:00:00" - - Or to print events from ``2018-10-05 09:00:00`` to ``2018-10-05 17:00:00``, run: - - .. code-block:: bash - - cep print --pretty -start-datetime "2018-10-05 09:00:00" --end-datetime "2018-10-05 17:00:00" - -.. note:: - - When using datetime filters, use datetime in **UTC**. - -**Metric Filters** - -- To exclude metrics from the *Experiment Precis*, use ``--exclude-metric``. - -**Hardware Filters** - -- To exclude hardware information from the *Experiment Precis*, use ``--exclude-hardware``. diff --git a/source/technical/fpga.rst b/source/technical/fpga.rst index 76ba472..cea0590 100644 --- a/source/technical/fpga.rst +++ b/source/technical/fpga.rst @@ -1,129 +1,11 @@ -====== -FPGAs -====== +:orphan: -.. attention:: - **Our previously supported Altera FPGA nodes at CHI@UC and CHI@TACC are in the process of being decommissioned.** +.. meta:: + :http-equiv=refresh: 0; url=./fpga/index.html - We are actively enhancing our FPGA capabilities with new offerings in development. **Altera nodes** previously supported on Chameleon **will be unavailable after Feb. 7, 2025**. - - While these updates are in progress, users can **continue accessing** our existing **Xilinx FPGA resources** **using documentation below**. Our team is working on implementing improved workflows to streamline current FPGA development and deployment processes. These enhancements will provide more efficient ways to utilize this hardware. +This page has moved +=================== - **If you have ideas or suggestions regarding the use of FPGAs on Chameleon, please reach out to us at** - contact@chameleoncloud.org. - - We will provide additional updates as new features and hardware become available. Stay connected with our platform for the latest announcements. +This page has been reorganized. You will be redirected to the new location. -____________ -Introduction -____________ - -Chameleon provides access to two `Xilinx Alveo U280 `_ FPGA nodes, both of which are available at |CHI@UC|. Each node is equipped with: - -- Memory: - - 32 GB DDR4 memory (two 16 GB channels) - - 8 GB HBM (High Bandwidth Memory) - - On-chip PLRAM memory -- Connectivity: - - PCIe Gen3 x16 interface - - Two QSFP28 ports supporting up to 100Gbps network access (*Note: These ports are not configured currently but will be available in a future update*) -- Additional Specifications: - - 300 MHz default clock - - Three Super Logic Regions (SLRs) with dedicated compute and memory resources - -The workflow for using these FPGAs on Chameleon consists of four main steps: - -1. Reserve a bare metal node with FPGA hardware -2. Launch an instance using a supported base image -3. Install required Xilinx software tools in your environment -4. Load your pre-compiled bistream onto the FPGA and run your application - -.. important:: - Chameleon does not provide FPGA compilation services or development tools. Users need to compile their code elsewhere before running it on Chameleon's FPGAs. We are currently exploring new ways to provide FPGA development tools and workflows in the future. - -__________________________ -Reserverving FPGA Hardware -__________________________ - -CHI@UC provides two `compute_cascadelake_r` nodes equipped with Xilinx Alveo U280 FPGAs: - -- Node ``P3-CPU-038`` (UUID: ``89e48f7e-d04f-4455-b093-2a4318fb7026``) -- Node ``P3-CPU-042`` (UUID: ``926a9c99-3b27-45a7-818c-e6525b9ce89c``) - -To ensure you get the correct hardware, reserve these nodes specifically by UUID, `as explained here in our documentation `_. - -_________________________ -Launching Your Instance -_________________________ - -After your reservation becomes active: - -- Launch an instance using a supported upstream image for the Xilinx Runtime. Chameleon suggests using our supported CC-Ubuntu 24.04 as a base image. Advanced users may choose to use any upstream image that Xilinx supports. Find a list of supported upstream images `here `_ -- Connect to your instance via SSH - -_______________________ -Installing Xilinx Tools -_______________________ - -Compiling code for FPGAs requires the Xilinx Vitis™ software platform, which provides a comprehensive development environment for creating FPGA-accelerated applications. The Vitis platform includes the Vitis Unified Software Platform, Vitis Core Development Kit, and Vitis AI Development Kit. - -Flashing the FPGA with your bitstream requires the Xilinx Runtime (XRT) tools, which are part of the Vitis platform. The XRT tools provide a command-line interface for managing FPGA devices, including programming the FPGA with your bitstream. You can also install the XRT environment separately from the Vitis platform, although functionality may be limited. - -Guidelines for installing the Vitis platform can be found in the `AMD documentation `_. The installation requirements for Ubuntu are also provided in the documentation `here `_. - -Guidelines for installing the Xilinx Runtime (XRT) tools can be found in the `XRT documentation `_. - -___________________________ -Loading Your Bitstream -___________________________ - -After installing the required tools, you can program the Xilinx Alveo U280 FPGA with your pre-compiled bitstream using the `Xilinx Runtime (XRT) tools `_. The steps below are the basic workflow for flashing, but we encourage users to review the AMD documentation for more detailed instructions. - -**1. Verify the FPGA Device** - -First, ensure that the FPGA device is recognized and ready. Use the ``xbmgmt`` tool to examine the device status: - -.. code-block:: bash - - sudo xbmgmt examine --verbose - -Look for the device BDF (Bus:Device.Function) identifier, which you'll need in the next steps. For example, it might be ``0000:81:00.0``. - -**2. Program the FPGA with Your Bitstream** - -Use the ``xbmgmt`` tool to program the FPGA with your bitstream. Replace ```` with your device's BDF and ```` with the path to your ``.xclbin`` file: - -.. code-block:: bash - - sudo xbmgmt program --device --base --image - -For example: - -.. code-block:: bash - - sudo xbmgmt program --device 0000:81:00.0 --base --image /path/to/your_bitstream.xclbin - -This command will program the FPGA with your specified bitstream. - -**3. Reboot the System** - -After programming the FPGA, it's recommended to perform a cold reboot to ensure the new image is properly loaded: - -.. code-block:: bash - - sudo reboot - -**4. Verify the New Configuration** - -Once the system restarts, verify that the new configuration is active: - -.. code-block:: bash - sudo xbmgmt examine --verbose - -Ensure that the device is ready and the new platform UUID matches your programmed bitstream. - -.. important:: - - Ensure that your bitstream (``.xclbin`` file) is compatible with the Alveo U280 FPGA. - - The ``xbmgmt`` tool is part of the XRT installation and is used for managing FPGA devices. - - For detailed instructions and troubleshooting, refer to the `XRT documentation `_. - - Additional AMD instructions for bringing up and validating your card `here `_. +If you are not redirected automatically, please visit :doc:`fpga/index`. \ No newline at end of file diff --git a/source/technical/fpga/index.rst b/source/technical/fpga/index.rst new file mode 100644 index 0000000..7935223 --- /dev/null +++ b/source/technical/fpga/index.rst @@ -0,0 +1,130 @@ +====== +FPGAs +====== + +.. attention:: + **Our previously supported Altera FPGA nodes at CHI@UC and CHI@TACC are in the process of being decommissioned.** + + We are actively enhancing our FPGA capabilities with new offerings in development. **Altera nodes** previously supported on Chameleon **will be unavailable after Feb. 7, 2025**. + + While these updates are in progress, users can **continue accessing** our existing **Xilinx FPGA resources** **using documentation below**. Our team is working on implementing improved workflows to streamline current FPGA development and deployment processes. These enhancements will provide more efficient ways to utilize this hardware. + + **If you have ideas or suggestions regarding the use of FPGAs on Chameleon, please reach out to us at** + contact@chameleoncloud.org. + + We will provide additional updates as new features and hardware become available. Stay connected with our platform for the latest announcements. + +____________ +Introduction +____________ + +Chameleon provides access to two `Xilinx Alveo U280 `_ FPGA nodes, both of which are available at |CHI@UC|. Each node is equipped with: + +- Memory: + - 32 GB DDR4 memory (two 16 GB channels) + - 8 GB HBM (High Bandwidth Memory) + - On-chip PLRAM memory +- Connectivity: + - PCIe Gen3 x16 interface + - Two QSFP28 ports supporting up to 100Gbps network access (*Note: These ports are not configured currently but will be available in a future update*) +- Additional Specifications: + - 300 MHz default clock + - Three Super Logic Regions (SLRs) with dedicated compute and memory resources + +The workflow for using these FPGAs on Chameleon consists of four main steps: + +1. Reserve a bare metal node with FPGA hardware +2. Launch an instance using a supported base image +3. Install required Xilinx software tools in your environment +4. Load your pre-compiled bitstream onto the FPGA and run your application + +.. important:: + Chameleon does not provide FPGA compilation services or development tools. Users need to compile their code elsewhere before running it on Chameleon's FPGAs. We are currently exploring new ways to provide FPGA development tools and workflows in the future. + +_________________________ +Reserving FPGA Hardware +_________________________ + +CHI@UC provides two `compute_cascadelake_r` nodes equipped with Xilinx Alveo U280 FPGAs: + +- Node ``P3-CPU-038`` (UUID: ``89e48f7e-d04f-4455-b093-2a4318fb7026``) +- Node ``P3-CPU-042`` (UUID: ``926a9c99-3b27-45a7-818c-e6525b9ce89c``) + +To ensure you get the correct hardware, reserve these nodes specifically by UUID, `as explained here in our documentation `_. + +_________________________ +Launching Your Instance +_________________________ + +After your reservation becomes active: + +- Launch an instance using a supported upstream image for the Xilinx Runtime. Chameleon suggests using our supported CC-Ubuntu 24.04 as a base image. Advanced users may choose to use any upstream image that Xilinx supports. Find a list of supported upstream images `here `_ +- Connect to your instance via SSH + +_______________________ +Installing Xilinx Tools +_______________________ + +Compiling code for FPGAs requires the Xilinx Vitis™ software platform, which provides a comprehensive development environment for creating FPGA-accelerated applications. The Vitis platform includes the Vitis Unified Software Platform, Vitis Core Development Kit, and Vitis AI Development Kit. + +Flashing the FPGA with your bitstream requires the Xilinx Runtime (XRT) tools, which are part of the Vitis platform. The XRT tools provide a command-line interface for managing FPGA devices, including programming the FPGA with your bitstream. You can also install the XRT environment separately from the Vitis platform, although functionality may be limited. + +Guidelines for installing the Vitis platform can be found in the `AMD documentation `_. The installation requirements for Ubuntu are also provided in the documentation `here `_. + +Guidelines for installing the Xilinx Runtime (XRT) tools can be found in the `XRT documentation `_. + +___________________________ +Loading Your Bitstream +___________________________ + +After installing the required tools, you can program the Xilinx Alveo U280 FPGA with your pre-compiled bitstream using the `Xilinx Runtime (XRT) tools `_. The steps below are the basic workflow for flashing, but we encourage users to review the AMD documentation for more detailed instructions. + +**1. Verify the FPGA Device** + +First, ensure that the FPGA device is recognized and ready. Use the ``xbmgmt`` tool to examine the device status: + +.. code-block:: bash + + sudo xbmgmt examine --verbose + +Look for the device BDF (Bus:Device.Function) identifier, which you'll need in the next steps. For example, it might be ``0000:81:00.0``. + +**2. Program the FPGA with Your Bitstream** + +Use the ``xbmgmt`` tool to program the FPGA with your bitstream. Replace ```` with your device's BDF and ```` with the path to your ``.xclbin`` file: + +.. code-block:: bash + + sudo xbmgmt program --device --base --image + +For example: + +.. code-block:: bash + + sudo xbmgmt program --device 0000:81:00.0 --base --image /path/to/your_bitstream.xclbin + +This command will program the FPGA with your specified bitstream. + +**3. Reboot the System** + +After programming the FPGA, it's recommended to perform a cold reboot to ensure the new image is properly loaded: + +.. code-block:: bash + + sudo reboot + +**4. Verify the New Configuration** + +Once the system restarts, verify that the new configuration is active: + +.. code-block:: bash + + sudo xbmgmt examine --verbose + +Ensure that the device is ready and the new platform UUID matches your programmed bitstream. + +.. important:: + - Ensure that your bitstream (``.xclbin`` file) is compatible with the Alveo U280 FPGA. + - The ``xbmgmt`` tool is part of the XRT installation and is used for managing FPGA devices. + - For detailed instructions and troubleshooting, refer to the `XRT documentation `_. + - Additional AMD instructions for bringing up and validating your card `here `_. diff --git a/source/technical/gui.rst b/source/technical/gui.rst index 3810aed..2e7dbdf 100644 --- a/source/technical/gui.rst +++ b/source/technical/gui.rst @@ -1,474 +1,11 @@ -.. _gui: +:orphan: -=============================== -Graphical User Interface (GUI) -=============================== +.. meta:: + :http-equiv=refresh: 0; url=./gui/index.html -Introduction -============ +This page has moved +=================== -The Graphical User Interface (GUI) provides a point-and-click experience for -working with Chameleon resources. From the GUI, you may perform tasks such as -manage and launch instances, and configure custom networking. Additionally, you -may download an *OpenStack RC* file from the GUI if you wish to work with the -:ref:`Command Line Interface `, instead. The Chameleon GUI is built on top -of `OpenStack Horizon `_. There are -two Chameleon resource sites, each with its own URL (though it is possible to -easily switch from one to other, see :ref:`gui-project-menu`). +This page has been reorganized. You will be redirected to the new location. -- The Texas Advanced Computing Center resources (CHI\@TACC) are available at: - - https://chi.tacc.chameleoncloud.org - -- The University of Chicago resources (CHI\@UC) are available at: - - https://chi.uc.chameleoncloud.org - -Chameleon also hosts an *OpenStack KVM* implementation where you may work with -virtual machines. This site **does not** have access to bare metal resources. It -is available at: - - https://kvm.tacc.chameleoncloud.org - -This section provides an overview of features available on the GUI for the bare -metal sites at the Texas Advanced Computing Center (|CHI@TACC|) and the -University of Chicago (|CHI@UC|). For information about *OpenStack KVM*, please -see :ref:`kvm`. - -You may login to either site using your Chameleon portal username and password. - -.. _bare-metal-sites-independent: -.. attention:: - Each Chameleon testbed sites---|CHI@TACC|, |CHI@UC|, and |KVM@TACC|---are - **independent**, so snapshots, keypairs, Swift containers, Gnocchi - metrics and other objects are unique to each site. For example, a - keypair created at the |CHI@TACC| site is **not** available at the |CHI@UC| - site. In addition, the bare metal resource types vary between sites. - -GUI Features -============ - -Upon logging in to the GUI at a Chameleon site, you will see your project's -Overview page. - -.. figure:: gui/gui.png - :alt: The Chameleon GUI - - The Chameleon GUI - -.. _gui-project-menu: - -Project and Site Menu ---------------------- - -To switch among the projects you belong to, use the project and site menu---the -dropdown on the upper left of the screen next to the Chameleon logo. You can -also use this menu to switch from one Chameleon site to another. This allows you -to easily perform multi-site experiments. - -.. figure:: gui/project_dropdown.png - :alt: Switching between projects - - Switching between projects - -.. _gui-user-menu: - -User Menu ---------- - -To access user specific settings and download *OpenStack RC* files, use the user -menu---the dropdown on the upper right of the screen where you will see your -account name. - -.. figure:: gui/user_dropdown.png - :alt: The user dropdown menu - - The user dropdown menu - -.. _gui-settings: - -Settings --------- - -In the settings menu, you can change user specific settings such as the -Timezone. - -.. figure:: gui/user_settings.png - :alt: User settings - - User settings - -.. note:: - - Updating your timezone is **highly** recommended. When you make reservations - for bare metal resources, your local time will be used. UTC is the default - Timezone. - - -Help ----- - -The *Help* menu item will take you to this documentation site. - - -OpenStack RC File ------------------ - -Clicking on this menu items will download a customized `RC file -`_ for use with the OpenStack -Command Line Interface. Source the RC file using ``source`` command to configure -environment variables that allow you to easily log in using the :ref:`Command -Line Interface `. For more information about *OpenStack RC* script, please -see :ref:`cli-rc-script`. - - -Themes ------- - -You may change the GUI theme by selecting the provided menu items. - - -Sign Out --------- - -Use the *sign out* menu item to sign out from your current site. - -.. note:: - - If you do not sign out manually, your session will expire in one hour. - - -Navigating the GUI -================== - -The navigation sidebar allows you to access different sections. - -.. figure:: gui/sidebar.png - :alt: The GUI sidebar - -.. _gui-api-access: - -API Access -========== - -The API Access page lists all the available REST APIs that are used for -configuring the :ref:`cli`. In addition, you may download :ref:`cli-rc-script` -scripts via this page. - -.. note:: - - Typically, the key generated from your computer will be at - ``~/.ssh/id_rsa.pub``. On Mac OS X, you can run in a terminal: ``cat - ~/.ssh/id_rsa.pub | pbcopy``. It copies the content of the public key to your - copy/paste buffer. Then you can simply paste in the "Public Key" box. - -.. figure:: gui/api_access.png - :alt: The API Access page - - The API Access page - -.. _gui-compute: - -Compute -======= - -Use *Compute* section for reserving, configuring and managing your instances. - -Overview --------- - -The Overview page provides a graphical summary of your project's current -resource usage. - -.. figure:: gui/overview.png - :alt: The Overview page - -.. note:: - At the bare metal sites, you may launch as many instances as you like, but - bounded by the project :ref:`Service Unit ` allocation. - However, at the OpenStack KVM site, your project is limited to a certain - number of virtual machines. By default, each project is allowed to allocate - 50 *Floating IP addresses* and use 10 *Security Groups*. You may request - additional resources by submitting a ticket on the |Help Desk|. - -.. _gui-compute-instances: - -Instances ---------- - -The Instances page allows you to work with your instances. You may launch, -terminate, monitor, or reboot an instance. Clicking on the dropdown list in -*Action* column to see what you are eligible to do to your instances. - -.. figure:: gui/instances.png - :alt: The Instances page - -Images ------- - -The Images page allows you to view, upload and edit the images. You may also use -this page to launch instance using selected images. - -.. note:: You can only edit the images you own. - -.. figure:: gui/images.png - :alt: The Images page - -.. tip:: Search for images using the filter bar. - -.. _gui-key-pairs: - -Key Pairs ---------- - -The Key Pairs page allows you to create, import and manage SSH key pairs -associated with your user account. - -.. figure:: gui/key_pairs.png - :alt: The Key Pairs page - -.. note:: - - Chameleon **only** stores the *public key* for each SSH key pair. **Do not** - upload your *private key* to the portal! Private keys look like this: - - .. code-block:: - - -----BEGIN RSA PRIVATE KEY----- - -To delete a SSH key pair, click on the *Delete Key Pair* button in the *Action* -column. You may delete multiple key pairs by selecting them via the checkbox and -clicking the *Delete Key Pairs* button. - -Creating a Key Pair -~~~~~~~~~~~~~~~~~~~ - -To create a key pair, click the *+ Create Key Pair* button. In the prompted -dialog, provide a name for your new key pair and then click the *Create Key -Pair* button. - -.. figure:: gui/create_key_pair_name.png - :alt: Specifying a key pair name - - Specifying a key pair name - -A ``.pem`` file that contains the *Private Key* should be automatically -downloaded. In addition, the *Public Key* associated with the *Private Key* -should be saved automatically to Chameleon. Clicking on the *Regenerate and -download Key Pair* button will generate a new *Public/Private Key Pair* and -initiate a new download of the *Private Key*. - -.. tip:: - Save the *Private Key* to a location you will remember at your local file - system. Your *home* directory is recommanded for macOS and Linux systems. - -.. _importing-key-pair: - -Importing a Key Pair -~~~~~~~~~~~~~~~~~~~~ - -Alternatively, you may import a key pair that you have generated on your -computer. Clicking the *Import Key Pair* button to prompt the dialog. Then, -provide a name for your imported key pair and paste the *Public Key*. - -.. tip:: - The prompted dialog contains the instructions on how to generate a key pair - using the Linux/macOS command. - -.. figure:: gui/import_key_pair.png - :alt: Importing a public key - - Importing a public key - -.. tip:: - Typically, the key generated from your computer will be at - ``~/.ssh/id_rsa.pub``. On Mac OS X, you can run in a terminal: ``cat - ~/.ssh/id_rsa.pub | pbcopy``. It copies the content of the public key to your - copy/paste buffer. Then you can simply paste in the "Public Key" box. - -Network -======= - -The Network section allows you to work with virtual network resources, such as -configuring routers and virtual networks. For more information, please see -:ref:`networking`. - -Network Topology ----------------- - -The Network Topology page displays your current virtual network topology in -either the *Topology* or *Graph* formats. You may also use this section to -directly launch instances, create networks or create routers. - -.. figure:: gui/network_topology.png - :alt: The Network Topology page - - The Network Topology page - -Networks --------- - -The Networks page lists all the Virtual Networks of the selected project. You -may use this section to create, delete and modify Virtual Networks. Clicking on -the dropdown list (if shown) in *Action* column to see what you are eligible to -do to your virtual networks. - -.. figure:: gui/networks.png - :alt: The Networks page - - The Networks page - -Routers -------- - -Same as the Networks page, the Routers page allows you to work on the Routers of -the selected project. - -.. figure:: gui/routers.png - :alt: The Routers page - - The Routers page - - -Security Groups ---------------- - -Use the Security Groups page to create, delete, and modify the Security Groups -of the selected project. - -.. figure:: gui/security_groups.png - :alt: The Security Groups page - - The Security Groups page - -.. attention:: - Chameleon bare metal sites---|CHI@TACC| and |CHI@UC|---**do not** support - security groups, i.e., all ports are open to the public. - - -Floating IPs ------------- - -The Floating IPs page allows you to work with the Floating IP addresses -allocated for the selected project, including associating with instances and -releasing back to the pool. Clicking on the dropdown list (if shown) in *Action* -column to see what you are eligible to do to your Floating IPs. - -.. figure:: gui/floating_ips.png - :alt: The Floating IPs page - - The Floating IPs page - -Releasing Floating IP Addresses -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. important:: - - The Chameleon Floating IP address pool is a shared and finite resource. - **Please be responsible and release the Floating IP addresses that are not - used, so other Chameleon users and projects can use them!** - -To release a single Floating IP address, click on the dropdown in the *Actions* -column and select *Release Floating IP* . You may also release multiple -addresses by selecting them via checkboxes and clicking the *Release Floating -IPs* button. - -.. figure:: gui/releasing.png - :alt: Releasing a Floating IP address - - Releasing a Floating IP address - -Orchestration -============= - -The Orchestration section allows you to work with the :ref:`Chameleon's Complex -Appliances `. - - -Stacks ------- - -A deployed complex appliance is referred to as a “stack” – just as a deployed -single appliance is typically referred to as an “instance”. The Stacks page -allows you to launch, rebuild, or terminate stacks. - -.. figure:: gui/stacks.png - :alt: The Stacks page - - The Stacks page - -.. tip:: - - After launching a stack, all the instances launched with the stack can be - viewed at :ref:`Compute - Instances ` section as well. - -.. note:: - - When you terminate a stack, all instances launched with the stack will be - terminated. - -Resource Types --------------- - -The Resource Types page lists the currently available Orchestration Resource -Types of Chameleon. You may click on the resource types to get details. The -Orchestration Resource Types are used when writing *OpenStack Heat Orchestration -Template*. For more information about *OpenStack Heat*, please see `the -OpenStack Heat documentation `_. - -.. figure:: gui/resource_types.png - :alt: The Resource Types page - - The Resource Types page - -Template Versions ------------------ - -The Template Versions are also used when writing *OpenStack Heat Orchestration -Template*. Clicking on the version to get supported features of the specific -version. - -.. figure:: gui/template_versions.png - :alt: The Template Versions page - - The Template Versions page - -Object Store -============ - -The *Containers* section under *Object Store* gives an easy access to your -Chameleon object/blob store. You may create, delete, upload objects to or remove -objects from containers via this page. For more information about Chameleon -Object Store, please see :ref:`object-store`. - -.. figure:: gui/containers.png - :alt: The Containers page - - The Containers page - -Reservations -============ - -The Reservations section allows you to manage your leases of the selected -project, including creating and deleting leases. For more information, see -:ref:`reservations`. - -.. figure:: gui/leases.png - :alt: The Leases page - - The Leases page - -.. tip:: - Check *Lease Calendar*, so you can schedule your experiments efficiently. - -Identity -======== - -The Project section under Identity allows you to check what projects you belong -to. You can set your default project by clicking the *Set as Active Project* -button in the *Actions* column. - -.. figure:: gui/projects.png - :alt: The Projets page - - The Projects page +If you are not redirected automatically, please visit :doc:`gui/index`. \ No newline at end of file diff --git a/source/technical/gui/api_access.rst b/source/technical/gui/api_access.rst new file mode 100644 index 0000000..c27d3c3 --- /dev/null +++ b/source/technical/gui/api_access.rst @@ -0,0 +1,20 @@ +.. _gui-api-access: + +API Access +========== + +The API Access page lists all the available REST APIs that are used for +configuring the :ref:`cli`. In addition, you may download :ref:`cli-rc-script` +scripts via this page. + +.. note:: + + Typically, the key generated from your computer will be at + ``~/.ssh/id_rsa.pub``. On Mac OS X, you can run in a terminal: ``cat + ~/.ssh/id_rsa.pub | pbcopy``. It copies the content of the public key to your + copy/paste buffer. Then you can simply paste in the "Public Key" box. + +.. figure:: api_access.png + :alt: The API Access page + + The API Access page \ No newline at end of file diff --git a/source/technical/gui/features.rst b/source/technical/gui/features.rst new file mode 100644 index 0000000..a660802 --- /dev/null +++ b/source/technical/gui/features.rst @@ -0,0 +1,84 @@ +GUI Features +============ + +Upon logging in to the GUI at a Chameleon site, you will see your project's +Overview page. + +.. figure:: gui.png + :alt: The Chameleon GUI + + The Chameleon GUI + +.. _gui-project-menu: + +Project and Site Menu +--------------------- + +To switch among the projects you belong to, use the project and site menu---the +dropdown on the upper left of the screen next to the Chameleon logo. You can +also use this menu to switch from one Chameleon site to another. This allows you +to easily perform multi-site experiments. + +.. figure:: project_dropdown.png + :alt: Switching between projects + + Switching between projects + +.. _gui-user-menu: + +User Menu +--------- + +To access user specific settings and download *OpenStack RC* files, use the user +menu---the dropdown on the upper right of the screen where you will see your +account name. + +.. figure:: user_dropdown.png + :alt: The user dropdown menu + + The user dropdown menu + +.. _gui-settings: + +Settings +~~~~~~~~ + +In the settings menu, you can change user specific settings such as the +Timezone. + +.. figure:: user_settings.png + :alt: User settings + + User settings + +.. note:: + + Updating your timezone is **highly** recommended. When you make reservations + for bare metal resources, your local time will be used. UTC is the default + Timezone. + + +Help +~~~~ + +The *Help* menu item will take you to this documentation site. + + +OpenStack RC File +~~~~~~~~~~~~~~~~~ + +Clicking on this menu items will download a customized `RC file +`_ for use with the OpenStack +Command Line Interface. Source the RC file using ``source`` command to configure +environment variables that allow you to easily log in using the :ref:`Command +Line Interface `. For more information about *OpenStack RC* script, please +see :ref:`cli-rc-script`. + +Sign Out +~~~~~~~~ + +Use the *sign out* menu item to sign out from your current site. + +.. note:: + + If you do not sign out manually, your session will expire in one hour. \ No newline at end of file diff --git a/source/technical/gui/index.rst b/source/technical/gui/index.rst new file mode 100644 index 0000000..9e1c140 --- /dev/null +++ b/source/technical/gui/index.rst @@ -0,0 +1,47 @@ +.. _gui: + +=============================== +Graphical User Interface (GUI) +=============================== + +The Graphical User Interface (GUI) provides a point-and-click experience for +working with Chameleon resources. From the GUI, you may perform tasks such as +manage and launch instances, and configure custom networking. Additionally, you +may download an *OpenStack RC* file from the GUI if you wish to work with the +:ref:`Command Line Interface `, instead. The Chameleon GUI is built on top +of `OpenStack Horizon `_ (running OpenStack Antelope). Chameleon +has multiple resource sites, each with its own URL (though it is possible to +easily switch from one to other, see :ref:`gui-project-menu`). + +- |CHI@TACC| - Texas Advanced Computing Center: https://chi.tacc.chameleoncloud.org +- |CHI@UC| - University of Chicago: https://chi.uc.chameleoncloud.org +- |CHI@NCAR| - National Center for Atmospheric Research: https://chi.ncar.chameleoncloud.org +- |CHI@Edge| - Edge computing testbed: https://chi.edge.chameleoncloud.org + +Chameleon also hosts |KVM@TACC|, a traditional OpenStack cloud where you may work with +virtual machines. This site **does not** have access to bare metal resources. It +is available at: https://kvm.tacc.chameleoncloud.org. + +This section provides an overview of GUI interface navigation and basic functionality. +For detailed instructions on using specific Chameleon features, see the dedicated +documentation sections: :ref:`baremetal`, :ref:`networking`, :ref:`reservations`, +:ref:`images`, :ref:`object-store`, and :ref:`complex`. + +You may login to either site using your Chameleon portal username and password. + +.. _bare-metal-sites-independent: +.. attention:: + + Each Chameleon testbed site---|CHI@TACC|, |CHI@UC|, |CHI@NCAR|, |CHI@Edge|, and |KVM@TACC|---is + **independent**, so snapshots, keypairs, Swift containers, and other objects + are unique to each site. For example, a keypair created at the |CHI@TACC| + site is **not** available at the |CHI@UC| site. In addition, the bare metal + resource types vary between sites. + +.. toctree:: + :maxdepth: 1 + :caption: GUI Topics + + features + api_access + navigation \ No newline at end of file diff --git a/source/technical/gui/navigation.rst b/source/technical/gui/navigation.rst new file mode 100644 index 0000000..a5c9bc3 --- /dev/null +++ b/source/technical/gui/navigation.rst @@ -0,0 +1,145 @@ +GUI Navigation +============== + +The navigation sidebar on the left allows you to access different sections +of the interface. The main navigation elements are described below. + +.. figure:: sidebar.png + :alt: The GUI sidebar + + +.. _gui-compute: + +Compute +------- + +The *Compute* section provides interfaces for managing instances, images, and SSH key pairs. + +Overview +~~~~~~~~ + +The Overview page provides a graphical summary of your project's current resource usage. + +.. figure:: overview.png + :alt: The Overview page + +.. _gui-compute-instances: + +Instances +~~~~~~~~~ + +The *Instances* page displays your running instances with options to launch, terminate, +monitor, or reboot them. For detailed instructions on launching and managing instances, +see :ref:`baremetal`. + +.. figure:: instances.png + :alt: The Instances page + +Images +~~~~~~ + +The *Images* page allows you to view available images and launch instances from them. +You can only edit images you own. For comprehensive image management including uploading +and sharing, see :ref:`images`. + +.. figure:: images.png + :alt: The Images page + +.. _gui-key-pairs: + +Key Pairs +~~~~~~~~~ + +The *Key Pairs* page allows you to create, import and manage SSH key pairs for instance access. + +.. figure:: key_pairs.png + :alt: The Key Pairs page + +For detailed instructions on creating and importing key pairs, see the +:ref:`baremetal instance launch guide `. + +Network +------- + +The *Network* section provides interfaces for managing virtual network resources. +For comprehensive networking instructions, see :ref:`networking`. + +Network Topology +~~~~~~~~~~~~~~~~ + +The *Network Topology* page displays your current virtual network topology in +topology or graph formats. + +.. figure:: network_topology.png + :alt: The Network Topology page + + The Network Topology page + +Networks, Routers, and Floating IPs +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The *Networks*, *Routers*, and *Floating IPs* pages allow you to create and manage +these network resources for your project. + +.. figure:: networks.png + :alt: The Networks page + +.. attention:: + Chameleon bare metal sites (|CHI@TACC|, |CHI@UC|, |CHI@NCAR|) **do not** support + security groups - all ports are open to the public. + +For detailed networking procedures including floating IP management, see :ref:`networking`. + +Orchestration +------------- + +The *Orchestration* section provides interfaces for working with complex appliances +and Heat templates. For comprehensive instructions, see :ref:`complex`. + + +Stacks +~~~~~~ + +A deployed complex appliance is referred to as a "stack" – just as a deployed +single appliance is typically referred to as an "instance". The Stacks page +allows you to launch, rebuild, or terminate stacks. + +.. figure:: stacks.png + :alt: The Stacks page + + The Stacks page + + + +Object Store +------------ + +The *Containers* section provides access to Chameleon's object/blob storage. +For detailed object store instructions, see :ref:`object-store`. + +.. figure:: containers.png + :alt: The Containers page + + The Containers page + +Reservations +------------ + +The *Reservations* section allows you to manage your resource leases. +For comprehensive reservation instructions, see :ref:`reservations`. + +.. figure:: leases.png + :alt: The Leases page + + The Leases page + +Identity +-------- + +The *Projects* section under *Identity* shows projects you belong to and allows +you to set your default project. + +.. figure:: projects.png + :alt: The Projects page + + The Projects page \ No newline at end of file diff --git a/source/technical/images.rst b/source/technical/images.rst index 6a73244..e852a28 100644 --- a/source/technical/images.rst +++ b/source/technical/images.rst @@ -1,296 +1,11 @@ -.. _images: +:orphan: -==================== -Images -==================== +.. meta:: + :http-equiv=refresh: 0; url=./images/index.html -All instances in Chameleon, whether KVM or bare metal, are running off disk images. The content of these disk images can be snapshotted at any point in time, which allows you to save your work and launch new instances from updated images later. While OpenStack KVM has built-in support for snapshotting in the Horizon web interface and via the command line, bare metal instances require a more complex process. +This page has moved +=================== -To work around this limitation, we provide the ``cc-snapshot`` utility that you can execute from inside your running instance. The ``cc-snapshot`` utility is pre-installed in all Chameleon supported appliances. You can find our appliances from the `Appliance Catalog `_. +This page has been reorganized. You will be redirected to the new location. -The image service on Chameleon uses `OpenStack Glance `_. This documentation demonstrates how to accomplish common tasks with *Images* using the GUI and the CLI. - -__________________________________ -Chameleon Supported Images -__________________________________ - -There are a number of images built and supported by the Chameleon team, -specifically: - -- CC-Ubuntu24.04 -- CC-Ubuntu24.04-CUDA -- CC-Ubuntu24.04-ROCm -- CC-Ubuntu24.04-ARM64 -- CC-Ubuntu22.04 -- CC-Ubuntu22.04-CUDA -- CC-Ubuntu22.04-ARM64 -- CC-Centos9-Stream - -The CUDA images, such as `Ubuntu24.04-CUDA `_, -contain various settings, software, and drivers specifically -for NVIDIA GPU nodes. The ROCm images contain similar settings, software, -and drivers for AMD GPU nodes. And finally, the ARM64 images, such as -`Ubuntu22.04-ARM64 `_, -are images specifically built with ARM support for ARM nodes. Non-ARM -images all assume x86-based architectures. - -.. warning:: - Any images with operating system versions that are end-of-life, such as - Ubuntu18.04 are no longer offered as Chameleon-supported images. However, - they can still be used on Chameleon with caution. Please note that these - images are EOL so they no longer receive security updates and bug fixes. - They may stop working at any point, therefore, make sure to upgrade and - move your environments to newer versions of the operating system as soon as - possible. If you do need to launch older images on Chameleon, please consider - using a `bastion host `_ - that is up-to-date and provides SSH access. From there, you can jump to your - instances running older images that are not exposed directly on the Internet. - -You may also build your own images and share them with the community. These images -may be built from scratch or based on the Chameleon-supported images. However, -the Chameleon team cannot offer support for user-provided images. - -.. _cc-snapshot-utility: - -_________________________________________________ -The ``cc-snapshot`` Utility -_________________________________________________ - -The ``cc-snapshot`` utility implements snapshotting a bare metal instance from command line and uploads it to `Glance `_, so that it can be immediately used to boot a new bare metal instance. The snapshot images created with this tool are whole disk images. - -For ease of use, ``cc-snapshot`` has been installed in all the appliances supported by the Chameleon project. If you would like to use it in a different setting, it can be downloaded and installed from the `github repository `_. - -To make a snapshot of a bare metal instance, run the following command from inside the instance: - -.. code-block:: bash - - sudo cc-snapshot - -.. tip:: - You may get warnings, such as "image too large", during snapshotting, and get prompted to confirm. If you are confident about what you are trying to do, you can skip all warnings by using the ``-f`` flag. - - .. code-block:: bash - - sudo cc-snapshot -f - - In addition, you can exclude directories by using the ``-e`` flag. - - .. code-block:: bash - - sudo cc-snapshot -e -e - - To see all available options for ``cc-snapshot``, run ``sudo cc-snapshot -h``. - -You will be prompted to enter your username and password. - -.. tip:: You can skip entering username and password by setting the ``OS_USERNAME`` and ``OS_PASSWORD`` environment variables. You can set those environment variables manually or using :ref:`cli-rc-script`. - -.. note:: When using the ``cc-snapshot``, it will create an image within your project with the ``shared`` visibility. Anyone with access to your project can access this image. - -.. note:: If you choose an *Image* name that already exists, the previous one **will not** be overwritten. A new *Image* with the same name but a different *UUID* will be generated. - -.. note:: If you install a custom kernel, please make sure the size of your running kernel (``/lib/modules/``) is less than 4GB. To find out which kernel version you're running, run ``uname -r``. - -.. _updating-snapshot: - -.. error:: - If you receive the following error: - - .. code:: - - public endpoint for image service in regionOne not found Unable to contact Glance, check username and password - - it means that you have an outdated copy of ``cc-snapshot`` and you will need to update ``cc-snapshot``. - This usually happens when you use an older images that contains an outdated version of ``cc-snapshot``. - - You may also want to get new functionalities added to the latest version of ``cc-snapshot``. - - Run the following commands from your instance: - - .. code:: - - curl -O https://raw.githubusercontent.com/ChameleonCloud/cc-snapshot/master/cc-snapshot - sudo mv cc-snapshot /usr/bin/ - sudo chmod +x /usr/bin/cc-snapshot - -__________________________________ -Managing Images using the GUI -__________________________________ - -To manage your images, use the *Images* page at |CHI@TACC| or |CHI@UC|, by clicking on *Project* > *Compute* > *Images*. - -.. figure:: images/imagespagev3.png - :alt: The Images page - - The Images page - -.. note:: The Chameleon logo next to an image's name indicates that this image is an appliance supported by the Chameleon project, and is part of the Appliance Catalog. - -.. tip:: Select *Details* from the dropdown menu to the right of any Chameleon supported appliance to view the relevant entry from the `Chameleon Appliance Catalog `_. - -.. note:: Images at each site are stored independently. An Image made at |CHI@TACC| **will not** be available at |CHI@UC| (or vice versa) unless transferred manually. - -Uploading an Image -__________________ - -Use *+ Create Image* button to upload an image. - -.. figure:: images/createimage.png - :alt: THe Create Image dialog - - The Create Image dialog - -In the *Create Image* dialog: - -#. Enter an *Image Name* and, optionally, a description. -#. Click *Browse* to select a file on your local machine to upload. -#. Select a *Format* of the image. Images created by the ``cc-snapshot`` utility are *QCOW2* images. -#. To add additional metadata for your image, use the *Metadata* section by clicking *Metadata* in the sidebar. -#. Click the *Create Image* button to upload your image. - -Launching Instance using an Image -__________________________________ - -During the process of :ref:`launching instance ` from the *Instance* page, it will ask you to select an image. Alternatively, you can launch instances with a selected image from the *Image* page by simply clicking on the *Launch* button located in the same row of the targeted image. - -.. tip:: Other than *Launch*, there are other actions you may perfom on the image. Clicking on the dropdown to explore more on what you can do. - -Viewing Image Details -_____________________ - -To view image details, click on the name of the Image. - -.. figure:: images/imagedetails.png - :alt: Image details - - Image details - -The dropdown list in the top right corner allows you to perform various actions on the selected image, such as *Launch*, *Edit Image*, and *Update Metadata*. - -.. tip:: The *ID* on the image details' page is useful when you work on the image using the CLI. - -.. _simple-publish: - -Publishing Images to the Appliance Catalog -__________________________________________ - -.. figure:: images/publishappliance.png - :alt: Publish to Appliance Catalog - -The dropdown menu to the right of listed images allows their owners to publish an appliance to the `Appliance Catalog `_. Select *Publish to Appliance Catalog*. - -The *Create Appliances* web form will open automatically with most fields pre-populated. Complete the form and select *Create an Appliance*. - -Entering a descriptive name, author and support contact information, the version, and an informative description can be helpful and is encouraged. **The description is used by others to determine if an appliance contains the tools needed for their research.** - -.. tip:: To make your description effective you may want to ask the following questions: - - - What does the appliance contain? - - - What are the specific packages and their versions? - - - What is it useful for? - - - Where can it be deployed and/or what restrictions/limitations does it have? - - - How should users connect to it and what accounts are enabled? - -________________________________________________ -Managing Images using the CLI -________________________________________________ - -.. tip:: Reading :ref:`cli` is highly recommanded before continuing on the following sections. - -Uploading an Image -__________________ - -After configuring the environment variables using :ref:`cli-rc-script`, run the following command: - -.. code-block:: bash - - openstack image create --file --disk-format - -Provide the path to and the name of your image file in your local file system as the value of the ``file`` parameter. Also, indicate the image format using the ``format`` switch, such as ``QCOW2``. Finally, name your image via the ``image-name`` switch. - -Downloading an Image -____________________ - -Downloading an image file to your local machine is **only** available via the CLI. You may find it useful when transferring images from one Chameleon site to another. To download an image file, run the following command: - -.. code-block:: bash - - openstack image save --file - -Use ``filename`` to indicate where you would like to save the image in your local file system. Also, replace ``image`` with either the name or the *ID* of the image on Chameleon. - -.. important:: - If you do not provide the ``--file`` parameter, it will print out the binary image data in your terminal. - -Retrieving Images -___________________________ - -You may list all images of your project by typing: - -.. code-block:: bash - - openstack image list - -Optionally, you may add filters to the list, such as ``--shared`` to only display the images shared within your project. Use ``openstack image list --help`` to see all the available filters. - -Viewing Image Details -_____________________ - -You may view details of an image with the command: - -.. code-block:: bash - - openstack image show - -Replace ``image`` with either an image name or it's *UUID*. - -Sharing an Image -________________ - -You may share images several ways. If you wish to share an image with everyone, use: - -.. code-block:: bash - - openstack image set --public - -Replace ``image`` with the image *UUID*. - -If you would like to share an image with another project, first set the image visibility to shared: - -.. code-block:: bash - - openstack image set --shared - -Next add the project you wish to share the image with: - -.. code-block:: bash - - openstack image add project - -Replace ``image`` and ``project`` with the corresponding *UUIDs* - -Finally the project that the image is shared to must accept the shared image. Run this command with a user in the second project: - -.. code-block:: bash - - openstack image set --accept - -Replace ``image`` with the image *UUID* and the second project should now be able to use the image! - -.. important:: - Only the owner of the image can modify it or any properties. However a project who has an image shared to it can remove themselves from the list of image members. - -Editing an Image -________________ - -You may edit an image using the command: - -.. code-block:: bash - - openstack image set ... - -Replace ``image`` with either an image name or it's *UUID*. You must provide additional flags to update an image. Use ``openstack image set --help`` to see all the options. +If you are not redirected automatically, please visit :doc:`images/index`. \ No newline at end of file diff --git a/source/technical/images/cc_snapshot.rst b/source/technical/images/cc_snapshot.rst new file mode 100644 index 0000000..644ec54 --- /dev/null +++ b/source/technical/images/cc_snapshot.rst @@ -0,0 +1,65 @@ +.. _cc-snapshot-utility: + +=========================== +The ``cc-snapshot`` Utility +=========================== + +The ``cc-snapshot`` utility implements snapshotting a bare metal instance from command line and uploads it to `Glance `_, so that it can be immediately used to boot a new bare metal instance. The snapshot images created with this tool are whole disk images. + +For ease of use, ``cc-snapshot`` has been installed in all the appliances supported by the Chameleon project. If you would like to use it in a different setting, it can be downloaded and installed from the `github repository `_. + +To make a snapshot of a bare metal instance, run the following command from inside the instance: + +.. code-block:: bash + + sudo cc-snapshot + +.. tip:: + You may get warnings, such as "image too large", during snapshotting, and get prompted to confirm. If you are confident about what you are trying to do, you can skip all warnings by using the ``-f`` flag. + + .. code-block:: bash + + sudo cc-snapshot -f + + In addition, you can exclude directories by using the ``-e`` flag. + + .. code-block:: bash + + sudo cc-snapshot -e -e + + To see all available options for ``cc-snapshot``, run ``sudo cc-snapshot -h``. + +You will be prompted to enter your username and password. + +.. tip:: You can skip entering username and password by setting the ``OS_USERNAME`` and ``OS_PASSWORD`` environment variables. You can set those environment variables manually or using :ref:`cli-rc-script`. + +.. note:: When using the ``cc-snapshot``, it will create an image within your project with the ``shared`` visibility. Anyone with access to your project can access this image. + +.. note:: If you choose an *Image* name that already exists, the previous one **will not** be overwritten. A new *Image* with the same name but a different *UUID* will be generated. + +.. note:: If you install a custom kernel, make sure the size of your running kernel (``/lib/modules/``) is less than 4GB. To find out which kernel version you're running, run ``uname -r``. + +.. _updating-snapshot: + +Updating cc-snapshot +-------------------- + +.. error:: + If you receive the following error: + + .. code:: + + public endpoint for image service in regionOne not found Unable to contact Glance, check username and password + + it means that you have an outdated copy of ``cc-snapshot`` and you will need to update ``cc-snapshot``. + This usually happens when you use an older images that contains an outdated version of ``cc-snapshot``. + + You may also want to get new functionalities added to the latest version of ``cc-snapshot``. + + Run the following commands from your instance: + + .. code:: + + curl -O https://raw.githubusercontent.com/ChameleonCloud/cc-snapshot/master/cc-snapshot + sudo mv cc-snapshot /usr/bin/ + sudo chmod +x /usr/bin/cc-snapshot \ No newline at end of file diff --git a/source/technical/images/cli_management.rst b/source/technical/images/cli_management.rst new file mode 100644 index 0000000..fdcaebe --- /dev/null +++ b/source/technical/images/cli_management.rst @@ -0,0 +1,101 @@ +.. _images-cli-management: + +============================= +Managing Images using the CLI +============================= + +.. tip:: Reading :ref:`cli` is highly recommended before continuing on the following sections. + +Uploading an Image +================== + +After configuring the environment variables using :ref:`cli-rc-script`, run the following command: + +.. code-block:: bash + + openstack image create --file --disk-format + +Provide the path to and the name of your image file in your local file system as the value of the ``file`` parameter. Also, indicate the image format using the ``format`` switch, such as ``QCOW2``. Finally, name your image via the ``image-name`` switch. + +Downloading an Image +==================== + +Downloading an image file to your local machine is **only** available via the CLI. You may find it useful when transferring images from one Chameleon site to another. To download an image file, run the following command: + +.. code-block:: bash + + openstack image save --file + +Use ``filename`` to indicate where you would like to save the image in your local file system. Also, replace ``image`` with either the name or the *ID* of the image on Chameleon. + +.. important:: + If you do not provide the ``--file`` parameter, it will print out the binary image data in your terminal. + +Retrieving Images +================= + +You may list all images of your project by typing: + +.. code-block:: bash + + openstack image list + +Optionally, you may add filters to the list, such as ``--shared`` to only display the images shared within your project. Use ``openstack image list --help`` to see all the available filters. + +Viewing Image Details +===================== + +You may view details of an image with the command: + +.. code-block:: bash + + openstack image show + +Replace ``image`` with either an image name or it's *UUID*. + +Sharing an Image +================ + +You may share images several ways. If you wish to share an image with everyone, use: + +.. code-block:: bash + + openstack image set --public + +Replace ``image`` with the image *UUID*. + +If you would like to share an image with another project, first set the image visibility to shared: + +.. code-block:: bash + + openstack image set --shared + +Next add the project you wish to share the image with: + +.. code-block:: bash + + openstack image add project + +Replace ``image`` and ``project`` with the corresponding *UUIDs* + +Finally the project that the image is shared to must accept the shared image. Run this command with a user in the second project: + +.. code-block:: bash + + openstack image set --accept + +Replace ``image`` with the image *UUID* and the second project should now be able to use the image! + +.. important:: + Only the owner of the image can modify it or any properties. However a project who has an image shared to it can remove themselves from the list of image members. + +Editing an Image +================ + +You may edit an image using the command: + +.. code-block:: bash + + openstack image set ... + +Replace ``image`` with either an image name or it's *UUID*. You must provide additional flags to update an image. Use ``openstack image set --help`` to see all the options. \ No newline at end of file diff --git a/source/technical/images/gui_management.rst b/source/technical/images/gui_management.rst new file mode 100644 index 0000000..a3017c5 --- /dev/null +++ b/source/technical/images/gui_management.rst @@ -0,0 +1,83 @@ +.. _images-gui-management: + +============================= +Managing Images using the GUI +============================= + +To manage your images, use the *Images* page at |CHI@TACC| or |CHI@UC|, by clicking on *Project* > *Compute* > *Images*. + +.. figure:: imagespagev3.png + :alt: The Images page + + The Images page + +.. note:: The Chameleon logo next to an image's name indicates that this image is an appliance supported by the Chameleon project, and is part of the Appliance Catalog. + +.. tip:: Select *Details* from the dropdown menu to the right of any Chameleon supported appliance to view the relevant entry from the `Chameleon Appliance Catalog `_. + +.. note:: Images at each site are stored independently. An Image made at |CHI@TACC| **will not** be available at |CHI@UC| (or vice versa) unless transferred manually. + +Uploading an Image +================== + +Use *+ Create Image* button to upload an image. + +.. figure:: createimage.png + :alt: The Create Image dialog + + The Create Image dialog + +In the *Create Image* dialog: + +#. Enter an *Image Name* and, optionally, a description. +#. Click *Browse* to select a file on your local machine to upload. +#. Select a *Format* of the image. Images created by the ``cc-snapshot`` utility are *QCOW2* images. +#. To add additional metadata for your image, use the *Metadata* section by clicking *Metadata* in the sidebar. +#. Click the *Create Image* button to upload your image. + +Launching Instance using an Image +================================= + +During the process of :ref:`launching instance ` from the *Instance* page, it will ask you to select an image. Alternatively, you can launch instances with a selected image from the *Image* page by simply clicking on the *Launch* button located in the same row of the targeted image. + +.. tip:: Other than *Launch*, there are other actions you may perform on the image. Clicking on the dropdown to explore more on what you can do. + +Viewing Image Details +===================== + +To view image details, click on the name of the Image. + +.. figure:: imagedetails.png + :alt: Image details + + Image details + +The dropdown list in the top right corner allows you to perform various actions on the selected image, such as *Launch*, *Edit Image*, and *Update Metadata*. + +.. tip:: The *ID* on the image details' page is useful when you work on the image using the CLI. + +.. _simple-publish: + +Publishing Images to the Appliance Catalog +========================================== + +.. figure:: publishappliance.png + :alt: Publish to Appliance Catalog + +The dropdown menu to the right of listed images allows their owners to publish an appliance to the `Appliance Catalog `_. Select *Publish to Appliance Catalog*. + +The *Create Appliances* web form will open automatically with most fields pre-populated. Complete the form and select *Create an Appliance*. + +Entering a descriptive name, author and support contact information, the version, and an informative description can be helpful and is encouraged. **The description is used by others to determine if an appliance contains the tools needed for their research.** + +.. tip:: To make your description effective you may want to ask the following questions: + + - What does the appliance contain? + + - What are the specific packages and their versions? + + - What is it useful for? + + - Where can it be deployed and/or what restrictions/limitations does it have? + + - How should users connect to it and what accounts are enabled? \ No newline at end of file diff --git a/source/technical/images/index.rst b/source/technical/images/index.rst new file mode 100644 index 0000000..2935aa7 --- /dev/null +++ b/source/technical/images/index.rst @@ -0,0 +1,20 @@ +.. _images: + +==================== +Images +==================== + +All instances in Chameleon, whether KVM or bare metal, are running off disk images. The content of these disk images can be snapshotted at any point in time, which allows you to save your work and launch new instances from updated images later. While OpenStack KVM has built-in support for snapshotting in the Horizon web interface and via the command line, bare metal instances require a more complex process. + +To work around this limitation, we provide the ``cc-snapshot`` utility that you can execute from inside your running instance. The ``cc-snapshot`` utility is pre-installed in all Chameleon supported appliances. You can find our appliances from the `Appliance Catalog `_. + +The image service on Chameleon uses `OpenStack Glance `_. This documentation demonstrates how to accomplish common tasks with *Images* using the GUI and the CLI. + +.. toctree:: + :maxdepth: 2 + :caption: Images Topics + + supported_images + cc_snapshot + gui_management + cli_management \ No newline at end of file diff --git a/source/technical/images/supported_images.rst b/source/technical/images/supported_images.rst new file mode 100644 index 0000000..2ea3cf7 --- /dev/null +++ b/source/technical/images/supported_images.rst @@ -0,0 +1,41 @@ +.. _chameleon-supported-images: + +========================== +Chameleon Supported Images +========================== + +There are a number of images built and supported by the Chameleon team, +specifically: + +- CC-Ubuntu24.04 +- CC-Ubuntu24.04-CUDA +- CC-Ubuntu24.04-ROCm +- CC-Ubuntu24.04-ARM64 +- CC-Ubuntu22.04 +- CC-Ubuntu22.04-CUDA +- CC-Ubuntu22.04-ARM64 +- CC-Centos9-Stream + +The CUDA images, such as `Ubuntu24.04-CUDA `_, +contain various settings, software, and drivers specifically +for NVIDIA GPU nodes. The ROCm images contain similar settings, software, +and drivers for AMD GPU nodes. And finally, the ARM64 images, such as +`Ubuntu22.04-ARM64 `_, +are images specifically built with ARM support for ARM nodes. Non-ARM +images all assume x86-based architectures. + +.. warning:: + Any images with operating system versions that are end-of-life, such as + Ubuntu18.04 are no longer offered as Chameleon-supported images. However, + they can still be used on Chameleon with caution. Please note that these + images are EOL so they no longer receive security updates and bug fixes. + They may stop working at any point, therefore, make sure to upgrade and + move your environments to newer versions of the operating system as soon as + possible. If you do need to launch older images on Chameleon, please consider + using a `bastion host `_ + that is up-to-date and provides SSH access. From there, you can jump to your + instances running older images that are not exposed directly on the Internet. + +You may also build your own images and share them with the community. These images +may be built from scratch or based on the Chameleon-supported images. However, +the Chameleon team cannot offer support for user-provided images. \ No newline at end of file diff --git a/source/technical/index.rst b/source/technical/index.rst index c123d05..d014740 100644 --- a/source/technical/index.rst +++ b/source/technical/index.rst @@ -2,23 +2,27 @@ Overview ========= -This section provides in-depth knowledge for utilizing Chameleon's advanced -features. +The following sections contain in-depth knowledge for utilizing Chameleon's +advanced features. -- :doc:`discovery`: Discover Chameleon bare metal resources by node type and +- :doc:`discovery/index`: Discover Chameleon bare metal resources by node type and view node information. -- :doc:`reservations`: Reserve Chameleon resources for use in your Project. -- :doc:`baremetal`: Launch and manage Instances on Chameleon bare metal +- :doc:`reservations/index`: Reserve Chameleon resources for use in your Project. +- :doc:`baremetal/index`: Launch and manage Instances on Chameleon bare metal resources. This is a core feature of Chameleon. -- :doc:`images`: Create images of Instances. -- :doc:`metrics`: Collect, manage and view experimental data from Chameleon - Instances. -- :doc:`complex`: Work with Complex Appliances, which automate the process of +- :doc:`images/index`: Create images of Instances. +- :doc:`power_monitoring/index`: Monitor power consumption and energy usage of your + experiments. +- :doc:`complex/index`: Work with Complex Appliances, which automate the process of deploying multiple Instances with reconfigurable networking. -- :doc:`swift`: Store user data such as files as Objects in portable Containers. -- :doc:`shares`: Provide file storage to an instance with the OpenStack Shared +- :doc:`swift/index`: Store user data such as files as Objects in portable Containers. +- :doc:`shares/index`: Provide file storage to an instance with the OpenStack Shared File System service. -- :doc:`networks`: Create Isolated virtual networks within Chameleon. -- :doc:`fpga`: Configure and work with FPGA nodes. -- :doc:`kvm`: Use non-bare metal virtual machine resources in Chameleon's +- :doc:`networks/index`: Create Isolated virtual networks within Chameleon. +- :doc:`fpga/index`: Configure and work with FPGA nodes. +- :doc:`sharing/index`: Share reproducible experiments and artifacts using the Trovi + Sharing Portal. +- :doc:`daypass/index`: Enable temporary access to your experiments for collaborators + or reviewers. +- :doc:`kvm/index`: Use non-bare metal virtual machine resources in Chameleon's OpenStack implementation. diff --git a/source/technical/jupyter.rst b/source/technical/jupyter.rst index 6a7378c..6b87349 100644 --- a/source/technical/jupyter.rst +++ b/source/technical/jupyter.rst @@ -1,100 +1,11 @@ -.. |python_chi| replace:: ``python-chi`` -.. _python_chi: https://github.com/chameleoncloud/python-chi +:orphan: -.. _jupyter: +.. meta:: + :http-equiv=refresh: 0; url=./jupyter/index.html -================= -Jupyter interface -================= +This page has moved +=================== -Jupyter Notebooks are an excellent tool for prototyping, exploring, and -ultimately documenting the entire experimental process. They combine the -benefits of explanatory text, executable code, and rich -visualization/interaction. +This page has been reorganized. You will be redirected to the new location. -Chameleon users can get a Jupyter Notebook server automatically provisioned for -them by logging in to the `JupyterHub server -`_ managed by Chameleon. Upon login, you -will be redirected to your Jupyter Notebook server. If there is not yet a -Notebook server allocated for your user, one will be created behind the scenes. -This can take a few moments. - -.. warning:: - The shared Jupyter environment places resource limits on your Jupyter server, - notably limiting it to 1 CPU core and 1GB of memory. If you are doing - computationally or memory-intensive work in a Notebook, it may be beneficial - to look in to :ref:`provisioning your own dedicated JupyterHub - `. - -.. _jupyterlab: - -JupyterLab interface overview ------------------------------ - -When you are logged in, you will land in the JupyterLab application environment. -For up-to-date documentation about the JupyterLab interface, please see the -`official JupyterLab documentation -`_. You will -see a file browser on the left-hand side - this is your working directory. It's -yours, so feel free to create and delete files as you see fit. Your working -directory is initially populated with a few examples to help you get started, -such as an example Notebook. Files that you save here will be persisted even if -your server is torn down; the next time you log in the data will be restored. -You should consider the rest of your server environment ephemeral, as updates to -the Jupyter interface can cause your server to be re-built. - -.. hint:: - Jupyter Notebooks do not deal well with large files, and you should avoid - trying to edit large files in the interface as it can cause instability, - slowness, or even crashes. If you need to deal with large files it is best to - process them on a :ref:`dedicated processing node `, such - as a baremetal node provisioned as part of your experiment. - -.. figure:: jupyter/landing.png - :alt: The JupyterLab interface - :figclass: screenshot - -.. _notebooks: - -Working with Notebooks ----------------------- - -Open the "Welcome.ipynb" Notebook to see some examples of how to interface with -the Chameleon testbed from within a Notebook. All Notebook servers come with -OpenStack python clients installed as well as the |python_chi|_ Chameleon -testbed helper library. Other python modules you may want to use in your -Notebook can be installed via the :ref:`console`. - -.. figure:: jupyter/notebook.png - :alt: An example Jupyter Notebook in JupyterLab - :figclass: screenshot - -.. _console: - -Console interface ------------------ - -You can open a web terminal console by going to File > New > Terminal. This -works just like a remote shell, and you will also have `sudo` access so you can -install additional software to support your needs. - -.. hint:: - All Chameleon Notebook servers are built from a common base image. This means - if your server is torn down (which can happen during an upgrade of the - Jupyter server), you may have to re-do any changes to the underlying system - you made since the server was created. For this reason it is a good idea to - put this setup code in a script in your working directory. Your working - directory is backed up and will persist across Jupyter server restarts. - -.. figure:: jupyter/console.png - :alt: The web terminal console in JupyterLab - :figclass: screenshot - -Advanced topics ---------------- - -.. toctree:: - :maxdepth: 1 - - jupyter/jupyter_dedicated - jupyter/jupyter_collaboration +If you are not redirected automatically, please visit :doc:`jupyter/index`. \ No newline at end of file diff --git a/source/technical/jupyter/index.rst b/source/technical/jupyter/index.rst new file mode 100644 index 0000000..25bed2e --- /dev/null +++ b/source/technical/jupyter/index.rst @@ -0,0 +1,100 @@ +.. |python_chi| replace:: ``python-chi`` +.. _python_chi: https://github.com/chameleoncloud/python-chi + +.. _jupyter: + +================= +Jupyter Interface +================= + +Jupyter Notebooks are an excellent tool for prototyping, exploring, and +ultimately documenting the entire experimental process. They combine the +benefits of explanatory text, executable code, and rich +visualization/interaction. + +Chameleon users can get a Jupyter Notebook server automatically provisioned for +them by logging in to the `JupyterHub server +`_ managed by Chameleon. Upon login, you +will be redirected to your Jupyter Notebook server. If there is not yet a +Notebook server allocated for your user, one will be created behind the scenes. +This can take a few moments. + +.. warning:: + The shared Jupyter environment places resource limits on your Jupyter server, + notably limiting it to 1 CPU core and 1GB of memory. If you are doing + computationally or memory-intensive work in a Notebook, it may be beneficial + to look in to :ref:`provisioning your own dedicated JupyterHub + `. + +.. _jupyterlab: + +JupyterLab interface overview +----------------------------- + +When you are logged in, you will land in the JupyterLab application environment. +For up-to-date documentation about the JupyterLab interface, see the +`official JupyterLab documentation +`_. You will +see a file browser on the left-hand side - this is your working directory. It's +yours, so feel free to create and delete files as you see fit. Your working +directory is initially populated with a few examples to help you get started, +such as an example Notebook. Files that you save here will be persisted even if +your server is torn down; the next time you log in the data will be restored. +You should consider the rest of your server environment ephemeral, as updates to +the Jupyter interface can cause your server to be re-built. + +.. hint:: + Jupyter Notebooks do not deal well with large files, and you should avoid + trying to edit large files in the interface as it can cause instability, + slowness, or even crashes. If you need to deal with large files it is best to + process them on a :ref:`dedicated processing node `, such + as a baremetal node provisioned as part of your experiment. + +.. figure:: landing.png + :alt: The JupyterLab interface + :figclass: screenshot + +.. _notebooks: + +Working with Notebooks +---------------------- + +Open the "Welcome.ipynb" Notebook to see some examples of how to interface with +the Chameleon testbed from within a Notebook. All Notebook servers come with +OpenStack python clients installed as well as the |python_chi|_ Chameleon +testbed helper library. Other python modules you may want to use in your +Notebook can be installed via the :ref:`console`. + +.. figure:: notebook.png + :alt: An example Jupyter Notebook in JupyterLab + :figclass: screenshot + +.. _console: + +Console interface +----------------- + +You can open a web terminal console by going to File > New > Terminal. This +works just like a remote shell, and you will also have `sudo` access so you can +install additional software to support your needs. + +.. hint:: + All Chameleon Notebook servers are built from a common base image. This means + if your server is torn down (which can happen during an upgrade of the + Jupyter server), you may have to re-do any changes to the underlying system + you made since the server was created. For this reason it is a good idea to + put this setup code in a script in your working directory. Your working + directory is backed up and will persist across Jupyter server restarts. + +.. figure:: console.png + :alt: The web terminal console in JupyterLab + :figclass: screenshot + +Advanced topics +--------------- + +.. toctree:: + :maxdepth: 1 + + jupyter_dedicated + jupyter_collaboration diff --git a/source/technical/kvm.rst b/source/technical/kvm.rst index 913b1e9..1a986bb 100644 --- a/source/technical/kvm.rst +++ b/source/technical/kvm.rst @@ -1,304 +1,11 @@ -.. _kvm: +:orphan: -KVM -=== +.. meta:: + :http-equiv=refresh: 0; url=./kvm/index.html -Introduction ------------- +This page has moved +=================== -OpenStack is an Infrastructure as a Service (IaaS) platform that allows you to -create and manage virtual environments. Chameleon provides an installation of -OpenStack `Xena `_ using the -KVM virtualization technology at the `KVM\@TACC -`_ site. Since the KVM hypervisor is used -on this cloud, any virtual machines you upload must be compatible with KVM. +This page has been reorganized. You will be redirected to the new location. -This documentation provide basic information about how to use the OpenStack web -interface and provides some information specific to using OpenStack KVM on -Chameleon. The interface is similar to the bare metal sites |CHI@TACC| and -|CHI@UC|. However, the resources that you are using are virtual, rather than -being tied to physical nodes. Familiarity with some concepts, such as -:ref:`gui-key-pairs` are also required for KVM. - -Work with KVM using the GUI ---------------------------- - -An easy way to use OpenStack KVM on Chameleon is via the GUI, which is similar -to the GUIs for |CHI@TACC| and |CHI@UC|. You log into the web interface using -your Chameleon username and password. - -After a successful log in, you will see the *Overview* page as shown below. This -page provides a summary of your current and recent usage and provides links to -various other pages. Most of the tasks you will perform are done via the menu on -the lower left and will be described below. One thing to note is that on the -left, your current project is displayed. If you have multiple Chameleon -projects, you can change which of them is your current project. All of the -information displayed and actions that you take apply to your current project. -So in the screen shot below, the quota and usage apply to the current project -you have selected and no information about your other projects is shown. - -.. figure:: kvm/new_overview.png - -Managing Virtual Machine Instances -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -One of the main activities you’ll be performing in the GUI is management of -virtual machines, or instances. Go to *Project* > *Compute* > *Instances* in the -navigation sidebar. For instances that you have running, you can click on the -name of the instance to get more information about it and to access the VNC -interface to the console. The dropdown menu to the right of the instance lets -you perform a variety of tasks such as suspending, terminating, or rebooting the -instance. - -.. figure:: kvm/new_instances.png - -Launching Instances -~~~~~~~~~~~~~~~~~~~ - -To launch an *Instance*, click the *Launch Instance* button. This will open the -*Launch Instance* dialog. - -.. figure:: kvm/new_launchdetails.png - -On the *Details* tab, provide a name for this instance (to help you identify -instances that you are running). - -Next, go to the *Source* tab to select media to launch. - -.. figure:: kvm/new_launchsource.png - -Select the *Boot Source* of the instance, which is either an *Image*, an -*Instance Snapshot* (an image created from a running virtual machine), a -*Volume* (a persistent virtual disk that can be attached to a virtual machine), -or a "Volume Snapshot". If you select "Image" as the *Boot Source*, the *Image -Name* dropdown presents a list of virtual machine images that we have provided, -that other Chameleon users have uploaded and made public, or images that you -have uploaded for yourself. If you select *Boot from snapshot*, the *Instance -Snapshot* dropdown presents a list of virtual machine images that you have -created from your running virtual machines. - -Go to the *Flavor* Tab and select the amount of resources (Flavor) to allocate -to the instance. - -.. figure:: kvm/new_launchflavor.png - -Flavors refer to the virtual machine's assigned memory and and disk size. -Different images and snapshots may require a larger Flavor. For example, the -``CC-CentOS7`` image requires at least an ``m1.small`` flavor. - - .. tip:: - If you select different flavors from the Flavor dropdown, their - characteristics are displayed on the right. - - If your VM needs differ from our defaults, you can request a custom - flavor for your project from the - `help desk `__ - -When you are finished with this step, go to the *Key Pair* Tab. - -.. figure:: kvm/new_launchaccess.png - -Select an SSH keypair that will be inserted into your virtual machine. You will -need to select a keypair here to be able to access an instance created from one -of the public images Chameleon provides. These images are not configured with a -default root password and you will not be able to log in to them without -configuring an SSH key. - -Then, go to the *Security Groups* Tab. - -.. figure:: kvm/new_secgroups.png - -If you have previously defined *Security Groups*, you may select them here. -Alternatively, you can configure them later. - -Set up network using *Network* tab. - -.. figure:: kvm/new_launchnetwork.png - -Select which network should be associated with the instance. Click the Up arrow -next to your project’s private network (PROJECT_NAME-net), not ``ext-net``. - -Now you can launch your instance by clicking on the *Launch* button and the -*Instances* page will show progress as it starts. - -.. _kvm-associate-ip: - -Associating a Floating IP Address -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -You may assign a Floating IP Address to your Instance by selecting *Associate -Floating IP* in the dropdown menu next to your Instance on the *Instances* page. - -.. figure:: kvm/new_associatemenu.png - -This process is similar to :ref:`baremetal-gui-associate-ip` on |CHI@TACC| and -|CHI@UC| bare metal sites. - -Key Pairs -~~~~~~~~~ - -You will need to import or create SSH :ref:`gui-key-pairs`. This process is -similar to the process performed on |CHI@TACC| and |CHI@UC| bare metal sites. - -.. _kvm-security-groups: - -Security Groups -~~~~~~~~~~~~~~~ - -*Security Groups* allow you to specify what inbound and outbound traffic is -allowed or blocked to Instances. Unlike the |CHI@TACC| and |CHI@UC| bare metal -sites, `KVM\@TACC `_ observes Security -Groups for Instances. - -.. note:: - By default, all inbound traffic is blocked to `KVM\@TACC - `_ Instances, including SSH. You must - apply a Security Group that allows TCP port 22 inbound to access your - instance via SSH. - -To create a Security Group, click *Projects* > *Network* > *Security Groups* in -the navigation side bar. - -.. figure:: kvm/new_securitytab.png - -Click the *+Create Security Group* button to open the *Create Security Group* -page. - -.. figure:: kvm/new_createsecurity.png - -Enter a *Name* for your *Security Group*, and optionally provide a -*Description*. Then click the *Create Security Group* button. Now, you should -see your *Security Group* listed on the *Access and Security* page. - -.. figure:: kvm/new_grouplist.png - -Click the *Manage Rules* button in the *Action* column to open the *Manage -Security Group Rules* page. - -.. figure:: kvm/new_managerules.png - -The default Security Group allows outbound IPv4 and IPv6 traffic for *Any IP -Protocol* and *Port Range*. If no entry for *Ingress*, no inbound traffic will -be allowed. You may add an additional rule by clicking on the *+Add Rule* to -open the *Add Rule* dialog. - -.. figure:: kvm/new_addrule.png - -In this dialog, you can specify *Custom TCP Rule* (or *Custom UDP Rule* or -*Custom ICMP Rule*), a *Direction* (*Ingress* for inbound traffic to your -Instance or *Egress* for outbound traffic) and a *Port*. Alternatively, you can -use a pre-defined rule in the *Rule* dropdown, such as *SSH*. when you are -finished, click *Add*. - -.. _kvm-security-group: - -Adding a Security Group to an Instance -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Once you have defined a *Security Group*, you may apply it to an Instance by -clicking *Project* > *Compute* > *Instances* in the navigation sidebar and -clicking the *Edit Security Groups* option in the *Actions* dropdown. - -.. figure:: kvm/new_editaction.png - -The *Security Groups* tab in the *Edit Instance* dialog will pop up. - -.. figure:: kvm/new_editinstance.png - -You may click the *+* button next to the Security Group you wish to apply in the -*All Security Groups* list on the left. Once you are finished, click *Save* to -finish the process. - -Load Balancer as a Service -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Available on KVM@TACC is the OpenStack Octavia Load Balancer as a Service (LBaas). This service allows a single IP address to be used to distribute connections among a number of virtual machine instances. -For the following description, it is assumed that there are already several virtual machines running an HTTP server on port 80, serving a page at the root path. -To create a *Load Balancer*, click on *Project* > *Network* > *Load Balancers* in the navigation sidebar, then the *Create Load Balancer* button. This will open the *Create Load Balancer* dialog. - -.. figure:: kvm/lbaas_create_loadbalancer.png - -Give your load balancer a name, and select the subnet that corresponds to the one used by the virtual machines. Click *Next*, or *Listener Details*. - -.. figure:: kvm/lbaas_listener_details.png - -The listener is the port that will accept incoming connetions. Select the appropriate protocol for the service, in this case *HTTP*. If selecting *TCP* or *UDP* also provide the desired port. Click *Next* or *Pool Details*. - -.. figure:: kvm/lbaas_pool_details.png - -Choose the desired load balancing algorithm. This will determine the way in which the load balancer will select which VM receives incoming requests. Click *Next* or *Pool Members*. - -.. figure:: kvm/lbaas_pool_members.png - -Here you will select the virtual machines that will participate in the load balacing. Click the *Add* button next to the instances, after which their IP address and subnet will be added to the *Allocated Members* list at the top. -You will need to provide the port number for the hosted service for each member. For our HTTP servers, it is port 80. This does not need to match the port of the load balancer's *listener*. - -.. figure:: kvm/lbaas_pool_member_add.png - -Once you've selected the pool members, click *Next* or *Monitor Details*. Here you will configure how the load balancer monitors the servies on the virtual machines to ensure that they are ready to receive traffic. -In our example, selecting *HTTP* adds configuration options for *HTTP Method*, *Expected Codes*, and *URL path*. Since the HTTP services on the VMs in the *pool members* are configured to serve a page on the root path, the default values will work. -Click *Create Load Balancer* - -.. figure:: kvm/lbaas_monitor_http.png - -While the load balancer is being created, the dashboard will show a *Provisioning Status* of *Pending Create* . Once the process is complete, the status should be *Active*, and *Operating Status* should be *Online*. -An *Operating Status* of "*Offline*" or "*Error*" indicates that the load balancer cannot satisfy the service check specified in *Monitor Details*. Ensure that the services are running on each VM, and that they return the expected status. - -.. figure:: kvm/lbaas_create_pending.png - -.. figure:: kvm/lbaas_active.png - -You can assign a Floating IP address to the load balancer by clicking on the down arrow button next to *Edit Load Balancer*, and selecting *Associate Floating IP*. This process is similar to associatig af Floating IP to a virtual machine instnace. -Making changes to the various components of the load balancer by clicking on the blue-colord name of the load balancer in the list. From here, the *listeners*, *pools*, and *health monitors* can be updated, if needed. - -To learn more about how to use the Octavia Load Balancer, refer to the `Basic Load Balancing Cookbook `_ on the official OpenStack documentation - -Work with KVM using the CLI ---------------------------- - -For general information on CLI authentication and use, please see the -`command-line-interface section -`_. - -Uploading qcow2 images to raw format for better instance launch performance -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -KVM images are stored on our Ceph cluster, which is able to serve raw images -much faster than qcow2 for instance launches. Openstack includes the -experimental command Glance image-create-via-import, which allows uploading of -images in various standard formats including qcow2 to then be automatically -converted to raw in the backend. - -In order to use this method, authenticate to KVM using the OpenStack RC script -downloaded from the `KVM\@TACC `_ site as -described in :ref:`cli-rc-script`. - -Next, issue the following command: - - .. code-block:: shell - - glance image-create-via-import --container-format bare --disk-format qcow2 --file --name - -Details and other options for this command are available via the Glance -`image-create-via-import documentation -`_. - -.. attention:: - Glance image-create-via-import is currently unable to handle conversion of - iso images to raw. - -Alternatively, you may convert qcow2 images to raw format before upload. -qemu-img is one tool that is able to this with the following command: - - .. code-block:: shell - - qemu-img convert -f qcow2 -O raw - -Once converted, use glance to upload the image: - - .. code-block:: shell - - openstack image create --file --disk-format raw - -Details and other options for this command are available within `Openstack -documentation `_. +If you are not redirected automatically, please visit :doc:`kvm/index`. \ No newline at end of file diff --git a/source/technical/kvm/index.rst b/source/technical/kvm/index.rst new file mode 100644 index 0000000..7f55eae --- /dev/null +++ b/source/technical/kvm/index.rst @@ -0,0 +1,27 @@ +.. _kvm: + +=== +KVM +=== + +OpenStack is an Infrastructure as a Service (IaaS) platform that allows you to +create and manage virtual environments. Chameleon provides an installation of +OpenStack `Xena `_ using the +KVM virtualization technology at the `KVM\@TACC +`_ site. Since the KVM hypervisor is used +on this cloud, any virtual machines you upload must be compatible with KVM. + +This documentation provide basic information about how to use the OpenStack web +interface and provides some information specific to using OpenStack KVM on +Chameleon. The interface is similar to the bare metal sites |CHI@TACC| and +|CHI@UC|. However, the resources that you are using are virtual, rather than +being tied to physical nodes. Familiarity with some concepts, such as +:ref:`gui-key-pairs` are also required for KVM. + +.. toctree:: + :maxdepth: 2 + :caption: KVM Topics + + kvm_gui + kvm_lbaas + kvm_cli \ No newline at end of file diff --git a/source/technical/kvm/kvm_cli.rst b/source/technical/kvm/kvm_cli.rst new file mode 100644 index 0000000..e84593a --- /dev/null +++ b/source/technical/kvm/kvm_cli.rst @@ -0,0 +1,49 @@ +Work with KVM using the CLI +=========================== + +For general information on CLI authentication and use, see the +`command-line-interface section +`_. + +Uploading qcow2 images to raw format for better instance launch performance +--------------------------------------------------------------------------- + +KVM images are stored on our Ceph cluster, which is able to serve raw images +much faster than qcow2 for instance launches. Openstack includes the +experimental command Glance image-create-via-import, which allows uploading of +images in various standard formats including qcow2 to then be automatically +converted to raw in the backend. + +In order to use this method, authenticate to KVM using the OpenStack RC script +downloaded from the `KVM\@TACC `_ site as +described in :ref:`cli-rc-script`. + +Next, issue the following command: + + .. code-block:: shell + + glance image-create-via-import --container-format bare --disk-format qcow2 --file --name + +Details and other options for this command are available via the Glance +`image-create-via-import documentation +`_. + +.. attention:: + Glance image-create-via-import is currently unable to handle conversion of + iso images to raw. + +Alternatively, you may convert qcow2 images to raw format before upload. +qemu-img is one tool that is able to this with the following command: + + .. code-block:: shell + + qemu-img convert -f qcow2 -O raw + +Once converted, use glance to upload the image: + + .. code-block:: shell + + openstack image create --file --disk-format raw + +Details and other options for this command are available within `Openstack +documentation `_. \ No newline at end of file diff --git a/source/technical/kvm/kvm_gui.rst b/source/technical/kvm/kvm_gui.rst new file mode 100644 index 0000000..afb44f7 --- /dev/null +++ b/source/technical/kvm/kvm_gui.rst @@ -0,0 +1,188 @@ +Work with KVM using the GUI +=========================== + +An easy way to use OpenStack KVM on Chameleon is via the GUI, which is similar +to the GUIs for |CHI@TACC| and |CHI@UC|. You log into the GUI using +your Chameleon username and password. + +After a successful log in, you will see the *Overview* page as shown below. This +page provides a summary of your current and recent usage and provides links to +various other pages. Most of the tasks you will perform are done via the menu on +the lower left and will be described below. One thing to note is that on the +left, your current project is displayed. If you have multiple Chameleon +projects, you can change which of them is your current project. All of the +information displayed and actions that you take apply to your current project. +So in the screen shot below, the quota and usage apply to the current project +you have selected and no information about your other projects is shown. + +.. figure:: new_overview.png + +Managing Virtual Machine Instances +---------------------------------- + +One of the main activities you'll be performing in the GUI is management of +virtual machines, or instances. Go to *Project* > *Compute* > *Instances* in the +navigation sidebar. For instances that you have running, you can click on the +name of the instance to get more information about it and to access the VNC +interface to the console. The dropdown menu to the right of the instance lets +you perform a variety of tasks such as suspending, terminating, or rebooting the +instance. + +.. figure:: new_instances.png + +Launching Instances +------------------- + +To launch an *Instance*, click the *Launch Instance* button. This will open the +*Launch Instance* dialog. + +.. figure:: new_launchdetails.png + +On the *Details* tab, provide a name for this instance (to help you identify +instances that you are running). + +Next, go to the *Source* tab to select media to launch. + +.. figure:: new_launchsource.png + +Select the *Boot Source* of the instance, which is either an *Image*, an +*Instance Snapshot* (an image created from a running virtual machine), a +*Volume* (a persistent virtual disk that can be attached to a virtual machine), +or a "Volume Snapshot". If you select "Image" as the *Boot Source*, the *Image +Name* dropdown presents a list of virtual machine images that we have provided, +that other Chameleon users have uploaded and made public, or images that you +have uploaded for yourself. If you select *Boot from snapshot*, the *Instance +Snapshot* dropdown presents a list of virtual machine images that you have +created from your running virtual machines. + +Go to the *Flavor* Tab and select the amount of resources (Flavor) to allocate +to the instance. + +.. figure:: new_launchflavor.png + +Flavors refer to the virtual machine's assigned memory and and disk size. +Different images and snapshots may require a larger Flavor. For example, the +``CC-CentOS7`` image requires at least an ``m1.small`` flavor. + + .. tip:: + If you select different flavors from the Flavor dropdown, their + characteristics are displayed on the right. + + If your VM needs differ from our defaults, you can request a custom + flavor for your project from the + `help desk `__ + +When you are finished with this step, go to the *Key Pair* Tab. + +.. figure:: new_launchaccess.png + +Select an SSH key pair that will be inserted into your virtual machine. You will +need to select a key pair here to be able to access an instance created from one +of the public images Chameleon provides. These images are not configured with a +default root password and you will not be able to log in to them without +configuring an SSH key. + +Then, go to the *Security Groups* Tab. + +.. figure:: new_secgroups.png + +If you have previously defined *Security Groups*, you may select them here. +Alternatively, you can configure them later. + +Set up network using *Network* tab. + +.. figure:: new_launchnetwork.png + +Select which network should be associated with the instance. Click the Up arrow +next to your project's private network (PROJECT_NAME-net), not ``ext-net``. + +Now you can launch your instance by clicking on the *Launch* button and the +*Instances* page will show progress as it starts. + +.. _kvm-associate-ip: + +Associating a Floating IP Address +--------------------------------- + +You may assign a Floating IP Address to your Instance by selecting *Associate +Floating IP* in the dropdown menu next to your Instance on the *Instances* page. + +.. figure:: new_associatemenu.png + +This process is similar to :ref:`baremetal-gui-associate-ip` on |CHI@TACC| and +|CHI@UC| bare metal sites. + +Key Pairs +--------- + +You will need to import or create SSH :ref:`gui-key-pairs`. This process is +similar to the process performed on |CHI@TACC| and |CHI@UC| bare metal sites. + +.. _kvm-security-groups: + +Security Groups +--------------- + +*Security Groups* allow you to specify what inbound and outbound traffic is +allowed or blocked to Instances. Unlike the |CHI@TACC| and |CHI@UC| bare metal +sites, `KVM\@TACC `_ observes Security +Groups for Instances. + +.. note:: + By default, all inbound traffic is blocked to `KVM\@TACC + `_ Instances, including SSH. You must + apply a Security Group that allows TCP port 22 inbound to access your + instance via SSH. + +To create a Security Group, click *Projects* > *Network* > *Security Groups* in +the navigation side bar. + +.. figure:: new_securitytab.png + +Click the *+Create Security Group* button to open the *Create Security Group* +page. + +.. figure:: new_createsecurity.png + +Enter a *Name* for your *Security Group*, and optionally provide a +*Description*. Then click the *Create Security Group* button. Now, you should +see your *Security Group* listed on the *Access and Security* page. + +.. figure:: new_grouplist.png + +Click the *Manage Rules* button in the *Action* column to open the *Manage +Security Group Rules* page. + +.. figure:: new_managerules.png + +The default Security Group allows outbound IPv4 and IPv6 traffic for *Any IP +Protocol* and *Port Range*. If no entry for *Ingress*, no inbound traffic will +be allowed. You may add an additional rule by clicking on the *+Add Rule* to +open the *Add Rule* dialog. + +.. figure:: new_addrule.png + +In this dialog, you can specify *Custom TCP Rule* (or *Custom UDP Rule* or +*Custom ICMP Rule*), a *Direction* (*Ingress* for inbound traffic to your +Instance or *Egress* for outbound traffic) and a *Port*. Alternatively, you can +use a pre-defined rule in the *Rule* dropdown, such as *SSH*. when you are +finished, click *Add*. + +.. _kvm-security-group: + +Adding a Security Group to an Instance +-------------------------------------- + +Once you have defined a *Security Group*, you may apply it to an Instance by +clicking *Project* > *Compute* > *Instances* in the navigation sidebar and +clicking the *Edit Security Groups* option in the *Actions* dropdown. + +.. figure:: new_editaction.png + +The *Security Groups* tab in the *Edit Instance* dialog will pop up. + +.. figure:: new_editinstance.png + +You may click the *+* button next to the Security Group you wish to apply in the +*All Security Groups* list on the left. Once you are finished, click *Save* to +finish the process. \ No newline at end of file diff --git a/source/technical/kvm/kvm_lbaas.rst b/source/technical/kvm/kvm_lbaas.rst new file mode 100644 index 0000000..e81eb2d --- /dev/null +++ b/source/technical/kvm/kvm_lbaas.rst @@ -0,0 +1,43 @@ +Load Balancer as a Service +========================== + +Available on KVM@TACC is the OpenStack Octavia Load Balancer as a Service (LBaas). This service allows a single IP address to be used to distribute connections among a number of virtual machine instances. +For the following description, it is assumed that there are already several virtual machines running an HTTP server on port 80, serving a page at the root path. +To create a *Load Balancer*, click on *Project* > *Network* > *Load Balancers* in the navigation sidebar, then the *Create Load Balancer* button. This will open the *Create Load Balancer* dialog. + +.. figure:: lbaas_create_loadbalancer.png + +Give your load balancer a name, and select the subnet that corresponds to the one used by the virtual machines. Click *Next*, or *Listener Details*. + +.. figure:: lbaas_listener_details.png + +The listener is the port that will accept incoming connetions. Select the appropriate protocol for the service, in this case *HTTP*. If selecting *TCP* or *UDP* also provide the desired port. Click *Next* or *Pool Details*. + +.. figure:: lbaas_pool_details.png + +Choose the desired load balancing algorithm. This will determine the way in which the load balancer will select which VM receives incoming requests. Click *Next* or *Pool Members*. + +.. figure:: lbaas_pool_members.png + +Here you will select the virtual machines that will participate in the load balacing. Click the *Add* button next to the instances, after which their IP address and subnet will be added to the *Allocated Members* list at the top. +You will need to provide the port number for the hosted service for each member. For our HTTP servers, it is port 80. This does not need to match the port of the load balancer's *listener*. + +.. figure:: lbaas_pool_member_add.png + +Once you've selected the pool members, click *Next* or *Monitor Details*. Here you will configure how the load balancer monitors the servies on the virtual machines to ensure that they are ready to receive traffic. +In our example, selecting *HTTP* adds configuration options for *HTTP Method*, *Expected Codes*, and *URL path*. Since the HTTP services on the VMs in the *pool members* are configured to serve a page on the root path, the default values will work. +Click *Create Load Balancer* + +.. figure:: lbaas_monitor_http.png + +While the load balancer is being created, the dashboard will show a *Provisioning Status* of *Pending Create* . Once the process is complete, the status should be *Active*, and *Operating Status* should be *Online*. +An *Operating Status* of "*Offline*" or "*Error*" indicates that the load balancer cannot satisfy the service check specified in *Monitor Details*. Ensure that the services are running on each VM, and that they return the expected status. + +.. figure:: lbaas_create_pending.png + +.. figure:: lbaas_active.png + +You can assign a Floating IP address to the load balancer by clicking on the down arrow button next to *Edit Load Balancer*, and selecting *Associate Floating IP*. This process is similar to associatig af Floating IP to a virtual machine instnace. +Making changes to the various components of the load balancer by clicking on the blue-colord name of the load balancer in the list. From here, the *listeners*, *pools*, and *health monitors* can be updated, if needed. + +To learn more about how to use the Octavia Load Balancer, refer to the `Basic Load Balancing Cookbook `_ on the official OpenStack documentation \ No newline at end of file diff --git a/source/technical/metrics.rst b/source/technical/metrics.rst deleted file mode 100644 index 71c558b..0000000 --- a/source/technical/metrics.rst +++ /dev/null @@ -1,333 +0,0 @@ -.. _metrics: - -========================= -Monitoring -========================= - -.. warning:: - Gnocchi is not supported on the testbed. If you are interested in power monitoring, you should see `this trovi artifact `_ - -Chameleon collects monitoring information, representing qualities such as CPU load or power consumption data, from various sources into an *aggregation service*. Data is kept in this service with resolution that decreases over time. Users can retrieve those metrics via a *command line interface (CLI)*. - -In Chameleon, the aggregation service is implemented using the `Gnocchi time series database `_. All Chameleon supported images, from which most of our user’s images are derived, are configured to send a selection of system metrics using the `collectd system statistics collection daemon `_. There is a wide range of qualities this daemon can gather; by default only selected metrics are sent but users can configure the daemon (see `Configuring collectd`_) to adapt this set anytime to monitor their experiments better. Another source of metrics is the infrastructure itself, for example the energy and power consumption metrics. - -.. tip:: Reading :ref:`cli` is highly recommended before continuing on the following sections. - -Setting up the Gnocchi CLI -__________________________ - -In addition to :ref:`cli-installing`, you must also install the Gnocchi client plugin. To install on your local machine, run the following command: - -.. code-block:: bash - - pip install gnocchiclient - -Then, set up your environment for OpenStack command line usage, as described in :ref:`cli-rc-script`. - -.. _retrieve-metric: - -Retrieving Metrics -__________________ - -Now, you can run the ``gnocchi`` command line utility. To show the different kinds of metrics collected for a specific instance, run: - -.. code-block:: bash - - gnocchi resource show - -.. tip:: - You can get the instance' *ID* from the GUI. - - You can get your list of instances by running: - - .. code-block:: bash - - gnocchi resource list - - It will print out a chart similar to below: - - .. code:: - - +--------------------------------------+---------+------------+ - | id | type | project_id | - +--------------------------------------+---------+------------+ - | 8d643431-9a90-4100-8e00-f43d56a68d1e | generic | None | - | 39ff85e4-cf4e-4969-9408-af47a372ad06 | generic | None | - | 3c6c81ba-0566-4cde-a8c5-7ae4d4644293 | generic | None | - | 219f2fec-0e90-4e04-a5d7-1a78c9fde93b | generic | None | - | 57f2ba05-e57c-4241-bd27-bf95cca9c027 | generic | None | - | a0cc7bb7-0169-4973-8d4a-08151c52dec6 | generic | None | - | afb1d1e2-85db-463c-9769-2a2752eb447e | generic | None | - | 87e52c8d-c66e-43f5-b9fc-da376eccdf2d | generic | None | - | bf383c17-d76a-4e50-b347-426c96020d3b | generic | None | - | 9f25dffd-79f5-4c34-86b6-79767b8db086 | generic | None | - | 4b8ee1ce-9733-4808-921f-6d8ca92a6752 | generic | None | - | 5887a427-286f-47ad-bd4a-d7b9278bbc0f | generic | None | - | f5856741-89d5-462f-a0a2-f2423d9bfc38 | generic | None | - | fea54e18-9668-4df0-a511-5b2af4c76945 | generic | None | - | 304dc702-c57a-471c-81df-6e711d793e50 | generic | None | - +--------------------------------------+---------+------------+ - -You will get a result like the following: - -.. code:: - - +-----------------------+-------------------------------------------------------------------+ - | Field | Value | - +-----------------------+-------------------------------------------------------------------+ - | created_by_project_id | 2c8f25efb722467eb9fc25f38996b7c4 | - | created_by_user_id | 7961a8c338ba4cb8a4ac6dfe0ab333f5 | - | creator | 7961a8c338ba4cb8a4ac6dfe0ab333f5:2c8f25efb722467eb9fc25f38996b7c4 | - | ended_at | None | - | id | 304dc702-c57a-471c-81df-6e711d793e50 | - | metrics | interface-eno1@if_dropped: 511abf80-d9e9-4e37-bde6-b34de19a7a87 | - | | interface-eno1@if_errors: 7bf316e3-ce63-424c-955c-1654541dafea | - | | interface-eno1@if_octets: 0b9a204b-38fd-4b4f-a5a1-c25b9b739c5c | - | | interface-eno1@if_packets: a62006be-d45a-4b2c-a201-4f1b4770f43c | - | | interface-eno2@if_dropped: 56bd5603-ed8c-401c-891e-05170e3b40a7 | - | | interface-eno2@if_errors: 5d2d1a60-1ca8-4256-a395-0125428cf395 | - | | interface-eno2@if_octets: 3f3daf4b-2ef8-4383-b031-294e51487ae9 | - | | interface-eno2@if_packets: 0aa3fb64-764f-402b-b9eb-6fb47e3d0efc | - | | interface-eno3@if_dropped: 23c59f0f-d018-4538-a387-90bd5809a0f0 | - | | interface-eno3@if_errors: c8ab32bb-02e7-48f7-8a67-92cf96aa6974 | - | | interface-eno3@if_octets: be37ef63-9ed5-4547-851e-46f1aa2e91d6 | - | | interface-eno3@if_packets: 149ae533-2f03-4a87-91a6-6aa0f8a541b3 | - | | interface-eno4@if_dropped: 6b8285d5-7e87-4f10-8abc-1ac848bf8240 | - | | interface-eno4@if_errors: 0dcd9925-c6e6-480d-88cb-6eb099cd4650 | - | | interface-eno4@if_octets: 4ff866fd-d5ef-4a55-aeab-7cfbe1ac1f28 | - | | interface-eno4@if_packets: 0fe10bf7-79ab-4bfb-aa6b-64efd3b925c1 | - | | interface-lo@if_dropped: 39318dc7-f008-4258-8832-457c90193924 | - | | interface-lo@if_errors: f3998907-786f-4ffd-a47b-bea1f4b9ad97 | - | | interface-lo@if_octets: f01791f8-8939-4bf3-aae7-abb1e4bffc2e | - | | interface-lo@if_packets: 6aaf06ee-5a8d-49f2-b7b9-c1d27841a89b | - | | load@load: 8d6132f8-6e60-409b-8d64-7092491aa9db | - | | memory@memory.buffered: a6ad6e20-f951-4152-aac3-d6d081c33c09 | - | | memory@memory.cached: ca0e3b30-b450-484b-ac41-a03424da279b | - | | memory@memory.free: 7aee53a8-93f9-4bac-92e3-7694b219c698 | - | | memory@memory.slab_recl: 074897b8-c40e-4538-9ef6-69338764bed3 | - | | memory@memory.slab_unrecl: 1bb6c19d-e788-40cd-98f0-0c5820e03563 | - | | memory@memory.used: 8b56e1ea-0aaa-4c1b-9462-f3698bad2ca7 | - | original_resource_id | 304dc702-c57a-471c-81df-6e711d793e50 | - | project_id | None | - | revision_end | None | - | revision_start | 2018-02-15T15:42:18.495824+00:00 | - | started_at | 2018-02-15T15:42:18.495781+00:00 | - | type | generic | - | user_id | None | - +-----------------------+-------------------------------------------------------------------+ - -To get all the measurements of a particular metric, run: - -.. code-block:: bash - - gnocchi measures show --resource-id --refresh - -For example, to get measurements of used memory over time for instance ``d17d5191-af60-4407-9ed2-e3d48e86ac6d``, run: - -.. code-block:: bash - - gnocchi measures show memory@memory.used --resource-id d17d5191-af60-4407-9ed2-e3d48e86ac6d --refresh - -.. tip:: You may notice that each metric has been assigned a *UUID* to it. Therefore, instead of providing ``metric name``, you can provide ``metric uuid``. - -This will show the latest measurements of that metric with granularity set to 1.0, as well as aggregate values (by default, the mean) over one minute and one hour. Other aggregation methods can be used with the ``--aggregation`` option, such as ``std``, ``count``, ``min``, ``max`` and ``sum``. Your results may appear like this: - -.. code:: - - +---------------------------+-------------+---------------+ - | timestamp | granularity | value | - +---------------------------+-------------+---------------+ - | 2017-12-22T18:00:00+01:00 | 3600.0 | 1222193280.0 | - | 2017-12-22T18:01:00+01:00 | 60.0 | 1222684672.0 | - | 2017-12-22T18:02:00+01:00 | 60.0 | 1222394538.67 | - | 2017-12-22T18:03:00+01:00 | 60.0 | 1222147413.33 | - | 2017-12-22T18:01:20+01:00 | 1.0 | 1222684672.0 | - | 2017-12-22T18:01:30+01:00 | 1.0 | 1222684672.0 | - | 2017-12-22T18:01:40+01:00 | 1.0 | 1222684672.0 | - | 2017-12-22T18:01:50+01:00 | 1.0 | 1222684672.0 | - | 2017-12-22T18:02:00+01:00 | 1.0 | 1222684672.0 | - | 2017-12-22T18:02:10+01:00 | 1.0 | 1222684672.0 | - | 2017-12-22T18:02:20+01:00 | 1.0 | 1222684672.0 | - | 2017-12-22T18:02:30+01:00 | 1.0 | 1221943296.0 | - | 2017-12-22T18:02:40+01:00 | 1.0 | 1222438912.0 | - | 2017-12-22T18:02:50+01:00 | 1.0 | 1221931008.0 | - | 2017-12-22T18:03:00+01:00 | 1.0 | 1221931008.0 | - | 2017-12-22T18:03:10+01:00 | 1.0 | 1221931008.0 | - | 2017-12-22T18:03:20+01:00 | 1.0 | 1221931008.0 | - | 2017-12-22T18:03:30+01:00 | 1.0 | 1222373376.0 | - | 2017-12-22T18:03:40+01:00 | 1.0 | 1222369280.0 | - | 2017-12-22T18:03:50+01:00 | 1.0 | 1222348800.0 | - +---------------------------+-------------+---------------+ - -By default, metrics are stored with an archive policy set to "high", which is defined to keep data as: - -- Per second granularity for the last hour -- Per minute granularity for the last week -- Per hour granularity for a year - -However, note that since ``collectd`` is configured to collect metrics only every 10 seconds, there is no metric measurement for each second but every 10 seconds. - -.. _configure-collectd: - -________________________ -Configuring ``collectd`` -________________________ - -While only a few ``collectd`` plugins are enabled by default, you can leverage the large collection of `available plugins `_. To enable a plugin on your instance, edit the instance's ``/etc/collectd.conf`` file. Uncomment each ``LoadPlugin `` line that you wish to enable. Then, restart collectd with the command: - -.. code-block:: bash - - sudo systemctl restart collectd - -The ``collectd`` configured to send measurements by batch to minimize network traffic. However, if you want to avoid any interference during your experiments, you can disable ``collectd`` with the command: - -.. code-block:: bash - - sudo systemctl stop collectd && sudo systemctl disable collectd - -_____________________________________________ -Metrics for bare metal nodes -_____________________________________________ - -Chameleon automatically collects power usage and temperature data on all nodes in the system. Instantaneous power usage data (in watts) and temperature readings (in Celsius) are collected through the IPMI interface on the chassis controller for the nodes. This “out-of-band” approach does not consume additional power on the node itself and runs even when the node is powered off. - -.. attention:: - Temperature metrics are currently collected from the CPU sensor on each node. These temperature readings are only reported while the node is powered on. - -As with the system metrics, retrieving these automatically collected metrics for a node requires the OpenStack CLI and Gnocchi client plugin (see installation instructions `Setting up the Gnocchi CLI`_ above). To get a list of metrics available for a node, use this command: - -.. code-block:: bash - - $ gnocchi resource show - -To retrieve a specifc reading: - -.. code-block:: bash - - $ gnocchi measures show --resource-id= --refresh - -.. tip:: - The node UUID and the instance UUID are different. You can get a node's UUID for a reservation from the Horizon GUI (https://chi.tacc.chameleoncloud.org for TACC reservations, https://chi.uc.chameleoncloud.org for UC reservations). Click on your lease name from within the list of leases on the Leases subtab within the Reservations tab. The node UUID is at the very bottom under the ``Nodes`` section. You can also find an individual instance node UUID on the instance details page. Click on your instance name on the Instances tab and see ``Physical Host Name`` - -For example, issuing the following command: - -.. code-block:: bash - - $ gnocchi measures show power --resource-id=05dd5e25-440f-4492-b3b8-9d39af83b8bc --refresh - -returns the following power results for node with id ``05dd5e25-440f-4492-b3b8-9d39af83b8bc``. The output below has been truncated: - -.. code:: - - +---------------------------+-------------+--------------------+ - | timestamp | granularity | value | - +---------------------------+-------------+--------------------+ - | 2018-03-21T07:00:00-05:00 | 3600.0 | 3.6990394736842047 | - | 2018-03-21T08:00:00-05:00 | 3600.0 | 3.6944069767441814 | - | 2018-03-21T09:00:00-05:00 | 3600.0 | 3.7072767295597435 | - | 2018-03-21T10:00:00-05:00 | 3600.0 | 3.674499999999995 | - | 2018-03-21T11:00:00-05:00 | 3600.0 | 3.708236024844716 | - | 2018-03-21T12:00:00-05:00 | 3600.0 | 3.6747818181818137 | - | 2018-03-21T13:00:00-05:00 | 3600.0 | 3.706847058823526 | - - . . . . . . - - | 2018-05-07T08:17:43-05:00 | 1.0 | 3.537 | - | 2018-05-07T08:18:03-05:00 | 1.0 | 3.996 | - | 2018-05-07T08:18:23-05:00 | 1.0 | 3.847 | - | 2018-05-07T08:19:03-05:00 | 1.0 | 4.145 | - | 2018-05-07T08:19:23-05:00 | 1.0 | 4.145 | - | 2018-05-07T08:19:43-05:00 | 1.0 | 3.686 | - | 2018-05-07T08:20:03-05:00 | 1.0 | 3.847 | - | 2018-05-07T08:20:23-05:00 | 1.0 | 3.686 | - | 2018-05-07T08:20:43-05:00 | 1.0 | 3.847 | - +---------------------------+-------------+--------------------+ - -To retrieve a metric for a specific time interval, pass the ``start`` and ``stop`` parameters; for example: - -.. code:: - - $ gnocchi measures show temperature_cpu --start 2018-11-27T02:00:00 --stop 2018-11-27T03:00:00 --resource-id=f3f47a67-d805-48d4-9584-f0143ae976cf --refresh - -returns: - -.. code:: - - +---------------------------+-------------+---------------+ - | timestamp | granularity | value | - +---------------------------+-------------+---------------+ - | 2018-11-27T02:00:00-06:00 | 300.0 | 61.0 | - | 2018-11-27T02:05:00-06:00 | 300.0 | 61.0 | - | 2018-11-27T02:10:00-06:00 | 300.0 | 61.0 | - | 2018-11-27T02:15:00-06:00 | 300.0 | 61.0 | - | 2018-11-27T02:20:00-06:00 | 300.0 | 58.6 | - | 2018-11-27T02:25:00-06:00 | 300.0 | 56.5333333333 | - | 2018-11-27T02:30:00-06:00 | 300.0 | 56.0 | - | 2018-11-27T02:35:00-06:00 | 300.0 | 56.0 | - | 2018-11-27T02:40:00-06:00 | 300.0 | 56.0 | - | 2018-11-27T02:45:00-06:00 | 300.0 | 56.0 | - | 2018-11-27T02:50:00-06:00 | 300.0 | 56.0 | - | 2018-11-27T02:55:00-06:00 | 300.0 | 56.0 | - +---------------------------+-------------+---------------+ - -_________________________________________________________ -Energy and Power Consumption Measurement with ``etrace2`` -_________________________________________________________ - -The `CC-CentOS7 `_, `CC-CentOS8 `_, `CC-Ubuntu16.04 `_ and `CC-Ubuntu18.04 `_ appliances, -as well as all Chameleon supported images dervied from them, now include support for reporting energy and power consumption of each CPU socket and of memory DIMMs. -It is done with the ``etrace2`` utility which relies on the `Intel RAPL (Running Average Power Limit) `_ interface. - -.. attention:: - Currenly, ``etrace2`` requires a kernel feature that is not supported on our ARM nodes. - -To spawn your program and print energy consumption: - -.. code-block:: bash - - etrace2 - -To print power consumption every 0.5 second: - -.. code-block:: bash - - etrace2 -i 0.5 - -To print power consumption every 1 second for 10 seconds: - -.. code-block:: bash - - etrace2 -i 1.0 -t 10 - -For example, to report energy consumption during the generation of a large RSA private key: - -.. code:: - - $ etrace2 openssl genrsa -out private.pem 4096 - # ETRACE2_VERSION=0.1 - Generating RSA private key, 4096 bit long modulus - ..............................................................................................................................................................................................................................................................................................................++ - .............................................................................................................................................................++ - e is 65537 (0x10001) - # ELAPSED=2.579472 - # ENERGY=365.788208 - # ENERGY_SOCKET0=99.037841 - # ENERGY_DRAM0=78.577698 - # ENERGY_SOCKET1=109.230103 - # ENERGY_DRAM1=80.336548 - -The energy consumption is reported in joules. - -``etrace2`` reports power and energy consumption of CPUs and memory of the node during the entire execution of the program. This will include consumption of other programs running during this period, as well as power and energy consumption of CPUs and memory under idle load. - -Note the following caveats: - -- `Intel `_ documents that the RAPL is not an analog power meter, but rather uses a software power model. This software power model estimates energy usage by using hardware performance counters and I/O models. Based on their measurements, they match actual power measurements. -- In some situations the total *ENERGY* value is incorrectly reported as a value equal or close to zero. However, the sum of *ENERGY_SOCKET* and *ENERGY_DRAM* values should be accurate. -- Monitoring periods larger than 10-15 minutes may be inaccurate due to RAPL registers overflowing if they're not read regularly. - -This `utility `_ was contributed by Chameleon user `Kazutomo Yoshii `_ of `Argonne National Laboratory `_. - -.. note:: - The Linux kernel version of `CC-Ubuntu16.04 `_ is too old to use ``etrace2`` on Chameleon **Skylake** nodes. - To solve the problem, simply upgrade the Linux kernel. diff --git a/source/technical/networks.rst b/source/technical/networks/index.rst similarity index 76% rename from source/technical/networks.rst rename to source/technical/networks/index.rst index 828b21b..40c1fa8 100644 --- a/source/technical/networks.rst +++ b/source/technical/networks/index.rst @@ -9,16 +9,15 @@ Networking on Chameleon is implemented using `OpenStack Neutron *Floating IPs* in the sidebar +2. To release a single IP: click the dropdown in the *Actions* column and select *Release Floating IP* +3. To release multiple IPs: select them via checkboxes and click the *Release Floating IPs* button + +You can also release floating IP addresses via the command line using ``openstack floating ip delete ``. Floating DNS Records ^^^^^^^^^^^^^^^^^^^^ @@ -94,7 +105,7 @@ These rules allow ssh traffic on port 22 over the public internet. .. warning:: By default, all firewall changes are **temporary**, and will be lost - on instance reboot. This is a saftey mechanism + on instance reboot. This is a safety mechanism to avoid locking yourself out. To make changes **permanent**, execute: .. code-block:: shell @@ -134,7 +145,7 @@ or within your own isolated networks on Chameleon. sudo firewall-cmd --zone=trusted --add-source= -To enable this by default for all private IP ranges, you can do the following, but please note that this can be +To enable this by default for all private IP ranges, you can do the following, but note that this can be insecure on shared or routed networks (sharednet1, sharedwan1 and similar). .. code-block:: shell @@ -143,9 +154,9 @@ insecure on shared or routed networks (sharednet1, sharedwan1 and similar). sudo firewall-cmd --zone=trusted --add-source=172.16.0.0/12 sudo firewall-cmd --zone=trusted --add-source=10.0.0.0/8 -Any other incomming connections will be denied. +Any other incoming connections will be denied. -For more examples and information, please see: +For more examples and information, see: - `Ubuntu's man page for firewalld `_ - `Fedora Linux Guide `_ diff --git a/source/technical/networks/networks_jumbo_frames.rst b/source/technical/networks/networks_jumbo_frames.rst index 111a359..66d05e2 100644 --- a/source/technical/networks/networks_jumbo_frames.rst +++ b/source/technical/networks/networks_jumbo_frames.rst @@ -7,7 +7,7 @@ By default, Ethernet frames for networks created on each Chameleon site default to 1500 byte MTU (maximum transmission unit). However, all TOR switches on Chameleon are configured to allow for payloads of up to 9000 bytes. If you would like to experiment with jumbo frames on your private networks or -over Layer 2 connections then please follow the steps below to implement. +over Layer 2 connections then follow the steps below to implement. .. note:: @@ -46,7 +46,7 @@ Enabling Jumbo Frames on Existing Network ----------------------------------------- You can also modify the MTU of an existing network using the command below. -Please note that this will only effect newly created baremetal instances. +Note that this will only affect newly created bare metal instances. .. code-block:: bash diff --git a/source/technical/power_monitoring.rst b/source/technical/power_monitoring.rst new file mode 100644 index 0000000..0c4b260 --- /dev/null +++ b/source/technical/power_monitoring.rst @@ -0,0 +1,11 @@ +:orphan: + +.. meta:: + :http-equiv=refresh: 0; url=./power_monitoring/index.html + +This page has moved +=================== + +This page has been reorganized. You will be redirected to the new location. + +If you are not redirected automatically, please visit :doc:`power_monitoring/index`. \ No newline at end of file diff --git a/source/technical/power_monitoring/index.rst b/source/technical/power_monitoring/index.rst new file mode 100644 index 0000000..a1432ad --- /dev/null +++ b/source/technical/power_monitoring/index.rst @@ -0,0 +1,44 @@ +.. _power-monitoring: + +================ +Power Monitoring +================ + +Chameleon provides comprehensive power monitoring capabilities to help researchers measure and understand the energy consumption of their experiments. + +.. tip:: + For detailed examples, tool installation instructions, and advanced techniques, see our `Power Measurement and Management blog post `_. + +Available Power Monitoring Methods +================================== + +**Infrastructure-level monitoring:** +- Automatic power and temperature data collection via IPMI/DCMI +- Works on most server-class Intel and AMD nodes +- Provides system-level power consumption data + +**Application-level monitoring:** +- ``etrace2``: Energy measurement for individual applications using Intel RAPL +- ``perf``: Quick RAPL energy measurements +- Scaphandre: Advanced per-process power tracking + +**Long-term monitoring:** +- Prometheus exporters and Grafana for continuous data collection and visualization + +Hardware Support +================ + +Power monitoring support varies by node type: +- **Full support**: Most Intel and AMD compute/GPU nodes +- **Limited support**: Specialized nodes (FPGAs, ARM64) +- **Temperature monitoring**: Only available when nodes are powered on + +Getting Started +=============== + +1. **For system-level monitoring**: Use ``ipmitool dcmi power reading`` to get current power consumption +2. **For application-level monitoring**: Use ``etrace2 `` to measure energy consumption of specific applications +3. **For detailed instructions**: See the `power monitoring blog post `_ + +.. note:: + Power monitoring tools use software-based estimation models and may include system overhead. For accurate measurements, consider baseline readings and validate with multiple tools when possible. \ No newline at end of file diff --git a/source/technical/reservations.rst b/source/technical/reservations.rst index 7e59ea0..f319b23 100644 --- a/source/technical/reservations.rst +++ b/source/technical/reservations.rst @@ -1,623 +1,11 @@ -.. _reservations: +:orphan: -============= -Reservations -============= +.. meta:: + :http-equiv=refresh: 0; url=./reservations/index.html -Unlike virtual resources on a regular on-demand cloud, physical resources on -Chameleon must be reserved before using them for an experiment. Once a -reservation has been accepted, users are guaranteed that resources will be -available at the time they chose (except in extraordinary circumstances such as -hardware or platform failures), which helps to plan large scale experiments. +This page has moved +=================== -Chameleon resources are reserved via `Blazar -`_ which provides Reservation as a -Service for OpenStack. +This page has been reorganized. You will be redirected to the new location. -Three types of resources can be reserved: physical bare metal hosts, network -segments (VLANs), and floating IPs. - -.. attention:: - - **A note on lease stacking** - - To prevent resource hoarding and ensure fair access to specialized hardware, - Chameleon discourages "lease stacking" or making multiple overlapping - reservations. Review our `lease stacking policy `_ - to align your reservations with community practices for efficient resource use. - -Provisioning and Managing Resources Using the GUI -================================================= - -To make reservations of the resources, first log into the Horizon web interface -- either |CHI@TACC| or |CHI@UC|. Then, choose a project and configure your local -timezone. For details on how to choose a project and update personalized -settings, please see :ref:`gui`. - -In the navigation sidebar, go to the *Reservations* section and click *Leases*. - -.. figure:: reservations/leasespage.png - :alt: The Leases page in the GUI - - The Leases page in the GUI - -.. _the-lease-calendars: - -The Lease Calendars -------------------- - -To discover when resources are available, You can access the lease calendars by -clicking on the *Host Calendar* button for physical hosts and clicking on the -*Network Calendar* button for VLANs. This will display a Gantt chart of the -reservations which allows you to find when resources are available. The *Y* axis -represents the different physical nodes in the system and the *X* axis -represents time. - -.. figure:: reservations/hostcalendar.png - :alt: The Host Calendar - - The Host Calendar - -.. tip:: - Education projects may require "sub-leases" to facillitate student access to a - node during a project, while keeping that node available to the project. This - ensures that resources required for a class are not unavailable before a deadline. - If this is required for your usage, we can temporarily grant exclusive access to - a node to your project. Please create a lease for the node, and contact the |Help Desk| - to request exclusive node access for your project. - -.. figure:: reservations/networkcalendar.png - :alt: The Network Calendar - - The Network Calendar - -.. tip:: - - The nodes and VLANs are identified by their *UUIDs*. The colors are used to - indicate different reservations, i.e. the resources that belong to the same - reservation are colored the same. Hovering over the chart provides the - details about the reservation. To change the display time frame, click on - ``1d``, ``1w``, and ``1m`` buttons or fill in the start and end times. - - -.. _reservations-create-lease-gui: - -Creating a Lease to Reserve Resources -------------------------------------- - -Once you have chosen a time period when you want to reserve resources, go back -to the *Leases* screen and click on the *Create Lease* button. It should bring -up the window displayed below: - -.. figure:: reservations/createleasedialog.png - :alt: The Create Lease dialog - - The Create Lease dialog - -#. Pick a name for the lease. The name needs to be unique across your project. - -#. Pick a start time and lease duration in days. If you would like to start your - lease as soon as possible, you may leave the start time blank and Chameleon - will attempt to reserve your nodes to begin immediately with a default Lease - duration of 1 day. - - .. note:: - - If you have not selected a timezone earlier, the default timezone is - **UTC**. Therefore, the date must be entered in **UTC**! - - .. tip:: You can get the UTC time by running ``date -u`` in your terminal. - -#. To reserve a bare metal node, navigate to the "Hosts" tab. - - .. figure:: reservations/nodereservationdialog.png - :alt: The Create Lease dialog Host reservation tab - - The Create Lease dialog Host reservation tab - - a. Check "Reserve Hosts". - - b. Choose the minimum and maximum number of hosts. - - c. Choose a node type in the drop down menu below the *node_type* and *=* drop down lists. - - .. note:: - - You may only request one type of node in each individual lease. If you - wish to request multiple node types, you must create separate Leases for - each node type. - - .. warning:: - - You must select = when matching the node_type to a specific selection. - Using other operators like >= may yield unexpected results. - -#. To reserve a vlan segment, navigate to the "Networks" tab. - - .. figure:: reservations/networkreservationdialog.png - :alt: The Create Lease dialog Network reservation tab - - The Create Lease dialog Network reservation tab - - a. Check "Reserve Network" - - b. Enter the network name and description - - .. note:: - - When a VLAN segment reservation ends, all Neutron resources attached to - the network will be automatically deleted. Bare metal instances using the - network will lose network connectivity. - - .. tip:: - - Network name is required when reserving VLAN segment. - -#. To reserve floating IPs, navigate to the "Networks" tab. - - a. Check "Reserve Floating IPs". - b. Choose the number of floating IPs. - -#. Click on the *Create* button. - -Once created, the lease details will be displayed. At the bottom of the page are -the details about the reservation. Initially the reservation is in the -``Pending`` status, and stays in this state until it reaches the start time. - - .. tip:: - - If you want Blazar to launch an instances or complex appliance as soon as - the lease starts, read the ``Advanced Reservation Orchestration`` section - our :ref:`complex` documentation. - -.. figure:: reservations/leasedetails.png - :alt: Lease details page - - Lease details page - -Once the start time of the lease is reached, the lease will be started and its -reservation will change to ``Active``; you may need to refresh the page to see -the updates. - -.. tip:: - - The lease is identified by a *UUID*. You may find it useful when using the - CLI or submitting tickets on our |Help Desk|. - -.. role:: redbold - -.. _lease-policy: - -.. attention:: - - To ensure fairness to all users, resource reservations (leases) are limited - to a duration of :redbold:`7 days`. However, an active lease within - :redbold:`48 hours` of its end time can be prolonged by :redbold:`up to 7 - days` from the moment of request if resources are available. - - Chameleon will send an email reminder to you 48 hours before your lease ends. - If your lease duration is less than 48 hours, Chameleon will send you an - email right after your lease is created. You can :ref:`disable the email - notification by using the command line `. - -Extending a Lease ------------------ - -To prolong a lease, click on the *Update Lease* button in *Actions* column. - -.. figure:: reservations/updatelease.png - :alt: The Update Lease Parameters dialog - - The Update Lease Parameters dialog - -Fill out the form by specifying the amount of additional time to add to the -lease. Then, click on the *Update* button to finish your request. - -.. tip:: - - If there is an advance reservation blocking your lease prolongation that - could potentially be moved, you can interact through the users mailing list - to coordinate with others users. Additionally, if you know from the start - that your lease will require longer than a week and can justify it, you can - submit a ticket on our |Help Desk| to request a **one-time exception** of - creating a longer lease. You may also other exceptions to our other policies, - such as idle lease termination, by submitting a request. - - -Changing the Number of Nodes of a Lease -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -It is now possible to change the number of nodes reserved in a lease. For -advance reservations that haven't yet started, the node count can be increased -or decreased. For reservations already started, only new nodes can be added. - -To change the number of nodes of a lease, click on the *Update Lease* button in -*Actions* column. - -.. figure:: reservations/updateleasenodecount.png - :alt: The Update Lease Parameters dialog, changing the number of reserved nodes - - The Update Lease Parameters dialog, changing the number of reserved nodes - - -Navigate to the "Hosts" tab, and fill out the form by specifying the new minimum -and maximum numbers of hosts. Then, click on the *Update* button to finish your request. - -Changing the Number of Floating IPs in a Lease -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -It is possible to change the number of floating IPs in a lease, whether the -lease is pending or active. In some situations, you cannot renew a lease due to -another user reserving the same floating IP in your lease. In this case, you -can set your lease to have 0 floating IPs, and create a second lease just for -reserving floating IPs. - -To change the number of floating IPs, click on the *Update Lease* button in -*Actions* column. - -.. figure:: reservations/updateleasefloatingipcount.png - :alt: The Update Lease Parameters dialog, changing the number of reserved IPs - - The Update Lease Parameters dialog, changing the number of reserved IPs - - -Navigate to the "Floating IPs" tab, and fill out the form by specifying the -amount of floating IPs. Then, click on the *Update* button to finish your request. - -Reserving a Node by UUID ------------------------- - -You may reserve a specific node by providing its *UUID*. To learn more about how -to find a node with a specific type, please see :ref:`resource-discovery`. In -the *Create Lease* dialog, select *uid* in the *Resource Type* dropdown. Then, -choose the *UUID* of the node you would like to reserve. - -.. figure:: reservations/uid.png - :alt: Selecting a node by UUID - - Selecting a node by UUID - -.. _reservations-extend-lease-gui: - -.. _reservation-cli: - -Provisioning and Managing Resources Using the CLI -================================================= - -The sections above present the most user friendly mode of usage, with most -actions performed via the web interface. However, Chameleon can be accessed via -the OpenStack command line tools which provides more capabilities. This section -presents some advanced usage using the command line tools. - -.. tip:: - - Reading :ref:`cli` is highly recommended before continuing on the following - sections. - -Blazar Client Installation --------------------------- - -To reserve specific nodes, based on their identifier or their resource -specifications, you must use the `Blazar -`_ command line client. To use the -CLI, you must install the ``python-blazarclient``. To install -``python-blazarclient``, run the following command: - -.. code-block:: bash - - pip install git+https://github.com/ChameleonCloud/python-blazarclient.git@chameleoncloud/xena - -.. note:: - To reserve VLAN segments or floating IPs, you must use a Chameleon fork of - the Blazar client, as above. - -Before using *Blazar Client*, You must configure the environment variables for -your project via ``source`` :ref:`the OpenStack RC Script ` or -use the CLI switches every time you run the commands. Type ``blazar`` in your -terminal session to enter the *Interactive Mode*. You may also use ``blazar`` in -the *Shell Mode*. - -Creating a Lease to Reserve Physical Hosts ------------------------------------------- - -To create a lease, use the ``lease-create`` command. The following arguments are -required: - -- ``--reservation`` with the ``min``, ``max``, ``resource_type``, and ``resource_properties`` attributes -- ``--start-date`` in ``"YYYY-MM-DD HH:MM"`` format -- ``--end-date`` in ``"YYYY-MM-DD HH:MM"`` format -- A lease name - -If ``--start-date`` is ommitted, then the current date and time will be used by default. - -For example, the following command will create a lease with the name of -``my-first-lease`` and the node type of ``compute_skylake`` that starts on June -17th, 2022 at 4:00pm and ends on June 17th, 2022 at 6:00pm: - -.. code-block:: bash - - openstack reservation lease create \ - --reservation min=1,max=1,resource_type=physical:host,resource_properties='["=", "$node_type", "compute_skylake"]' \ - --start-date "2022-06-17 16:00" \ - --end-date "2022-06-17 18:00" \ - my-first-lease - -Instead of specifying the node type, you may also reserve a specific node by -providing it's *UUID*. For example, to reserve the node with *UUID* of -``c9f98cc9-25e9-424e-8a89-002989054ec2``, you may run the command similar to the -following: - -.. code-block:: bash - - openstack reservation lease create \ - --reservation min=1,max=1,resource_type=physical:host,resource_properties='["=", "$uid", "c9f98cc9-25e9-424e-8a89-002989054ec2"]' \ - --start-date "2022-06-17 16:00" \ - --end-date "2022-06-17 18:00" \ - my-custom-lease - -To create a lease with multiple resource properties, you must combine them like -``["and", [property1], [property2], [...] ]``. For example, to reserve a node -with *$architecture.smt_size* of *48* and *node_type* of *compute_haswell*: - -.. code-block:: bash - - openstack reservation lease create \ - --reservation min=1,max=1,resource_type=physical:host,resource_properties='["and", ["=", "$architecture.smt_size", "48"], ["=", "$node_type", "compute_haswell"]]' \ - --start-date "2022-06-17 16:00" \ - --end-date "2022-06-17 18:00" \ - my-custom-lease - -.. _disable-blazar-notification: -.. attention:: - - To specify a ``before_end`` action, simply add ``before_end=`` - to ``reservation`` parameter. For example: - - .. code-block:: bash - - openstack reservation lease create \ - --reservation min=1,max=1,resource_type=physical:host,resource_properties='["=", "$uid", "c9f98cc9-25e9-424e-8a89-002989054ec2"]',before_end=email \ - --start-date "2022-06-17 16:00" \ - --end-date "2022-06-17 18:00" \ - my-custom-lease - - Currently supported ``before_end`` action types include - - +-----------------+---------------------------------------------------------+ - | **Action Type** | **Description** | - +-----------------+---------------------------------------------------------+ - | ``email`` | Send an email notification. | - +-----------------+---------------------------------------------------------+ - | ``default`` | Default action used when no action is specified; | - | | Currently set to ``email``. | - +-----------------+---------------------------------------------------------+ - | ``''`` | Do nothing. | - +-----------------+---------------------------------------------------------+ - - The default ``before_end`` action is set to ``email``. To disable the email - notification, set ``before_end=''``. - - -Actually, you may use any resource property that is in the resource registry to -reserve the nodes. To see the list of properties of nodes, first get the full -list of nodes with the command: - -.. code-block:: bash - - openstack reservation host list - -The output should look like: - -.. code-block:: text - - +------+--------------------------------------+-------+-----------+----------+ - | id | hypervisor_hostname | vcpus | memory_mb | local_gb | - +------+--------------------------------------+-------+-----------+----------+ - | 151 | 00401ba8-4fb0-4f1e-a7dc-e93065ebdd15 | 24 | 128000 | 200 | - | 233 | 004c89fa-ff13-4563-9012-f2d62c1a7aff | 24 | 128000 | 200 | - | 330 | 01029fb8-0a0b-4949-92b0-a756fb8588e5 | 24 | 128000 | 200 | - | 146 | 036b16e3-9fa6-442c-8e6d-cfe12ed5c8a3 | 24 | 128000 | 200 | - | 992 | 05dd5e25-440f-4492-b3b8-9d39af83b8bc | 8 | 3200 | 100 | - | 219 | 066d92f5-7cb9-49ea-8f05-842566672ebf | 24 | 128000 | 200 | - | 3216 | 06b164d5-3514-4ebe-8928-0bd2f9508b80 | 0 | 0 | 0 | - | 156 | 07030786-d6e8-46b4-b0f2-79b0b303b518 | 24 | 128000 | 200 | - | 212 | 07051549-c404-44af-8e73-8beb5891864a | 24 | 128000 | 200 | - | 175 | 07fd65f0-b814-429b-a2fb-3a4afa52de41 | 24 | 128000 | 200 | - | 255 | 081d2cb1-b6b5-4014-b226-7a42d8588307 | 24 | 128000 | 200 | - -To get resource properties of a host, run ``host-show`` command with the ``id`` -listed in the first column. For example, to get the resource properties of the -host 151, run: - -.. code-block:: bash - - openstack reservation host show 151 - -The output should look like: - -.. code-block:: text - - +----------------------------------+---------------------------------------------+ - | Field | Value | - +----------------------------------+---------------------------------------------+ - | architecture.platform_type | x86_64 | - | architecture.smp_size | 2 | - | architecture.smt_size | 48 | - | bios.release_date | 03/09/2022 | - | bios.vendor | Dell Inc. | - | bios.version | 1.2 | - | chassis.manufacturer | Dell Inc. | - | chassis.name | PowerEdge R630 | - | chassis.serial | 4VJGD42 | - | cpu_info | baremetal cpu | - | created_at | 2022-06-26 20:50:58 | - | gpu.gpu | False | - | hypervisor_hostname | 00401ba8-4fb0-4f1e-a7dc-e93065ebdd15 | - | hypervisor_type | ironic | - | hypervisor_version | 1 | - | id | 151 | - | uid | c9f98cc9-25e9-424e-8a89-002989054ec2 | - | updated_at | | - | vcpus | 48 | - | version | 78dbf26565cf24050718674dcf322331fab8ead5 | - +----------------------------------+---------------------------------------------+ - -Any of the property listed in the field column may be used to reserve the nodes. -For example, you can use ``resource_properties='["=", "$architecture.smp_size", -"2"]'`` to reserve a node with two physical processors. - -.. note:: Remember to use ``$`` in front of the property. - -Extending a Lease ------------------ - -To extend your lease, use ``lease-update`` command, and provide time duration -via ``--prolong-for`` switch. The format of the duration is a number followed by -a letter specifying the time unit. ``w`` is for weeks, ``d`` is for days and -``h`` is for hours. For example, if you would like to extend the -``my-first-lease`` by one day, run the following command: - -.. code-block:: bash - - openstack reservation lease update --prolong-for "1d" my-first-lease - -Chameleon Node Types --------------------- - -The following node types are reservable on Chameleon. - -+--------------------------+------------------------------------------------------------------------------+ -| Node Type | ``resource_properties='["=", "$node_type", ""]'`` | -+--------------------------+------------------------------------------------------------------------------+ -| Skylake compute nodes | ``compute_skylake`` | -+--------------------------+------------------------------------------------------------------------------+ -| Storage nodes | ``storage`` | -+--------------------------+------------------------------------------------------------------------------+ -| Haswell Infiniband nodes | ``compute_haswell_ib`` | -+--------------------------+------------------------------------------------------------------------------+ -| Storage Hierarchy nodes | ``storage_hierarchy`` | -+--------------------------+------------------------------------------------------------------------------+ -| NVIDIA K80 nodes | ``gpu_k80`` | -+--------------------------+------------------------------------------------------------------------------+ -| NVIDIA M40 nodes | ``gpu_m40`` | -+--------------------------+------------------------------------------------------------------------------+ -| NVIDIA P100 nodes | ``gpu_p100`` | -+--------------------------+------------------------------------------------------------------------------+ -| NVIDIA P100 NVLink nodes | ``gpu_p100_nvlink`` | -+--------------------------+------------------------------------------------------------------------------+ -| NVIDIA RTX 6000 nodes | ``gpu_rtx_6000`` | -+--------------------------+------------------------------------------------------------------------------+ -| FPGA nodes | ``fpga`` | -+--------------------------+------------------------------------------------------------------------------+ -| Low power Xeon nodes | ``lowpower_xeon`` | -+--------------------------+------------------------------------------------------------------------------+ -| Atom nodes | ``atom`` | -+--------------------------+------------------------------------------------------------------------------+ -| ARM64 nodes | ``arm64`` | -+--------------------------+------------------------------------------------------------------------------+ - -.. _reservation-cli-vlan: - -Creating a Lease to Reserve a VLAN Segment ------------------------------------------- - -To create a lease, use the ``lease-create`` command. The following arguments are -required: - -- ``--reservation`` with the ``resource_type`` and ``network_name`` attributes -- ``--start-date`` in ``"YYYY-MM-DD HH:MM"`` format -- ``--end-date`` in ``"YYYY-MM-DD HH:MM"`` format -- A lease name - -Optional attributes include ``network_description`` and ``resource_properties`` -which can both be added to the ``--reservation`` argument. - -For example, the following command will create a lease with the name of -``my-first-vlan-lease`` and the network name ``my-network`` that starts on June -17th, 2022 at 4:00pm and ends on June 17th, 2022 at 6:00pm: - -.. code-block:: bash - - openstack reservation lease create --reservation resource_type=network,network_name="my-network" --start-date "2022-06-17 16:00" --end-date "2022-06-17 18:00" my-first-vlan-lease - -Adding the ``network_description`` attribute provides its value as the -description field when creating the Neutron network, allowing to leverage -Chameleon :ref:`sdn` features. - -.. code-block:: bash - - openstack reservation lease create --reservation resource_type=network,network_name="my-network",network_description="OFController=${OF_CONTROLLER_IP}:${OF_CONTROLLER_PORT}" --start-date "2022-06-17 16:00" --end-date "2022-06-17 18:00" my-first-vlan-lease - -Adding the ``resource_properties`` attribute allows you to reserve a specific -*network segment* or *physical network* type. There are currently only two -physical network types ``physnet1`` and ``exogeni``. You can read more about -both types in :ref:`networking`. The following two examples show how to reserve -a network by ``segment_id`` or ``physical_network``. - -.. code-block:: bash - - openstack reservation lease create --reservation resource_type=network,network_name=my-network,resource_properties='["==","$segment_id","3501"]' --start-date "2022-06-17 16:00" --end-date "2022-06-17 18:00" my-first-vlan-lease - -.. code-block:: bash - - openstack reservation lease create --reservation resource_type=network,network_name=my-network,resource_properties='["==","$physical_network","physnet1"]' --start-date "2022-06-17 16:00" --end-date "2022-06-17 18:00" my-first-vlan-lease - -While separate leases can be created to reserve nodes and VLAN segments, it is -also possible to combine multiple reservations within a single lease. The -following example creates a lease reserving one Skylake compute node and one -VLAN segment: - -.. code-block:: bash - - openstack reservation lease create --reservation min=1,max=1,resource_type=physical:host,resource_properties='["=", "$node_type", "compute_skylake"]' --reservation resource_type=network,network_name="my-network" --start-date "2022-06-17 16:00" --end-date "2022-06-17 18:00" my-combined-lease - -.. _reservation-cli-fip: - - -Creating a Lease to Reserve Floating IPs ----------------------------------------- - -To create a lease, use the ``lease-create`` command. The following arguments are required: - -- ``--reservation`` with the ``resource_type`` and ``network_id`` attributes -- ``--start-date`` in ``"YYYY-MM-DD HH:MM"`` format -- ``--end-date`` in ``"YYYY-MM-DD HH:MM"`` format -- A lease name - -Multiple floating IPs can be reserved using the ``amount`` attribute. If -ommitted, only one floating IP is reserved. - -For example, the following command will create a lease with the name of -``my-first-fip-lease`` that starts on June 17th, 2022 at 4:00pm and ends on -June 17th, 2022 at 6:00pm and reserves three floating IPs: - -.. code-block:: bash - - pip install python-openstackclient - PUBLIC_NETWORK_ID=$(openstack network show public -c id -f value) - openstack reservation lease create --reservation resource_type=virtual:floatingip,network_id=${PUBLIC_NETWORK_ID},amount=3 --start-date "2022-06-17 16:00" --end-date "2022-06-17 18:00" my-first-fip-lease - - -Reallocating a Node in Your Lease ---------------------------------- - -After creating your lease, you can view its details in the Horizon web -interface. On this page, at the bottom, you can see a list of nodes in your -lease. If you wish to reallocate one of the nodes in your lease, you can press -the red "Re-Allocate Host" button next to it. - -.. figure:: reservations/reallocatehost.png - :alt: The re-allocate buttons on the lease detail page - - The nodes on the lease detail page. - -You can also do the same on the command-line. Run the command that follows, -entering your lease ID and the node ID where appropriate. - -.. code-block:: bash - - openstack reservation host reallocate --lease-id LEASE_ID NODE_ID - -If you re-allocate a host because it is malfunctioning, please make sure to -report it to the `Help Desk `_ so that -we can fix it. +If you are not redirected automatically, please visit :doc:`reservations/index`. \ No newline at end of file diff --git a/source/technical/reservations/cli_reservations.rst b/source/technical/reservations/cli_reservations.rst new file mode 100644 index 0000000..bb80ba0 --- /dev/null +++ b/source/technical/reservations/cli_reservations.rst @@ -0,0 +1,341 @@ +.. _reservation-cli: + +Provisioning and Managing Resources Using the CLI +================================================= + +The sections above present the most user friendly mode of usage, with most +actions performed via the GUI. However, Chameleon can be accessed via +the OpenStack command line tools which provides more capabilities. This section +presents some advanced usage using the command line tools. + +.. tip:: + + Reading :ref:`cli` is highly recommended before continuing on the following + sections. + +Blazar Client Installation +-------------------------- + +To reserve specific nodes, based on their identifier or their resource +specifications, you must use the `Blazar +`_ command line client. To use the +CLI, you must install the ``python-blazarclient``. To install +``python-blazarclient``, run the following command: + +.. code-block:: bash + + pip install git+https://github.com/ChameleonCloud/python-blazarclient.git@chameleoncloud/xena + +.. note:: + To reserve VLAN segments or floating IPs, you must use a Chameleon fork of + the Blazar client, as above. + +Before using *Blazar Client*, You must configure the environment variables for +your project via ``source`` :ref:`the OpenStack RC Script ` or +use the CLI switches every time you run the commands. Type ``blazar`` in your +terminal session to enter the *Interactive Mode*. You may also use ``blazar`` in +the *Shell Mode*. + +Creating a Lease to Reserve Physical Hosts +------------------------------------------ + +To create a lease, use the ``lease-create`` command. The following arguments are +required: + +- ``--reservation`` with the ``min``, ``max``, ``resource_type``, and ``resource_properties`` attributes +- ``--start-date`` in ``"YYYY-MM-DD HH:MM"`` format +- ``--end-date`` in ``"YYYY-MM-DD HH:MM"`` format +- A lease name + +If ``--start-date`` is ommitted, then the current date and time will be used by default. + +For example, the following command will create a lease with the name of +``my-first-lease`` and the node type of ``compute_skylake`` that starts on June +17th, 2022 at 4:00pm and ends on June 17th, 2022 at 6:00pm: + +.. code-block:: bash + + openstack reservation lease create \ + --reservation min=1,max=1,resource_type=physical:host,resource_properties='["=", "$node_type", "compute_skylake"]' \ + --start-date "2022-06-17 16:00" \ + --end-date "2022-06-17 18:00" \ + my-first-lease + +Instead of specifying the node type, you may also reserve a specific node by +providing it's *UUID*. For example, to reserve the node with *UUID* of +``c9f98cc9-25e9-424e-8a89-002989054ec2``, you may run the command similar to the +following: + +.. code-block:: bash + + openstack reservation lease create \ + --reservation min=1,max=1,resource_type=physical:host,resource_properties='["=", "$uid", "c9f98cc9-25e9-424e-8a89-002989054ec2"]' \ + --start-date "2022-06-17 16:00" \ + --end-date "2022-06-17 18:00" \ + my-custom-lease + +To create a lease with multiple resource properties, you must combine them like +``["and", [property1], [property2], [...] ]``. For example, to reserve a node +with *$architecture.smt_size* of *48* and *node_type* of *compute_haswell*: + +.. code-block:: bash + + openstack reservation lease create \ + --reservation min=1,max=1,resource_type=physical:host,resource_properties='["and", ["=", "$architecture.smt_size", "48"], ["=", "$node_type", "compute_haswell"]]' \ + --start-date "2022-06-17 16:00" \ + --end-date "2022-06-17 18:00" \ + my-custom-lease + +.. _disable-blazar-notification: +.. attention:: + + To specify a ``before_end`` action, simply add ``before_end=`` + to ``reservation`` parameter. For example: + + .. code-block:: bash + + openstack reservation lease create \ + --reservation min=1,max=1,resource_type=physical:host,resource_properties='["=", "$uid", "c9f98cc9-25e9-424e-8a89-002989054ec2"]',before_end=email \ + --start-date "2022-06-17 16:00" \ + --end-date "2022-06-17 18:00" \ + my-custom-lease + + Currently supported ``before_end`` action types include + + +-----------------+---------------------------------------------------------+ + | **Action Type** | **Description** | + +-----------------+---------------------------------------------------------+ + | ``email`` | Send an email notification. | + +-----------------+---------------------------------------------------------+ + | ``default`` | Default action used when no action is specified; | + | | Currently set to ``email``. | + +-----------------+---------------------------------------------------------+ + | ``''`` | Do nothing. | + +-----------------+---------------------------------------------------------+ + + The default ``before_end`` action is set to ``email``. To disable the email + notification, set ``before_end=''``. + + +Actually, you may use any resource property that is in the resource registry to +reserve the nodes. To see the list of properties of nodes, first get the full +list of nodes with the command: + +.. code-block:: bash + + openstack reservation host list + +The output should look like: + +.. code-block:: text + + +------+--------------------------------------+-------+-----------+----------+ + | id | hypervisor_hostname | vcpus | memory_mb | local_gb | + +------+--------------------------------------+-------+-----------+----------+ + | 151 | 00401ba8-4fb0-4f1e-a7dc-e93065ebdd15 | 24 | 128000 | 200 | + | 233 | 004c89fa-ff13-4563-9012-f2d62c1a7aff | 24 | 128000 | 200 | + | 330 | 01029fb8-0a0b-4949-92b0-a756fb8588e5 | 24 | 128000 | 200 | + | 146 | 036b16e3-9fa6-442c-8e6d-cfe12ed5c8a3 | 24 | 128000 | 200 | + | 992 | 05dd5e25-440f-4492-b3b8-9d39af83b8bc | 8 | 3200 | 100 | + | 219 | 066d92f5-7cb9-49ea-8f05-842566672ebf | 24 | 128000 | 200 | + | 3216 | 06b164d5-3514-4ebe-8928-0bd2f9508b80 | 0 | 0 | 0 | + | 156 | 07030786-d6e8-46b4-b0f2-79b0b303b518 | 24 | 128000 | 200 | + | 212 | 07051549-c404-44af-8e73-8beb5891864a | 24 | 128000 | 200 | + | 175 | 07fd65f0-b814-429b-a2fb-3a4afa52de41 | 24 | 128000 | 200 | + | 255 | 081d2cb1-b6b5-4014-b226-7a42d8588307 | 24 | 128000 | 200 | + +To get resource properties of a host, run ``host-show`` command with the ``id`` +listed in the first column. For example, to get the resource properties of the +host 151, run: + +.. code-block:: bash + + openstack reservation host show 151 + +The output should look like: + +.. code-block:: text + + +----------------------------------+---------------------------------------------+ + | Field | Value | + +----------------------------------+---------------------------------------------+ + | architecture.platform_type | x86_64 | + | architecture.smp_size | 2 | + | architecture.smt_size | 48 | + | bios.release_date | 03/09/2022 | + | bios.vendor | Dell Inc. | + | bios.version | 1.2 | + | chassis.manufacturer | Dell Inc. | + | chassis.name | PowerEdge R630 | + | chassis.serial | 4VJGD42 | + | cpu_info | baremetal cpu | + | created_at | 2022-06-26 20:50:58 | + | gpu.gpu | False | + | hypervisor_hostname | 00401ba8-4fb0-4f1e-a7dc-e93065ebdd15 | + | hypervisor_type | ironic | + | hypervisor_version | 1 | + | id | 151 | + | uid | c9f98cc9-25e9-424e-8a89-002989054ec2 | + | updated_at | | + | vcpus | 48 | + | version | 78dbf26565cf24050718674dcf322331fab8ead5 | + +----------------------------------+---------------------------------------------+ + +Any of the property listed in the field column may be used to reserve the nodes. +For example, you can use ``resource_properties='["=", "$architecture.smp_size", +"2"]'`` to reserve a node with two physical processors. + +.. note:: Remember to use ``$`` in front of the property. + +Extending a Lease +----------------- + +To extend your lease, use ``lease-update`` command, and provide time duration +via ``--prolong-for`` switch. The format of the duration is a number followed by +a letter specifying the time unit. ``w`` is for weeks, ``d`` is for days and +``h`` is for hours. For example, if you would like to extend the +``my-first-lease`` by one day, run the following command: + +.. code-block:: bash + + openstack reservation lease update --prolong-for "1d" my-first-lease + +Chameleon Node Types +-------------------- + +The following node types are reservable on Chameleon. + ++--------------------------+------------------------------------------------------------------------------+ +| Node Type | ``resource_properties='["=", "$node_type", ""]'`` | ++--------------------------+------------------------------------------------------------------------------+ +| Skylake compute nodes | ``compute_skylake`` | ++--------------------------+------------------------------------------------------------------------------+ +| Storage nodes | ``storage`` | ++--------------------------+------------------------------------------------------------------------------+ +| Haswell Infiniband nodes | ``compute_haswell_ib`` | ++--------------------------+------------------------------------------------------------------------------+ +| Storage Hierarchy nodes | ``storage_hierarchy`` | ++--------------------------+------------------------------------------------------------------------------+ +| NVIDIA K80 nodes | ``gpu_k80`` | ++--------------------------+------------------------------------------------------------------------------+ +| NVIDIA M40 nodes | ``gpu_m40`` | ++--------------------------+------------------------------------------------------------------------------+ +| NVIDIA P100 nodes | ``gpu_p100`` | ++--------------------------+------------------------------------------------------------------------------+ +| NVIDIA P100 NVLink nodes | ``gpu_p100_nvlink`` | ++--------------------------+------------------------------------------------------------------------------+ +| NVIDIA RTX 6000 nodes | ``gpu_rtx_6000`` | ++--------------------------+------------------------------------------------------------------------------+ +| FPGA nodes | ``fpga`` | ++--------------------------+------------------------------------------------------------------------------+ +| Low power Xeon nodes | ``lowpower_xeon`` | ++--------------------------+------------------------------------------------------------------------------+ +| Atom nodes | ``atom`` | ++--------------------------+------------------------------------------------------------------------------+ +| ARM64 nodes | ``arm64`` | ++--------------------------+------------------------------------------------------------------------------+ + +.. _reservation-cli-vlan: + +Creating a Lease to Reserve a VLAN Segment +------------------------------------------ + +To create a lease, use the ``lease-create`` command. The following arguments are +required: + +- ``--reservation`` with the ``resource_type`` and ``network_name`` attributes +- ``--start-date`` in ``"YYYY-MM-DD HH:MM"`` format +- ``--end-date`` in ``"YYYY-MM-DD HH:MM"`` format +- A lease name + +Optional attributes include ``network_description`` and ``resource_properties`` +which can both be added to the ``--reservation`` argument. + +For example, the following command will create a lease with the name of +``my-first-vlan-lease`` and the network name ``my-network`` that starts on June +17th, 2022 at 4:00pm and ends on June 17th, 2022 at 6:00pm: + +.. code-block:: bash + + openstack reservation lease create --reservation resource_type=network,network_name="my-network" --start-date "2022-06-17 16:00" --end-date "2022-06-17 18:00" my-first-vlan-lease + +Adding the ``network_description`` attribute provides its value as the +description field when creating the Neutron network for advanced networking configurations. + +.. code-block:: bash + + openstack reservation lease create --reservation resource_type=network,network_name="my-network",network_description="OFController=${OF_CONTROLLER_IP}:${OF_CONTROLLER_PORT}" --start-date "2022-06-17 16:00" --end-date "2022-06-17 18:00" my-first-vlan-lease + +Adding the ``resource_properties`` attribute allows you to reserve a specific +*network segment* or *physical network* type. There are currently only two +physical network types ``physnet1`` and ``exogeni``. You can read more about +both types in :ref:`networking`. The following two examples show how to reserve +a network by ``segment_id`` or ``physical_network``. + +.. code-block:: bash + + openstack reservation lease create --reservation resource_type=network,network_name=my-network,resource_properties='["==","$segment_id","3501"]' --start-date "2022-06-17 16:00" --end-date "2022-06-17 18:00" my-first-vlan-lease + +.. code-block:: bash + + openstack reservation lease create --reservation resource_type=network,network_name=my-network,resource_properties='["==","$physical_network","physnet1"]' --start-date "2022-06-17 16:00" --end-date "2022-06-17 18:00" my-first-vlan-lease + +While separate leases can be created to reserve nodes and VLAN segments, it is +also possible to combine multiple reservations within a single lease. The +following example creates a lease reserving one Skylake compute node and one +VLAN segment: + +.. code-block:: bash + + openstack reservation lease create --reservation min=1,max=1,resource_type=physical:host,resource_properties='["=", "$node_type", "compute_skylake"]' --reservation resource_type=network,network_name="my-network" --start-date "2022-06-17 16:00" --end-date "2022-06-17 18:00" my-combined-lease + +.. _reservation-cli-fip: + + +Creating a Lease to Reserve Floating IPs +---------------------------------------- + +To create a lease, use the ``lease-create`` command. The following arguments are required: + +- ``--reservation`` with the ``resource_type`` and ``network_id`` attributes +- ``--start-date`` in ``"YYYY-MM-DD HH:MM"`` format +- ``--end-date`` in ``"YYYY-MM-DD HH:MM"`` format +- A lease name + +Multiple floating IPs can be reserved using the ``amount`` attribute. If +ommitted, only one floating IP is reserved. + +For example, the following command will create a lease with the name of +``my-first-fip-lease`` that starts on June 17th, 2022 at 4:00pm and ends on +June 17th, 2022 at 6:00pm and reserves three floating IPs: + +.. code-block:: bash + + pip install python-openstackclient + PUBLIC_NETWORK_ID=$(openstack network show public -c id -f value) + openstack reservation lease create --reservation resource_type=virtual:floatingip,network_id=${PUBLIC_NETWORK_ID},amount=3 --start-date "2022-06-17 16:00" --end-date "2022-06-17 18:00" my-first-fip-lease + + +Reallocating a Node in Your Lease +--------------------------------- + +After creating your lease, you can view its details in the Horizon web +interface. On this page, at the bottom, you can see a list of nodes in your +lease. If you wish to reallocate one of the nodes in your lease, you can press +the red "Re-Allocate Host" button next to it. + +.. figure:: reservations/reallocatehost.png + :alt: The re-allocate buttons on the lease detail page + + The nodes on the lease detail page. + +You can also do the same on the command-line. Run the command that follows, +entering your lease ID and the node ID where appropriate. + +.. code-block:: bash + + openstack reservation host reallocate --lease-id LEASE_ID NODE_ID + +If you re-allocate a host because it is malfunctioning, please make sure to +report it to the `Help Desk `_ so that +we can fix it. \ No newline at end of file diff --git a/source/technical/reservations/gui_reservations.rst b/source/technical/reservations/gui_reservations.rst new file mode 100644 index 0000000..e39c13d --- /dev/null +++ b/source/technical/reservations/gui_reservations.rst @@ -0,0 +1,252 @@ +Provisioning and Managing Resources Using the GUI +================================================= + +To make reservations of the resources, first log into the Horizon GUI +- either |CHI@TACC| or |CHI@UC|. Then, choose a project and configure your local +timezone. For details on how to choose a project and update personalized +settings, see :ref:`gui`. + +In the navigation sidebar, go to the *Reservations* section and click *Leases*. + +.. figure:: reservations/leasespage.png + :alt: The Leases page in the GUI + + The Leases page in the GUI + +.. _the-lease-calendars: + +The Lease Calendars +------------------- + +To discover when resources are available, You can access the lease calendars by +clicking on the *Host Calendar* button for physical hosts and clicking on the +*Network Calendar* button for VLANs. This will display a Gantt chart of the +reservations which allows you to find when resources are available. The *Y* axis +represents the different physical nodes in the system and the *X* axis +represents time. + +.. figure:: reservations/hostcalendar.png + :alt: The Host Calendar + + The Host Calendar + +.. tip:: + Education projects may require "sub-leases" to facillitate student access to a + node during a project, while keeping that node available to the project. This + ensures that resources required for a class are not unavailable before a deadline. + If this is required for your usage, we can temporarily grant exclusive access to + a node to your project. Create a lease for the node, and contact the |Help Desk| + to request exclusive node access for your project. + +.. figure:: reservations/networkcalendar.png + :alt: The Network Calendar + + The Network Calendar + +.. tip:: + + The nodes and VLANs are identified by their *UUIDs*. The colors are used to + indicate different reservations, i.e. the resources that belong to the same + reservation are colored the same. Hovering over the chart provides the + details about the reservation. To change the display time frame, click on + ``1d``, ``1w``, and ``1m`` buttons or fill in the start and end times. + + +.. _reservations-create-lease-gui: + +Creating a Lease to Reserve Resources +------------------------------------- + +Once you have chosen a time period when you want to reserve resources, go back +to the *Leases* screen and click on the *Create Lease* button. It should bring +up the window displayed below: + +.. figure:: reservations/createleasedialog.png + :alt: The Create Lease dialog + + The Create Lease dialog + +#. Pick a name for the lease. The name needs to be unique across your project. + +#. Pick a start time and lease duration in days. If you would like to start your + lease as soon as possible, you may leave the start time blank and Chameleon + will attempt to reserve your nodes to begin immediately with a default Lease + duration of 1 day. + + .. note:: + + If you have not selected a timezone earlier, the default timezone is + **UTC**. Therefore, the date must be entered in **UTC**! + + .. tip:: You can get the UTC time by running ``date -u`` in your terminal. + +#. To reserve a bare metal node, navigate to the "Hosts" tab. + + .. figure:: reservations/nodereservationdialog.png + :alt: The Create Lease dialog Host reservation tab + + The Create Lease dialog Host reservation tab + + a. Check "Reserve Hosts". + + b. Choose the minimum and maximum number of hosts. + + c. Choose a node type in the drop down menu below the *node_type* and *=* drop down lists. + + .. note:: + + You may only request one type of node in each individual lease. If you + wish to request multiple node types, you must create separate Leases for + each node type. + + .. warning:: + + You must select = when matching the node_type to a specific selection. + Using other operators like >= may yield unexpected results. + +#. To reserve a vlan segment, navigate to the "Networks" tab. + + .. figure:: reservations/networkreservationdialog.png + :alt: The Create Lease dialog Network reservation tab + + The Create Lease dialog Network reservation tab + + a. Check "Reserve Network" + + b. Enter the network name and description + + .. note:: + + When a VLAN segment reservation ends, all Neutron resources attached to + the network will be automatically deleted. Bare metal instances using the + network will lose network connectivity. + + .. tip:: + + Network name is required when reserving VLAN segment. + +#. To reserve floating IPs, navigate to the "Networks" tab. + + a. Check "Reserve Floating IPs". + b. Choose the number of floating IPs. + +#. Click on the *Create* button. + +Once created, the lease details will be displayed. At the bottom of the page are +the details about the reservation. Initially the reservation is in the +``Pending`` status, and stays in this state until it reaches the start time. + + .. tip:: + + If you want Blazar to launch an instances or complex appliance as soon as + the lease starts, read the ``Advanced Reservation Orchestration`` section + our :ref:`complex` documentation. + +.. figure:: reservations/leasedetails.png + :alt: Lease details page + + Lease details page + +Once the start time of the lease is reached, the lease will be started and its +reservation will change to ``Active``; you may need to refresh the page to see +the updates. + +.. tip:: + + The lease is identified by a *UUID*. You may find it useful when using the + CLI or submitting tickets on our |Help Desk|. + +.. role:: redbold + +.. _lease-policy: + +.. attention:: + + To ensure fairness to all users, resource reservations (leases) are limited + to a duration of :redbold:`7 days`. However, an active lease within + :redbold:`48 hours` of its end time can be prolonged by :redbold:`up to 7 + days` from the moment of request if resources are available. + + Chameleon will send an email reminder to you 48 hours before your lease ends. + If your lease duration is less than 48 hours, Chameleon will send you an + email right after your lease is created. You can :ref:`disable the email + notification by using the command line `. + +Extending a Lease +----------------- + +To prolong a lease, click on the *Update Lease* button in *Actions* column. + +.. figure:: reservations/updatelease.png + :alt: The Update Lease Parameters dialog + + The Update Lease Parameters dialog + +Fill out the form by specifying the amount of additional time to add to the +lease. Then, click on the *Update* button to finish your request. + +.. tip:: + + If there is an advance reservation blocking your lease prolongation that + could potentially be moved, you can interact through the users mailing list + to coordinate with others users. Additionally, if you know from the start + that your lease will require longer than a week and can justify it, you can + submit a ticket on our |Help Desk| to request a **one-time exception** of + creating a longer lease. You may also other exceptions to our other policies, + such as idle lease termination, by submitting a request. + + +Changing the Number of Nodes of a Lease +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +It is now possible to change the number of nodes reserved in a lease. For +advance reservations that haven't yet started, the node count can be increased +or decreased. For reservations already started, only new nodes can be added. + +To change the number of nodes of a lease, click on the *Update Lease* button in +*Actions* column. + +.. figure:: reservations/updateleasenodecount.png + :alt: The Update Lease Parameters dialog, changing the number of reserved nodes + + The Update Lease Parameters dialog, changing the number of reserved nodes + + +Navigate to the "Hosts" tab, and fill out the form by specifying the new minimum +and maximum numbers of hosts. Then, click on the *Update* button to finish your request. + +Changing the Number of Floating IPs in a Lease +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +It is possible to change the number of floating IPs in a lease, whether the +lease is pending or active. In some situations, you cannot renew a lease due to +another user reserving the same floating IP in your lease. In this case, you +can set your lease to have 0 floating IPs, and create a second lease just for +reserving floating IPs. + +To change the number of floating IPs, click on the *Update Lease* button in +*Actions* column. + +.. figure:: reservations/updateleasefloatingipcount.png + :alt: The Update Lease Parameters dialog, changing the number of reserved IPs + + The Update Lease Parameters dialog, changing the number of reserved IPs + + +Navigate to the "Floating IPs" tab, and fill out the form by specifying the +amount of floating IPs. Then, click on the *Update* button to finish your request. + +Reserving a Node by UUID +------------------------ + +You may reserve a specific node by providing its *UUID*. To learn more about how +to find a node with a specific type, see :ref:`resource-discovery`. In +the *Create Lease* dialog, select *uid* in the *Resource Type* dropdown. Then, +choose the *UUID* of the node you would like to reserve. + +.. figure:: reservations/uid.png + :alt: Selecting a node by UUID + + Selecting a node by UUID + +.. _reservations-extend-lease-gui: \ No newline at end of file diff --git a/source/technical/reservations/index.rst b/source/technical/reservations/index.rst new file mode 100644 index 0000000..c511c8c --- /dev/null +++ b/source/technical/reservations/index.rst @@ -0,0 +1,34 @@ +.. _reservations: + +============= +Reservations +============= + +Unlike virtual resources on a regular on-demand cloud, physical resources on +Chameleon must be reserved before using them for an experiment. Once a +reservation has been accepted, users are guaranteed that resources will be +available at the time they chose (except in extraordinary circumstances such as +hardware or platform failures), which helps to plan large scale experiments. + +Chameleon resources are reserved via `Blazar +`_ which provides Reservation as a +Service for OpenStack. + +Three types of resources can be reserved: physical bare metal hosts, network +segments (VLANs), and floating IPs. + +.. attention:: + + **A note on lease stacking** + + To prevent resource hoarding and ensure fair access to specialized hardware, + Chameleon discourages "lease stacking" or making multiple overlapping + reservations. Review our `lease stacking policy `_ + to align your reservations with community practices for efficient resource use. + +.. toctree:: + :maxdepth: 1 + :caption: Reservation Topics + + gui_reservations + cli_reservations \ No newline at end of file diff --git a/source/technical/reservations/createleasedialog.png b/source/technical/reservations/reservations/createleasedialog.png similarity index 100% rename from source/technical/reservations/createleasedialog.png rename to source/technical/reservations/reservations/createleasedialog.png diff --git a/source/technical/reservations/hostcalendar.png b/source/technical/reservations/reservations/hostcalendar.png similarity index 100% rename from source/technical/reservations/hostcalendar.png rename to source/technical/reservations/reservations/hostcalendar.png diff --git a/source/technical/reservations/leasedetails.png b/source/technical/reservations/reservations/leasedetails.png similarity index 100% rename from source/technical/reservations/leasedetails.png rename to source/technical/reservations/reservations/leasedetails.png diff --git a/source/technical/reservations/leasedetails_vlan.png b/source/technical/reservations/reservations/leasedetails_vlan.png similarity index 100% rename from source/technical/reservations/leasedetails_vlan.png rename to source/technical/reservations/reservations/leasedetails_vlan.png diff --git a/source/technical/reservations/leasepage.png b/source/technical/reservations/reservations/leasepage.png similarity index 100% rename from source/technical/reservations/leasepage.png rename to source/technical/reservations/reservations/leasepage.png diff --git a/source/technical/reservations/leasespage.png b/source/technical/reservations/reservations/leasespage.png similarity index 100% rename from source/technical/reservations/leasespage.png rename to source/technical/reservations/reservations/leasespage.png diff --git a/source/technical/reservations/networkcalendar.png b/source/technical/reservations/reservations/networkcalendar.png similarity index 100% rename from source/technical/reservations/networkcalendar.png rename to source/technical/reservations/reservations/networkcalendar.png diff --git a/source/technical/reservations/networkreservationdialog.png b/source/technical/reservations/reservations/networkreservationdialog.png similarity index 100% rename from source/technical/reservations/networkreservationdialog.png rename to source/technical/reservations/reservations/networkreservationdialog.png diff --git a/source/technical/reservations/nodereservationdialog.png b/source/technical/reservations/reservations/nodereservationdialog.png similarity index 100% rename from source/technical/reservations/nodereservationdialog.png rename to source/technical/reservations/reservations/nodereservationdialog.png diff --git a/source/technical/reservations/reallocatehost.png b/source/technical/reservations/reservations/reallocatehost.png similarity index 100% rename from source/technical/reservations/reallocatehost.png rename to source/technical/reservations/reservations/reallocatehost.png diff --git a/source/technical/reservations/uid.png b/source/technical/reservations/reservations/uid.png similarity index 100% rename from source/technical/reservations/uid.png rename to source/technical/reservations/reservations/uid.png diff --git a/source/technical/reservations/updatelease.png b/source/technical/reservations/reservations/updatelease.png similarity index 100% rename from source/technical/reservations/updatelease.png rename to source/technical/reservations/reservations/updatelease.png diff --git a/source/technical/reservations/updateleasefloatingipcount.png b/source/technical/reservations/reservations/updateleasefloatingipcount.png similarity index 100% rename from source/technical/reservations/updateleasefloatingipcount.png rename to source/technical/reservations/reservations/updateleasefloatingipcount.png diff --git a/source/technical/reservations/updateleasenodecount.png b/source/technical/reservations/reservations/updateleasenodecount.png similarity index 100% rename from source/technical/reservations/updateleasenodecount.png rename to source/technical/reservations/reservations/updateleasenodecount.png diff --git a/source/technical/shares.rst b/source/technical/shares.rst index 5f6a110..8842b17 100644 --- a/source/technical/shares.rst +++ b/source/technical/shares.rst @@ -1,244 +1,11 @@ -.. _shares: +:orphan: -==================== -Shares -==================== +.. meta:: + :http-equiv=refresh: 0; url=./shares/index.html -Chameleon provides a shared file system service through the `OpenStack Manila `_ interface. -With the service, you can create a shared file system, mount to the bare metal instances, and manage some of its properties, such as visibility. +This page has moved +=================== - .. hint:: - - Chameleon shared file system service is currently backed by a CephFS. Same as our :ref:`object store ` service, the data is - replicated and the replication should provide good availability in case of hardware failures. +This page has been reorganized. You will be redirected to the new location. - *Difference between shared file system and object store* - You can choose either shared file system or object store to store, manage, and share your data with your collaborators. The object store - is suitable for storing large objects at scale, but isn’t suitable for transactional data, as objects are immutable and updated in their - entirety. Chameleon shared file system instead provides a NFS mount to the bare metal instances, with the NFS protocol managing locking - and data integrity processes required to provide multiple concurrent access to data. - -The shared file system service is available at `CHI@UC `_ and `CHI@TACC `_. -Each region has its own service and the shares created at one region are not available to the other. As all other Chameleon services, you can create -and manage your shares using both GUI and CLI. - -.. _storage_network: - -Storage Networks -================================ - -To provide isolation among shares created by different projects, accessing a share requires a storage network, which are special networks you can -reserve to use. When reserving a storage network, add `usage_type=storage` to the resource properties. To learn more about reserving networks, read -the :ref:`reservations documentation `. All bare metal instances that are created on the storage network have access to all the project -shares. - - .. tip:: - - To attach floating IP to your instance created on a storage network, you need to create a router with `public` external network. Then connect - the storage subnet to the router. You must specify an unused IP address which belongs to the selected subnet. To learn more about creating - router and connecting subnet, please read :ref:`isolated network VLANs `. - -Shares -================================ - -Visibility --------------------------------- - -Shares are owned by the project. By default, all shares have `private` visibility and can only be listed and accessed within your project. -All bare metal instances owned by the project have read and write permissions to the project’s shares. You can also make your shares `public`. -All Chameleon users and projects can list public shares, and with a storage network, all projects have read-only access to a public share. - -Accessibility --------------------------------- - -A share is a pre-allocated storage space at a CephFS. You can :ref:`mount your shares to your bare metal instances via NFS protocol `. -The accessibility of the shares are controlled internally by the reservation service. You are not allowed to edit the access rules of a share. - -Quotas --------------------------------- - -We do not charge SUs for the storage spaces of your shares. However, we do limit the total size and the number of shares you can create within -your project. The maximum number of shares is 10 and the maximum size allowed for all shares in a project is 2000 GiB. If you need to increase -the default quota, please submit a ticket via the `Help Desk `_. - -Managing Shares using GUI -================================ - -To manage your share, use the `Shares` page at `CHI@UC `_ or `CHI@TACC `_ -by navigating to `Project > Share > Shares`. - - .. figure:: shares/sharespage.png - :alt: The Shares page - - The Shares page - -Create Share --------------------------------- - -Click the `Create Share` button. In the `Create Share` dialog, provide a name and the size of your share, and then click the `Create` button to -create a share. - - .. figure:: shares/createshare.png - :alt: The Create Share dialog - - The Create Share dialog - - .. note:: - - A storage network is not required for creating shares. It’s only required to access the shares. - -.. _view-share-gui: - -View Share --------------------------------- - -You can look at the details of a share by clicking the share name in the `Shares` page. Note that the paths of the `export locations` are important -as you will use this path to mount your share to your bare metal instances. You can also see the other properties, such as visibility and size. -The access rules are listed in the `share details` page, though you can not edit the rules, as they are controlled by the reservation service. - - .. figure:: shares/sharedetails.png - :alt: The Share details - - The Share details - -Edit Share --------------------------------- -You can manage the properties and extend the size of a share by clicking the `Action` dropdown in the `Shares` page. - - .. figure:: shares/manageshare.png - :alt: The Action dropdown - - The Action dropdown - -Delete Share --------------------------------- -You can use the `Action` dropdown to delete a single share, or select multiple shares and click the `Delete Shares` button. - - .. important:: - - Be careful when deleting shares, as the action is irreversible. However, the termination of your storage network reservation **DOES NOT** delete your share. - Your shares persist until you manually delete them. - - -Managing Shares using CLI -================================ - -As all other Chameleon services, you can manage your shares via CLI as well. - - .. tip:: - - Reading :ref:`Command Line Interface (CLI) ` is highly recommended before continuing on the following sections. - -In addition to installing the CLI, you must also install `python-manilaclient` package: - - .. code-block:: bash - - pip install python-manilaclient - -Then, you must set environment variables for your account and project using :ref:`The OpenStack RC Script `. - - .. tip:: - - If you get HTTP 406 error of ``version is not supported by the API``, please add ``--os-share-api-version 2.65`` to - the command to specify manila minor version. - -List Shares --------------------------------- - -To list all shares of your project, run the following command: - - .. code-block:: bash - - openstack share list - -You can filter the results by the share name via adding a ``--name`` argument to the list command. - -Create Share --------------------------------- - -To create a share, using the following command: - - .. code-block:: bash - - openstack share create --name NFS - -For example, for creating a 1 GiB share with name of ``my-first-share``, run: - - .. code-block:: bash - - openstack share create --name my-first-share NFS 1 - - .. note:: - - Only the NFS protocol is supported. - -You can add the ``--public true`` to make your share public. - -Edit Share --------------------------------- - -To change the visibility of a share, run: - - .. code-block:: bash - - openstack share set --public - -To update the name or the description of a share, run: - - .. code-block:: bash - - openstack share set --name --description - -To extend/shrink the size of a share, run: - - .. code-block:: bash - - openstack share resize - -.. _view-share-cli: - -View Share --------------------------------- - -To view the details of a share, run: - - .. code-block:: bash - - openstack share show - -Delete Share --------------------------------- - -To delete a share, run the following command: - - .. code-block:: bash - - openstack share delete - -.. _mount-share: - -Mounting Shares to Instances -================================ - -In order to allow your instances to access the share, you need to create your instances using the :ref:`pre-reserved storage network `. -To learn more about how to create a bare metal instance on a network, read :ref:`the bare metal instances section `. - - .. important:: - - The shares are independent of the storage networks. You can create shares any time regardless of the status of the storage networks. - The storage networks are only used to access your data stored in the share. - -After your instance becomes active, find the export location path of the share using :ref:`GUI ` or :ref:`CLI `. -To mount the share, run the following command: - - .. code-block:: bash - - sudo mount -t nfs -o nfsvers=4.2,proto=tcp - -Now, you can read and write to the share and it behaves identically to a regular file system. - -To unmount, run the following command: - - .. code-block:: bash - - sudo umount +If you are not redirected automatically, please visit :doc:`shares/index`. \ No newline at end of file diff --git a/source/technical/shares/cli_management.rst b/source/technical/shares/cli_management.rst new file mode 100644 index 0000000..8c92fab --- /dev/null +++ b/source/technical/shares/cli_management.rst @@ -0,0 +1,94 @@ +Managing Shares using CLI +========================= + +As all other Chameleon services, you can manage your shares via CLI as well. + +.. tip:: + + Reading :ref:`Command Line Interface (CLI) ` is highly recommended before continuing on the following sections. + +In addition to installing the CLI, you must also install `python-manilaclient` package: + +.. code-block:: bash + + pip install python-manilaclient + +Then, you must set environment variables for your account and project using :ref:`The OpenStack RC Script `. + +.. tip:: + + If you get HTTP 406 error of ``version is not supported by the API``, add ``--os-share-api-version 2.65`` to + the command to specify manila minor version. + +List Shares +------------ + +To list all shares of your project, run the following command: + +.. code-block:: bash + + openstack share list + +You can filter the results by the share name via adding a ``--name`` argument to the list command. + +Create Share +------------ + +To create a share, using the following command: + +.. code-block:: bash + + openstack share create --name NFS + +For example, for creating a 1 GiB share with name of ``my-first-share``, run: + +.. code-block:: bash + + openstack share create --name my-first-share NFS 1 + +.. note:: + + Only the NFS protocol is supported. + +You can add the ``--public true`` to make your share public. + +Edit Share +---------- + +To change the visibility of a share, run: + +.. code-block:: bash + + openstack share set --public + +To update the name or the description of a share, run: + +.. code-block:: bash + + openstack share set --name --description + +To extend/shrink the size of a share, run: + +.. code-block:: bash + + openstack share resize + +.. _view-share-cli: + +View Share +---------- + +To view the details of a share, run: + +.. code-block:: bash + + openstack share show + +Delete Share +------------ + +To delete a share, run the following command: + +.. code-block:: bash + + openstack share delete \ No newline at end of file diff --git a/source/technical/shares/concepts.rst b/source/technical/shares/concepts.rst new file mode 100644 index 0000000..0ebb654 --- /dev/null +++ b/source/technical/shares/concepts.rst @@ -0,0 +1,43 @@ +.. _shares-concepts: + +Shares Concepts +=============== + +.. _storage_network: + +Storage Networks +---------------- + +To provide isolation among shares created by different projects, accessing a share requires a storage network, which are special networks you can +reserve to use. When reserving a storage network, add `usage_type=storage` to the resource properties. To learn more about reserving networks, read +the :ref:`reservations documentation `. All bare metal instances that are created on the storage network have access to all the project +shares. + +.. tip:: + + To attach floating IP to your instance created on a storage network, you need to create a router with `public` external network. Then connect + the storage subnet to the router. You must specify an unused IP address which belongs to the selected subnet. To learn more about creating + router and connecting subnet, read :ref:`isolated network VLANs `. + +Shares +------ + +Visibility +---------- + +Shares are owned by the project. By default, all shares have `private` visibility and can only be listed and accessed within your project. +All bare metal instances owned by the project have read and write permissions to the project's shares. You can also make your shares `public`. +All Chameleon users and projects can list public shares, and with a storage network, all projects have read-only access to a public share. + +Accessibility +------------- + +A share is a pre-allocated storage space at a CephFS. You can :ref:`mount your shares to your bare metal instances via NFS protocol `. +The accessibility of the shares are controlled internally by the reservation service. You are not allowed to edit the access rules of a share. + +Quotas +------ + +We do not charge SUs for the storage spaces of your shares. However, we do limit the total size and the number of shares you can create within +your project. The maximum number of shares is 10 and the maximum size allowed for all shares in a project is 2000 GiB. If you need to increase +the default quota, submit a ticket via the |Help Desk|. \ No newline at end of file diff --git a/source/technical/shares/gui_management.rst b/source/technical/shares/gui_management.rst new file mode 100644 index 0000000..b6aafcf --- /dev/null +++ b/source/technical/shares/gui_management.rst @@ -0,0 +1,57 @@ +Managing Shares using GUI +========================= + +To manage your share, use the `Shares` page at `CHI@UC `_ or `CHI@TACC `_ +by navigating to `Project > Share > Shares`. + +.. figure:: sharespage.png + :alt: The Shares page + + The Shares page + +Create Share +------------ + +Click the `Create Share` button. In the `Create Share` dialog, provide a name and the size of your share, and then click the `Create` button to +create a share. + +.. figure:: createshare.png + :alt: The Create Share dialog + + The Create Share dialog + +.. note:: + + A storage network is not required for creating shares. It's only required to access the shares. + +.. _view-share-gui: + +View Share +---------- + +You can look at the details of a share by clicking the share name in the `Shares` page. Note that the paths of the `export locations` are important +as you will use this path to mount your share to your bare metal instances. You can also see the other properties, such as visibility and size. +The access rules are listed in the `share details` page, though you can not edit the rules, as they are controlled by the reservation service. + +.. figure:: sharedetails.png + :alt: The Share details + + The Share details + +Edit Share +---------- +You can manage the properties and extend the size of a share by clicking the `Action` dropdown in the `Shares` page. + +.. figure:: manageshare.png + :alt: The Action dropdown + + The Action dropdown + +Delete Share +------------ +You can use the `Action` dropdown to delete a single share, or select multiple shares and click the `Delete Shares` button. + +.. important:: + + Be careful when deleting shares, as the action is irreversible. However, the termination of your storage network reservation **DOES NOT** delete your share. + Your shares persist until you manually delete them. \ No newline at end of file diff --git a/source/technical/shares/index.rst b/source/technical/shares/index.rst new file mode 100644 index 0000000..63f5e01 --- /dev/null +++ b/source/technical/shares/index.rst @@ -0,0 +1,32 @@ +.. _shares: + +==================== +Shares +==================== + +Chameleon provides a shared file system service through the `OpenStack Manila `_ interface. +With the service, you can create a shared file system, mount to the bare metal instances, and manage some of its properties, such as visibility. + +.. hint:: + + Chameleon shared file system service is currently backed by a CephFS. Same as our :ref:`object store ` service, the data is + replicated and the replication should provide good availability in case of hardware failures. + + *Difference between shared file system and object store* + You can choose either shared file system or object store to store, manage, and share your data with your collaborators. The object store + is suitable for storing large objects at scale, but isn't suitable for transactional data, as objects are immutable and updated in their + entirety. Chameleon shared file system instead provides a NFS mount to the bare metal instances, with the NFS protocol managing locking + and data integrity processes required to provide multiple concurrent access to data. + +The shared file system service is available at `CHI@UC `_ and `CHI@TACC `_. +Each region has its own service and the shares created at one region are not available to the other. As all other Chameleon services, you can create +and manage your shares using both GUI and CLI. + +.. toctree:: + :maxdepth: 1 + :caption: Shares Topics + + concepts + gui_management + cli_management + mounting \ No newline at end of file diff --git a/source/technical/shares/mounting.rst b/source/technical/shares/mounting.rst new file mode 100644 index 0000000..bc94434 --- /dev/null +++ b/source/technical/shares/mounting.rst @@ -0,0 +1,27 @@ +.. _mount-share: + +Mounting Shares to Instances +============================= + +In order to allow your instances to access the share, you need to create your instances using the :ref:`pre-reserved storage network `. +To learn more about how to create a bare metal instance on a network, read :ref:`the bare metal instances section `. + +.. important:: + + The shares are independent of the storage networks. You can create shares any time regardless of the status of the storage networks. + The storage networks are only used to access your data stored in the share. + +After your instance becomes active, find the export location path of the share using :ref:`GUI ` or :ref:`CLI `. +To mount the share, run the following command: + +.. code-block:: bash + + sudo mount -t nfs -o nfsvers=4.2,proto=tcp + +Now, you can read and write to the share and it behaves identically to a regular file system. + +To unmount, run the following command: + +.. code-block:: bash + + sudo umount \ No newline at end of file diff --git a/source/technical/sharing.rst b/source/technical/sharing.rst index c623204..d3c9efc 100644 --- a/source/technical/sharing.rst +++ b/source/technical/sharing.rst @@ -1,237 +1,11 @@ -.. _trovi: +:orphan: -==================== -Trovi sharing portal -==================== +.. meta:: + :http-equiv=refresh: 0; url=./sharing/index.html -`Chameleon Trovi `_ is a -sharing portal that allows you to share digital research and education -artifacts, such as packaged experiments, workshop tutorials, or class materials. -Each research artifact is represented as a deposition (a remotely accessible -folder) where a user can put Jupyter notebooks, links to images, orchestration -templates, data, software, and other digital representations that together -represent a focused contribution that can be run on Chameleon. Users can use -these artifacts to recreate and rerun experiments or class exercises on a -Jupyter Notebook within Chameleon. They can also create their own artifacts and -publish them directly to Trovi from within :ref:`Chameleon's Jupyter server -`. +This page has moved +=================== -To get started, find the "Trovi" dropdown option under the "Experiment" section -of chameleoncloud.org. Once you're on the Trovi homepage, you'll see a list of -publicly available experiments and other digital artifacts. You can now browse -those artifacts or upload your own. +This page has been reorganized. You will be redirected to the new location. -.. figure:: sharing/sharing_dropdown.png - :alt: Location of Trovi in the UI. - :figclass: screenshot - - The "Trovi" option under the "Experiment" section takes you to Trovi. - -Browsing artifacts -================== - -Trovi allows you to browse artifacts, presented in a scrolling list format. On -the right hand side, there are multiple filtering options. The -"All" choice shows you all of the artifacts you have access to. You can also see -how many times people have downloaded and launched your notebook with the icons -in the bottom left corner of an artifact. - -.. note:: - - Some Trovi artifacts are supported by the Chameleon team and are denoted - with a small Chameleon logo. You can contact the |Help Desk| if you are - using these artifacts and encounter issues. - -Launching an artifact ---------------------- - -The most powerful feature available via Trovi is the ability to re-launch the -available artifacts within Chameleon. Clicking "Launch with JupyterHub" will -open a new Jupyter Notebook server with the artifacts downloaded (we support -artifacts up to 500MB in total size, please contact the |Help Desk| if you need -more space). The animation below shows how easy it is: - -.. figure:: sharing/sharing_launching.gif - :alt: Animation of clicking launch button. - :figclass: screenshot - - Clicking the "Launch with JupyterHub" button to import a Trovi artifact into - your own Jupyter server. - -Packaging shared artifacts -========================== - -You can publish new artifacts to Trovi either from your primary Jupyter server -or by editing a previously-shared artifact. In the latter case, you are -effectively creating a new "forked" artifact owned by you. - -When you've finished creating or making changes to an experiment, in the Jupyter -interface, select the directory (not a single file) you wish to package. Then, -click on the "Share" tab and select "Package as a new artifact". Your artifact -is now packaged and uploaded to Chameleon file storage, and you'll be prompted -to fill out descriptions about the artifact. Don't worry if you want to change -this later---you will be able to :ref:`edit them on the Trovi portal or within -Jupyter `. - -Congratulations! Your artifact is now uploaded to Trovi---but to make it -accessible to others you need to :ref:`adjust its sharing settings -`. - -.. figure:: sharing/sharing_packaging.gif - :alt: Animation of packaging a new artifact from Jupyter. - :figclass: screenshot - -.. _trovi-new-version: - -Saving new versions -------------------- - -If you make changes to your artifact, you can submit an updated version. Within -Jupyter, you navigate to the "Sharing" tab, but this time you click "Create new -artifact version". The different versions are viewable on the Trovi portal -after clicking on the artifact. - -.. figure:: sharing/sharing_new_version.gif - :alt: Animation of uploading a new artifact version from Jupyter. - :figclass: screenshot - -.. _trovi-edit: - -Editing artifacts ------------------ - -You can edit an artifact's metadata, including its title, description, and list -of authors at any time via the Jupyter interface. To delete single artifact -versions, click the "trashcan" icon next to it in the edit view. To delete the -entire artifact, click the red "Delete All" button. - -.. note:: - Any artifacts published to :ref:`Zenodo ` cannot be deleted. - -.. figure:: sharing/sharing_edit_meta.gif - :alt: Animation of editing an artifact's metadata. - :figclass: screenshot - -This edit view is also available from Trovi via the "Edit" button. - -.. _create-git-version: - -Creating a version from Git -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Under an artifact's edit settings, you will also see a button for creating a -new version from Git. This will allow you to enter a Git URL, the same one used -to clone your repository, and also a reference (lease as HEAD for the latest -commit). When a user launches your artifact, their notebook will checkout the -specified commit. - -.. _trovi-sharing: - -Adjusting sharing settings --------------------------- - -When you first upload your packaged artifact to Trovi, its visibility is set as -private, meaning only you can see or launch it. There are multiple options to -change the visibility of the artifact, and you have the option to decide how -visible you want it to be. - -1. **Publish with DOI**: this option allows you to :ref:`publish a version of your - artifact to Zenodo ` and receive a DOI, which you can use to - cite your artifact in, e.g., an academic paper. -2. **Publish without DOI**: this option allows any Chameleon user to find and - launch your artifact. It can be useful if you want to distribute the artifact - widely but do not necessarily with to publish it to Zenodo and get a DOI - for citation. -3. **Share via private link**: this option allows you to share the experiment to - select people, like individual colleagues, advisors, or students. Anybody in - possession of the link can view and launch any version of the artifact. - -To make your artifact shareable, select it in Trovi, click "Share", and check the box before "Enable all users to find and share". - -.. _trovi-roles: - -Assigning Roles to Other Users -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. figure:: sharing/sharing_edit_roles.png - :alt: Screenshot of the artifact role menu - :figclass: screenshot - -You can assign roles to other users which allow them to collaborate on your artifacts. -There are two roles: **Collaborator** and **Administrator**. - -**Collaborators** are allowed to edit artifact metadata, upload new versions, -delete old versions, and share private artifacts. - -**Administrators** have full control over the artifact, including -assigning roles to other users. - -Artifact owners cannot have their Adminstrator privileges removed. - -.. _trovi-zenodo: - -Publishing to Zenodo -^^^^^^^^^^^^^^^^^^^^ - -.. attention:: - You can only request a DOI for artifacts uploaded via the Jupyter interface. - You cannot request a DOI for an artifact version uploaded via git. - -Trovi is intended for sharing work in progress with a limited group of "friends -and family". However, once you complete your experiment package you may want to -publish it so that you can reference it from your paper. To do that Chameleon -supports integration with Zenodo, an open-access storage repository backed by -CERN, for permanent artifact hosting. To share your artifact and store it on -Zenodo, go to the "Share" page for the artifact. On the right-hand side you'll -see a list of all versions you've saved. Pick the version you want to publish to -Zenodo and check "Request DOI", then click "Save." - -.. important:: - - Once published, **Zenodo artifacts cannot be deleted** and are additionally - **publicly available**. Your artifact will appear in Trovi in the "Public" - section, and any Chameleon user can access it, as can anybody on the - Internet via Zenodo's own listing. - - If you wish to make your artifact public but don't to publish it, use the - "Publish without DOI" option. With this option it is possible to make the - artifact private later on if you wish; this is not possible when publishing - to Zenodo. - - You can only request a DOI one time per artifact. If you want to update your - experiment files and request a second DOI, you should instead create a new - artifact. - -This also creates a DOI, which you can easily include in your -paper. The artifacts shared on Zenodo also appear on Trovi. - -Importing an artifact ---------------------- - -Instead of creating an artifact inside Jupyterhub, you can package an existing -Git repository into an artifact. When a user launches the artifact, the -contents of the repository will be added to a Jupyter notebook. - -To create an artifact, click "Import Artifact" on the sidebar of Trovi. You are -first asked for the artifact's metadata. At the bottom of the form, there is -a button for "Import from Git." After clicking this, you will need to enter a -git remote URL, and choose which commit to tie the version to. - -To update the artifact, you must create a :ref:`new version `. -This ensures that a given version of your artifact always has the same contents. - -Exporting via git ------------------ - -If you wish to move your code and notebooks outside of your Jupyter notebook, -one option is to export it into a git repository. - -#. Click the "+" button on the top left of your notebook, and choose "Terminal". - -#. Run the command ``cd work``. If there is a specific directory you wish to - export, you can ``cd`` again into it. - -#. Follow the instructions to set up a repository per your git host. For GitHub - see `this document `_. - -#. After the repository is setup, you should be able to commit and push with - the git CLI. +If you are not redirected automatically, please visit :doc:`sharing/index`. \ No newline at end of file diff --git a/source/technical/sharing/browsing_artifacts.rst b/source/technical/sharing/browsing_artifacts.rst new file mode 100644 index 0000000..47e2942 --- /dev/null +++ b/source/technical/sharing/browsing_artifacts.rst @@ -0,0 +1,32 @@ +.. _trovi-browsing: + +Browsing artifacts +================== + +Trovi allows you to browse artifacts, presented in a scrolling list format. On +the right hand side, there are multiple filtering options. The +"All" choice shows you all of the artifacts you have access to. You can also see +how many times people have downloaded and launched your notebook with the icons +in the bottom left corner of an artifact. + +.. note:: + + Some Trovi artifacts are supported by the Chameleon team and are denoted + with a small Chameleon logo. You can contact the |Help Desk| if you are + using these artifacts and encounter issues. + +Launching an artifact +--------------------- + +The most powerful feature available via Trovi is the ability to re-launch the +available artifacts within Chameleon. Clicking "Launch with JupyterHub" will +open a new Jupyter Notebook server with the artifacts downloaded (we support +artifacts up to 500MB in total size, contact the |Help Desk| if you need +more space). The animation below shows how easy it is: + +.. figure:: ../sharing/sharing_launching.gif + :alt: Animation of clicking launch button. + :figclass: screenshot + + Clicking the "Launch with JupyterHub" button to import a Trovi artifact into + your own Jupyter server. \ No newline at end of file diff --git a/source/technical/sharing/index.rst b/source/technical/sharing/index.rst new file mode 100644 index 0000000..d4704e5 --- /dev/null +++ b/source/technical/sharing/index.rst @@ -0,0 +1,35 @@ +.. _trovi: + +==================== +Trovi Sharing Portal +==================== + +`Chameleon Trovi `_ is a +sharing portal that allows you to share digital research and education +artifacts, such as packaged experiments, workshop tutorials, or class materials. +Each research artifact is represented as a deposition (a remotely accessible +folder) where a user can put Jupyter notebooks, links to images, orchestration +templates, data, software, and other digital representations that together +represent a focused contribution that can be run on Chameleon. Users can use +these artifacts to recreate and rerun experiments or class exercises on a +Jupyter Notebook within Chameleon. They can also create their own artifacts and +publish them directly to Trovi from within :ref:`Chameleon's Jupyter server +`. + +To get started, find the "Trovi" dropdown option under the "Experiment" section +of chameleoncloud.org. Once you're on the Trovi homepage, you'll see a list of +publicly available experiments and other digital artifacts. You can now browse +those artifacts or upload your own. + +.. figure:: ../sharing/sharing_dropdown.png + :alt: Location of Trovi in the UI. + :figclass: screenshot + + The "Trovi" option under the "Experiment" section takes you to Trovi. + +.. toctree:: + :maxdepth: 2 + :caption: Trovi Sharing Portal Topics + + browsing_artifacts + packaging_artifacts \ No newline at end of file diff --git a/source/technical/sharing/packaging_artifacts.rst b/source/technical/sharing/packaging_artifacts.rst new file mode 100644 index 0000000..d80c7ca --- /dev/null +++ b/source/technical/sharing/packaging_artifacts.rst @@ -0,0 +1,179 @@ +.. _trovi-packaging: + +Packaging shared artifacts +========================== + +You can publish new artifacts to Trovi either from your primary Jupyter server +or by editing a previously-shared artifact. In the latter case, you are +effectively creating a new "forked" artifact owned by you. + +When you've finished creating or making changes to an experiment, in the Jupyter +interface, select the directory (not a single file) you wish to package. Then, +click on the "Share" tab and select "Package as a new artifact". Your artifact +is now packaged and uploaded to Chameleon file storage, and you'll be prompted +to fill out descriptions about the artifact. Don't worry if you want to change +this later---you will be able to :ref:`edit them on the Trovi portal or within +Jupyter `. + +Congratulations! Your artifact is now uploaded to Trovi---but to make it +accessible to others you need to :ref:`adjust its sharing settings +`. + +.. figure:: ../sharing/sharing_packaging.gif + :alt: Animation of packaging a new artifact from Jupyter. + :figclass: screenshot + +.. _trovi-new-version: + +Saving new versions +------------------- + +If you make changes to your artifact, you can submit an updated version. Within +Jupyter, you navigate to the "Sharing" tab, but this time you click "Create new +artifact version". The different versions are viewable on the Trovi portal +after clicking on the artifact. + +.. figure:: ../sharing/sharing_new_version.gif + :alt: Animation of uploading a new artifact version from Jupyter. + :figclass: screenshot + +.. _trovi-edit: + +Editing artifacts +----------------- + +You can edit an artifact's metadata, including its title, description, and list +of authors at any time via the Jupyter interface. To delete single artifact +versions, click the "trashcan" icon next to it in the edit view. To delete the +entire artifact, click the red "Delete All" button. + +.. note:: + Any artifacts published to :ref:`Zenodo ` cannot be deleted. + +.. figure:: ../sharing/sharing_edit_meta.gif + :alt: Animation of editing an artifact's metadata. + :figclass: screenshot + +This edit view is also available from Trovi via the "Edit" button. + +.. _create-git-version: + +Creating a version from Git +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Under an artifact's edit settings, you will also see a button for creating a +new version from Git. This will allow you to enter a Git URL, the same one used +to clone your repository, and also a reference (lease as HEAD for the latest +commit). When a user launches your artifact, their notebook will checkout the +specified commit. + +.. _trovi-sharing: + +Adjusting sharing settings +-------------------------- + +When you first upload your packaged artifact to Trovi, its visibility is set as +private, meaning only you can see or launch it. There are multiple options to +change the visibility of the artifact, and you have the option to decide how +visible you want it to be. + +1. **Publish with DOI**: this option allows you to :ref:`publish a version of your + artifact to Zenodo ` and receive a DOI, which you can use to + cite your artifact in, e.g., an academic paper. +2. **Publish without DOI**: this option allows any Chameleon user to find and + launch your artifact. It can be useful if you want to distribute the artifact + widely but do not necessarily with to publish it to Zenodo and get a DOI + for citation. +3. **Share via private link**: this option allows you to share the experiment to + select people, like individual colleagues, advisors, or students. Anybody in + possession of the link can view and launch any version of the artifact. + +To make your artifact shareable, select it in Trovi, click "Share", and check the box before "Enable all users to find and share". + +.. _trovi-roles: + +Assigning Roles to Other Users +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. figure:: ../sharing/sharing_edit_roles.png + :alt: Screenshot of the artifact role menu + :figclass: screenshot + +You can assign roles to other users which allow them to collaborate on your artifacts. +There are two roles: **Collaborator** and **Administrator**. + +**Collaborators** are allowed to edit artifact metadata, upload new versions, +delete old versions, and share private artifacts. + +**Administrators** have full control over the artifact, including +assigning roles to other users. + +Artifact owners cannot have their Adminstrator privileges removed. + +.. _trovi-zenodo: + +Publishing to Zenodo +^^^^^^^^^^^^^^^^^^^^ + +.. attention:: + You can only request a DOI for artifacts uploaded via the Jupyter interface. + You cannot request a DOI for an artifact version uploaded via git. + +Trovi is intended for sharing work in progress with a limited group of "friends +and family". However, once you complete your experiment package you may want to +publish it so that you can reference it from your paper. To do that Chameleon +supports integration with Zenodo, an open-access storage repository backed by +CERN, for permanent artifact hosting. To share your artifact and store it on +Zenodo, go to the "Share" page for the artifact. On the right-hand side you'll +see a list of all versions you've saved. Pick the version you want to publish to +Zenodo and check "Request DOI", then click "Save." + +.. important:: + + Once published, **Zenodo artifacts cannot be deleted** and are additionally + **publicly available**. Your artifact will appear in Trovi in the "Public" + section, and any Chameleon user can access it, as can anybody on the + Internet via Zenodo's own listing. + + If you wish to make your artifact public but don't to publish it, use the + "Publish without DOI" option. With this option it is possible to make the + artifact private later on if you wish; this is not possible when publishing + to Zenodo. + + You can only request a DOI one time per artifact. If you want to update your + experiment files and request a second DOI, you should instead create a new + artifact. + +This also creates a DOI, which you can easily include in your +paper. The artifacts shared on Zenodo also appear on Trovi. + +Importing an artifact +--------------------- + +Instead of creating an artifact inside Jupyterhub, you can package an existing +Git repository into an artifact. When a user launches the artifact, the +contents of the repository will be added to a Jupyter notebook. + +To create an artifact, click "Import Artifact" on the sidebar of Trovi. You are +first asked for the artifact's metadata. At the bottom of the form, there is +a button for "Import from Git." After clicking this, you will need to enter a +git remote URL, and choose which commit to tie the version to. + +To update the artifact, you must create a :ref:`new version `. +This ensures that a given version of your artifact always has the same contents. + +Exporting via git +----------------- + +If you wish to move your code and notebooks outside of your Jupyter notebook, +one option is to export it into a git repository. + +#. Click the "+" button on the top left of your notebook, and choose "Terminal". + +#. Run the command ``cd work``. If there is a specific directory you wish to + export, you can ``cd`` again into it. + +#. Follow the instructions to set up a repository per your git host. For GitHub + see `this document `_. + +#. After the repository is set up, you should be able to commit and push with + the git CLI. \ No newline at end of file diff --git a/source/technical/swift.rst b/source/technical/swift.rst index 852211a..de94a89 100644 --- a/source/technical/swift.rst +++ b/source/technical/swift.rst @@ -1,394 +1,11 @@ -.. _object-store: +:orphan: -============ -Object Store -============ +.. meta:: + :http-equiv=refresh: 0; url=./swift/index.html -Chameleon provides an object store service through the `OpenStack Swift -`_ interface. It is intended to be -used for storing and retrieving data used during experiments, such as input -files needed for your applications, or results produced by your experiments. +This page has moved +=================== -.. hint:: - Chameleon object store service is currently backed by a `Ceph - `_ cluster with more than 2.1 PB of capacity. The data is - replicated, keeping two copies of each object, effectively providing over 1 - PB of storage available to users. This storage capacity will increase as the - project goes on. The replication should provide good availability in case of - hardware failures. However, all copies are kept within the same data center - and are not backed up on a separate system; if you feel that this does not - provide sufficient reliability in your case, you should consider backing up - really critical data externally. +This page has been reorganized. You will be redirected to the new location. -Availability -============ - -You can access the *Object Store* from instances running on |CHI@TACC| and -|CHI@UC|. Each region has its own store, meaning that objects uploaded to one -are not visible to the other. In general you should use the store local to the -region where your instances are running for the best performance. To make it -easier for you to use the *Object Store* client, we installed it in all -appliances supported by Chameleon. Additionally, you can also access the *Object -Store* from the |CHI@TACC| or |CHI@UC| web interfaces under the *Object Store* -panel. - -.. hint:: - `KVM\@TACC `_ users can access the TACC - store by using their |CHI@TACC| :ref:`OpenStack RC file `. - -Objects and Containers -====================== - -*Objects* are equivalent to individual files. They are stored in *Containers*, -which are data structures that can contain multiple *Objects*. When uploading -*Objects*, they must be stored inside of *Containers*. You may perform -operations on individual *Objects* inside Containers, such as downloading or -deleting them. You may also work with entire *Containers* and perform operations -such as downloading an entire *Container*. - -Managing Object Store using the GUI -=================================== - -To access the *Object Store* using the GUI at |CHI@TACC| or |CHI@UC|, use the -navigation sidebar to go to *Project* > *Object Store* > *Containers*. - -.. figure:: swift/containerspage.png - :alt: The Containers page - :figclass: screenshot - - The Containers page - -Working with Containers ------------------------ - -To create a container, click the *+Container* button. This will open the *Create -Container* dialog. - -.. figure:: swift/createcontainer.png - :alt: The Create Container dialog - :figclass: screenshot - - The Create Container dialog - -Choose a unique name of your container and set the visibility to either *Public* -or *Not Public*. When you are finished, click the *Submit* button. You will see -your new *Container* appear in the list on the *Containers* page. - -.. figure:: swift/containerlist.png - :alt: The Container list - :figclass: screenshot - - The Container list - -You may click on a *Container* to see the details and work with *Objects* belong -to it. - -.. figure:: swift/containerdetail.png - :alt: Container details - :figclass: screenshot - - Container details - -.. attention:: - Downloading a container is not available from the GUI. Use the CLI to - download containers. - -You may delete a container by clicking the *Delete* icon in the upper right of -the *Container Detail Panel*. - -.. figure:: swift/containerdelete.png - :alt: The Delete Container button - :figclass: screenshot - - The Delete Container button - -Working with Objects --------------------- - -To upload a local file to a container, click the button with the *Upload* symbol -next to the search bar. - -.. figure:: swift/uploadobject.png - :alt: The Upload button - :figclass: screenshot - - The Upload button - -This will open the *Upload File* dialog. - -.. figure:: swift/uploaddialog.png - :alt: The Upload File dialog - :figclass: screenshot - - The Upload File dialog - -Choose a file to upload from your local file system and give a name to the -object. - -Working with Folders --------------------- - -If you wish to create a *Folder* within your *Container*, click the *+Folder* -button and give a name to your folder in the *Create Folder* dialog. - -.. figure:: swift/createfolder.png - :alt: The Create Folder dialog - :figclass: screenshot - - The Create Folder dialog - -Your new folder will appear in the *Container details*. - -.. figure:: swift/containerwithfolder.png - :alt: A Container with a Folder - :figclass: screenshot - - A Container with a Folder - -You may browse your folder and upload files to it by clicking on the folder. - -.. figure:: swift/containerfolder.png - :alt: A Folder within the Container - :figclass: screenshot - - A Folder within the Container - -.. _object-store-cli: - -Managing Object Store using the CLI -==================================== - -.. tip:: - Reading :ref:`cli` is highly recommanded before continuing on the following - sections. - -In addition to :ref:`cli-installing`, you must also install -``python-swiftclient`` package: - -.. code-block:: bash - - pip install python-swiftclient - -Then, you must set environment variables for your account and project using -:ref:`cli-rc-script`. - -Working with Containers ------------------------ - -To create a *Container*, use the following command: - -.. code-block:: bash - - openstack container create - -.. tip:: - By default, the *Container* created using the above command will not be - visible to the public. - -To view all containers that belong to your project, run: - -.. code-block:: bash - - openstack container list - -.. tip:: - You may use ``--prefix `` as a filter to list the containers whose - name starts with ````. - -To see details of a container, use the command: - -.. code-block:: bash - - openstack container show - -To view a list of objects within a container, use the command: - -.. code-block:: bash - - openstack object list - -To download a container with all the objects belong to it, use the following -command: - -.. code-block:: bash - - openstack container save - -To delete a container and wipe out all the objects belong to it, use the -following command, and **be careful**! - -.. code-block:: bash - - openstack container delete --recursive - -Working with Objects --------------------- - -You may upload a file from your local machine to a container using the following -command: - -.. code-block:: bash - - openstack object create - -.. tip:: - Optionally, you may name the object differently from it's original name in - your local machine by using the ``--name`` parameter. - -To delete an object from a container, run: - -.. code-block:: bash - - openstack object delete - -If you wish to download an individual object directly from a container, use the -command: - -.. code-block:: bash - - openstack object save - -Large object support -^^^^^^^^^^^^^^^^^^^^ - -The Swift CLI only supports objects up to 4GB. Larger objects are supported, -provided they are uploaded in segments. This advanced functionality is only -supported using a separate Swift interface. For a version compatible with -Chameleon's authentication, you need `python-swiftclient >= 3.11.1`, and -to generate and use an :ref:`Application Credential ` - -.. code-block:: bash - - pip install "python-swiftclient>=3.11.1" - -Instead of invoking commands via ``openstack``, you will instead use the -``swift`` command, which supports a ``--segment-size`` parameter, specifying -the segment size in bits. ``--segment-size 4831838208`` is close to the segment -limit of 4GB. - -There is also a ``--changed`` flag, which prevents uploading of the object if -the checksum has not changed: - -.. code-block:: bash - - swift --os-auth-type v3applicationcredential \ - --os-application-credential-id \ - --os-application-credential-secret \ - upload --changed --segment-size 4831838208 \ - - -Working with Folders --------------------- - -There isn't "folders" when you managing the *Object Store* with the CLI. -However, when you create an object, you may use the delimiter ``/`` to specify -the path. - -.. _cc-rclone: - -Mounting Object Store as a File System -====================================== - -.. tip:: - rclone can upload small and large files to the object store, however, - if you have trouble uploading larger objects, you may need to use the - Swift CLI instead. - -When logged into an instance using Chameleon-supported images, such as -`CC-CentOS9-Stream `_ and -`CC-Ubuntu24.04 `_, you will -find a README in the home directory for the `cc` user. The README describes -how to mount containers in the Chameleon Object Store into a directory -called ``cc_my_mounting_point`` in your home directory. Mounts are facilitated -by the `rclone `_ tool. If the directory does not exist, -this directory will be created the first time you mount a container. -Inside the ``cc_my_mounting_point`` directory, you will find directories -that map to containers you've mounted. If there is a directory inside -``cc_my_mounting_point`` that is not mounted it should have a file named -``THIS_IS_NOT_MOUNTED`` in it. Once you mount the container, the file -will no longer be visible until the container is unmounted. - -The tool can mount existing containers in the object store, or create them -if they don't exist. The containers are from the specific site where the -instance is located and only work at sites that have an object store -(currently ``CHI@UC`` and ``CHI@TACC``). For example, instances running at -``CHI@UC`` will interact with the object store also at ``CHI@UC``. You will -not be able to interact with object store data at other sites using this -method. - -.. important:: - - Some older Chameleon-supported images have an outdated mechanism for mounting - the object store using ``cc-cloudfuse``. This mechanism for mounting - the object store is no longer recommended or supported. On older images - you should use the Swift CLI directory to use the object store. - -To mount, use the following command: - -.. code-block:: bash - - cc-mount-object-store start your_container_name - -Now you can access your Chameleon Object Store as your local file system at: -`~/cc_my_mounting_point/your_container_name`. - -You can investigate if a mount is running for a container with: - -.. code-block:: bash - - cc-mount-object-store status your_container_name - -You can also list all running mounts with: - -.. code-block:: bash - - cc-mount-object-store list - -To unmount, use the following command: - -.. code-block:: bash - - cc-mount-object-store stop your_container_name - -.. important:: - **Limitations** - - The primary usage scenario of the ``rclone`` tool is to allow you to - interact with Chameleon Object Store using familiar file system operations. - Because the tool runs on top of an object store, it is important - to understand that not all functionality will behave identically to a regular - file system. - - #. Symbolic links, file permissions, and POSIX file locking operations are - not supported. - - #. Updating an existing file is an expensive operation as it downloads the - entire file to local disk before it can modify the contents. - - #. You can mount from multiple nodes, but there is no synchronization - between nodes regarding writes to Object Storage. - - #. The mounting root directory can only contain directories, as they are - mapped to Object Store containers. - - #. Renaming directories is not allowed. - - #. It keeps an in-memory cache of the directory structure, so it may not be - usable for large file systems. In addition, files added by other - applications will not show up until the cache expires. - - Please keep these limitations in mind when considering the use of this tool - to interact with the object store. - -.. warning:: - The use of ``rclone`` to sync files between your instance - and the object store is a best effort tool. It is the responsibilty - of the user to verify the files sync'd correctly and are valid. - - Given the challenges of mapping files in a file system to an object - store over a network, numerous problems can occur that may impact - the availability of files on the object store. If you attempt - to copy files into the mount point and receive errors, it is - important that you verify the existence and contents of the file - in the object store and not simply assume the file has been - persisted there (even if it is present in the mount point). +If you are not redirected automatically, please visit :doc:`swift/index`. \ No newline at end of file diff --git a/source/technical/swift/index.rst b/source/technical/swift/index.rst new file mode 100644 index 0000000..86ac953 --- /dev/null +++ b/source/technical/swift/index.rst @@ -0,0 +1,55 @@ +.. _object-store: + +============ +Object Store +============ + +Chameleon provides an object store service through the `OpenStack Swift +`_ interface. It is intended to be +used for storing and retrieving data used during experiments, such as input +files needed for your applications, or results produced by your experiments. + +.. hint:: + Chameleon object store service is currently backed by a `Ceph + `_ cluster with more than 2.1 PB of capacity. The data is + replicated, keeping two copies of each object, effectively providing over 1 + PB of storage available to users. This storage capacity will increase as the + project goes on. The replication should provide good availability in case of + hardware failures. However, all copies are kept within the same data center + and are not backed up on a separate system; if you feel that this does not + provide sufficient reliability in your case, you should consider backing up + really critical data externally. + +Availability +============ + +You can access the *Object Store* from instances running on |CHI@TACC| and +|CHI@UC|. Each region has its own store, meaning that objects uploaded to one +are not visible to the other. In general you should use the store local to the +region where your instances are running for the best performance. To make it +easier for you to use the *Object Store* client, we installed it in all +appliances supported by Chameleon. Additionally, you can also access the *Object +Store* from the |CHI@TACC| or |CHI@UC| GUIs under the *Object Store* +panel. + +.. hint:: + `KVM\@TACC `_ users can access the TACC + store by using their |CHI@TACC| :ref:`OpenStack RC file `. + +Objects and Containers +====================== + +*Objects* are equivalent to individual files. They are stored in *Containers*, +which are data structures that can contain multiple *Objects*. When uploading +*Objects*, they must be stored inside of *Containers*. You may perform +operations on individual *Objects* inside Containers, such as downloading or +deleting them. You may also work with entire *Containers* and perform operations +such as downloading an entire *Container*. + +.. toctree:: + :maxdepth: 2 + :caption: Object Store Topics + + swift_gui + swift_cli + swift_mount \ No newline at end of file diff --git a/source/technical/swift/swift_cli.rst b/source/technical/swift/swift_cli.rst new file mode 100644 index 0000000..c2c694f --- /dev/null +++ b/source/technical/swift/swift_cli.rst @@ -0,0 +1,130 @@ +.. _object-store-cli: + +Managing Object Store using the CLI +==================================== + +.. tip:: + Reading :ref:`cli` is highly recommended before continuing on the following + sections. + +In addition to :ref:`cli-installing`, you must also install +``python-swiftclient`` package: + +.. code-block:: bash + + pip install python-swiftclient + +Then, you must set environment variables for your account and project using +:ref:`cli-rc-script`. + +Working with Containers +----------------------- + +To create a *Container*, use the following command: + +.. code-block:: bash + + openstack container create + +.. tip:: + By default, the *Container* created using the above command will not be + visible to the public. + +To view all containers that belong to your project, run: + +.. code-block:: bash + + openstack container list + +.. tip:: + You may use ``--prefix `` as a filter to list the containers whose + name starts with ````. + +To see details of a container, use the command: + +.. code-block:: bash + + openstack container show + +To view a list of objects within a container, use the command: + +.. code-block:: bash + + openstack object list + +To download a container with all the objects belong to it, use the following +command: + +.. code-block:: bash + + openstack container save + +To delete a container and wipe out all the objects belong to it, use the +following command, and **be careful**! + +.. code-block:: bash + + openstack container delete --recursive + +Working with Objects +-------------------- + +You may upload a file from your local machine to a container using the following +command: + +.. code-block:: bash + + openstack object create + +.. tip:: + Optionally, you may name the object differently from it's original name in + your local machine by using the ``--name`` parameter. + +To delete an object from a container, run: + +.. code-block:: bash + + openstack object delete + +If you wish to download an individual object directly from a container, use the +command: + +.. code-block:: bash + + openstack object save + +Large object support +^^^^^^^^^^^^^^^^^^^^ + +The Swift CLI only supports objects up to 4GB. Larger objects are supported, +provided they are uploaded in segments. This advanced functionality is only +supported using a separate Swift interface. For a version compatible with +Chameleon's authentication, you need `python-swiftclient >= 3.11.1`, and +to generate and use an :ref:`Application Credential ` + +.. code-block:: bash + + pip install "python-swiftclient>=3.11.1" + +Instead of invoking commands via ``openstack``, you will instead use the +``swift`` command, which supports a ``--segment-size`` parameter, specifying +the segment size in bits. ``--segment-size 4831838208`` is close to the segment +limit of 4GB. + +There is also a ``--changed`` flag, which prevents uploading of the object if +the checksum has not changed: + +.. code-block:: bash + + swift --os-auth-type v3applicationcredential \ + --os-application-credential-id \ + --os-application-credential-secret \ + upload --changed --segment-size 4831838208 \ + + +Working with Folders +-------------------- + +There isn't "folders" when you managing the *Object Store* with the CLI. +However, when you create an object, you may use the delimiter ``/`` to specify +the path. \ No newline at end of file diff --git a/source/technical/swift/swift_gui.rst b/source/technical/swift/swift_gui.rst new file mode 100644 index 0000000..e10d344 --- /dev/null +++ b/source/technical/swift/swift_gui.rst @@ -0,0 +1,106 @@ +Managing Object Store using the GUI +=================================== + +To access the *Object Store* using the GUI at |CHI@TACC| or |CHI@UC|, use the +navigation sidebar to go to *Project* > *Object Store* > *Containers*. + +.. figure:: containerspage.png + :alt: The Containers page + :figclass: screenshot + + The Containers page + +Working with Containers +----------------------- + +To create a container, click the *+Container* button. This will open the *Create +Container* dialog. + +.. figure:: createcontainer.png + :alt: The Create Container dialog + :figclass: screenshot + + The Create Container dialog + +Choose a unique name of your container and set the visibility to either *Public* +or *Not Public*. When you are finished, click the *Submit* button. You will see +your new *Container* appear in the list on the *Containers* page. + +.. figure:: containerlist.png + :alt: The Container list + :figclass: screenshot + + The Container list + +You may click on a *Container* to see the details and work with *Objects* belong +to it. + +.. figure:: containerdetail.png + :alt: Container details + :figclass: screenshot + + Container details + +.. attention:: + Downloading a container is not available from the GUI. Use the CLI to + download containers. + +You may delete a container by clicking the *Delete* icon in the upper right of +the *Container Detail Panel*. + +.. figure:: containerdelete.png + :alt: The Delete Container button + :figclass: screenshot + + The Delete Container button + +Working with Objects +-------------------- + +To upload a local file to a container, click the button with the *Upload* symbol +next to the search bar. + +.. figure:: uploadobject.png + :alt: The Upload button + :figclass: screenshot + + The Upload button + +This will open the *Upload File* dialog. + +.. figure:: uploaddialog.png + :alt: The Upload File dialog + :figclass: screenshot + + The Upload File dialog + +Choose a file to upload from your local file system and give a name to the +object. + +Working with Folders +-------------------- + +If you wish to create a *Folder* within your *Container*, click the *+Folder* +button and give a name to your folder in the *Create Folder* dialog. + +.. figure:: createfolder.png + :alt: The Create Folder dialog + :figclass: screenshot + + The Create Folder dialog + +Your new folder will appear in the *Container details*. + +.. figure:: containerwithfolder.png + :alt: A Container with a Folder + :figclass: screenshot + + A Container with a Folder + +You may browse your folder and upload files to it by clicking on the folder. + +.. figure:: containerfolder.png + :alt: A Folder within the Container + :figclass: screenshot + + A Folder within the Container \ No newline at end of file diff --git a/source/technical/swift/swift_mount.rst b/source/technical/swift/swift_mount.rst new file mode 100644 index 0000000..a25a973 --- /dev/null +++ b/source/technical/swift/swift_mount.rst @@ -0,0 +1,108 @@ +.. _cc-rclone: + +Mounting Object Store as a File System +====================================== + +.. tip:: + rclone can upload small and large files to the object store, however, + if you have trouble uploading larger objects, you may need to use the + Swift CLI instead. + +When logged into an instance using Chameleon-supported images, such as +`CC-CentOS9-Stream `_ and +`CC-Ubuntu24.04 `_, you will +find a README in the home directory for the `cc` user. The README describes +how to mount containers in the Chameleon Object Store into a directory +called ``cc_my_mounting_point`` in your home directory. Mounts are facilitated +by the `rclone `_ tool. If the directory does not exist, +this directory will be created the first time you mount a container. +Inside the ``cc_my_mounting_point`` directory, you will find directories +that map to containers you've mounted. If there is a directory inside +``cc_my_mounting_point`` that is not mounted it should have a file named +``THIS_IS_NOT_MOUNTED`` in it. Once you mount the container, the file +will no longer be visible until the container is unmounted. + +The tool can mount existing containers in the object store, or create them +if they don't exist. The containers are from the specific site where the +instance is located and only work at sites that have an object store +(currently ``CHI@UC`` and ``CHI@TACC``). For example, instances running at +``CHI@UC`` will interact with the object store also at ``CHI@UC``. You will +not be able to interact with object store data at other sites using this +method. + +.. important:: + + Some older Chameleon-supported images have an outdated mechanism for mounting + the object store using ``cc-cloudfuse``. This mechanism for mounting + the object store is no longer recommended or supported. On older images + you should use the Swift CLI directory to use the object store. + +To mount, use the following command: + +.. code-block:: bash + + cc-mount-object-store start your_container_name + +Now you can access your Chameleon Object Store as your local file system at: +`~/cc_my_mounting_point/your_container_name`. + +You can investigate if a mount is running for a container with: + +.. code-block:: bash + + cc-mount-object-store status your_container_name + +You can also list all running mounts with: + +.. code-block:: bash + + cc-mount-object-store list + +To unmount, use the following command: + +.. code-block:: bash + + cc-mount-object-store stop your_container_name + +.. important:: + **Limitations** + + The primary usage scenario of the ``rclone`` tool is to allow you to + interact with Chameleon Object Store using familiar file system operations. + Because the tool runs on top of an object store, it is important + to understand that not all functionality will behave identically to a regular + file system. + + #. Symbolic links, file permissions, and POSIX file locking operations are + not supported. + + #. Updating an existing file is an expensive operation as it downloads the + entire file to local disk before it can modify the contents. + + #. You can mount from multiple nodes, but there is no synchronization + between nodes regarding writes to Object Storage. + + #. The mounting root directory can only contain directories, as they are + mapped to Object Store containers. + + #. Renaming directories is not allowed. + + #. It keeps an in-memory cache of the directory structure, so it may not be + usable for large file systems. In addition, files added by other + applications will not show up until the cache expires. + + Keep these limitations in mind when considering the use of this tool + to interact with the object store. + +.. warning:: + The use of ``rclone`` to sync files between your instance + and the object store is a best effort tool. It is the responsibilty + of the user to verify the files sync'd correctly and are valid. + + Given the challenges of mapping files in a file system to an object + store over a network, numerous problems can occur that may impact + the availability of files on the object store. If you attempt + to copy files into the mount point and receive errors, it is + important that you verify the existence and contents of the file + in the object store and not simply assume the file has been + persisted there (even if it is present in the mount point). \ No newline at end of file diff --git a/source/user/federation.rst b/source/user/federation.rst index 091eb7a..02ef6e9 100644 --- a/source/user/federation.rst +++ b/source/user/federation.rst @@ -2,9 +2,9 @@ .. _federation: -================================ -Sign in with federated identity -================================ +=============================== +Sign In with Federated Identity +=============================== Federated login enables users to use a single set of credentials to log into many different services. For example, federated login allows you to use your @@ -15,12 +15,13 @@ to sign in once and use multiple services. Chameleon uses `Globus Auth `_, a popular authentication service, to implement federated login and federates with entities supported by -Globus. Users can sign in using their existing institutional account if their -institution is an `InCommon`_ member, use their Google account, or create a +Globus. **We strongly recommend using federated login as it's the simplest and most +convenient way to access Chameleon.** Users can sign in using their existing institutional +account if their institution is an `InCommon`_ member, use their Google account, or create a `Globus ID `_ tied to an email and password that they provide. In addition, Chameleon also federates with the TAS entity. -Logging in +Logging In ========== To log in to the Chameleon user portal, where you can manage your projects, @@ -30,7 +31,7 @@ user profile, and submit |Help Desk| tickets, use the "Log in" button. :alt: Login button for the Chameleon user portal :figclass: screenshot -To log in to any of the testbed sites (|CHI@TACC|, |CHI@UC|, |KVM@TACC|) or the +To log in to any of the testbed sites (|CHI@TACC|, |CHI@UC|, |CHI@NCAR|, |CHI@Edge|, |KVM@TACC|) or the :ref:`Jupyter environment `, just click their item in the "Experiment" dropdown on |Home|. The login process is triggered automatically. @@ -50,19 +51,39 @@ automatically. :alt: Links for accessing testbed sites and the Jupyter interface :figclass: screenshot -You will be taken to a Single Sign On (SSO) page with a few options for how -to authenticate. The options are: +You will be taken to a Single Sign On (SSO) page with several authentication options. +**We recommend choosing "Sign in via federated identity" for the easiest experience.** -- **Sign in via federated identity**: this option allows you to re-use your - existing institution, research lab, or university credentials to log in to - Chameleon. It requires your host institution to participate in the `InCommon`_ - federation. -- **Google**: this option allows you to sign in with any Google account. -- **ORCiD**: you can also sign in with a valid ORCiD account. -- **TAS**: sign in via the TAS entity. -- For the time being, if you are an existing user, you can also leave SSO and - go back to the old sign in page, where you sign in with your Chameleon - username and password. +Authentication options +---------------------- + +- **Sign in via federated identity** ⭐ **RECOMMENDED**: This is the simplest way to access + Chameleon! Use your existing institution, research lab, or university credentials to log in. + This requires your host institution to participate in the `InCommon`_ federation. Most major + universities and research institutions are already members. + +- **Google**: Sign in with any Google account if your institution isn't part of InCommon or + if you prefer using your Google credentials. + +- **ORCiD**: Sign in with a valid ORCiD account for research credential integration. + +- **TAS**: Sign in via the TAS entity (primarily for TACC users). + +.. tip:: + **New users**: If you're unsure which option to choose, try "Sign in via federated identity" + first - most universities and research institutions support this method, making it the fastest + way to get started with Chameleon. + +.. warning:: + You may not find your institution on Globus. If so, you can still create an + account with another identity, such as GitHub, Google, or ORCiD. However, + we encourage users to sign up using their institution, as it helps the + Chameleon operators verify user identity, which is an essential step + to getting Principal Investigator status on the testbed. + +.. note:: + Chameleon has fully migrated to federated identity authentication. Legacy + Chameleon username/password accounts are no longer supported. .. figure:: federation/sso-login.png :alt: Single Sign On (SSO) portal login @@ -70,14 +91,40 @@ to authenticate. The options are: The Single Sign On (SSO) portal login page. -Legacy users (created before November 2020) -=========================================== +Terms and Conditions +==================== + +When creating an account, you will be asked to accept `terms and conditions +`_ of use, +note that as part of those terms and conditions you are requested to +acknowledge Chameleon in publications produced using the testbed. See our FAQ +for information on `how to reference Chameleon in your publications +`_ +and the suggested `acknowledgement text +`_. + +Troubleshooting Login Issues +============================ + +If you experience difficulty logging in, try these solutions: -Users who signed up for a Chameleon account prior to November 2020 must migrate -their account to use federated identity. Refer to the following step-by-step -guide for more details. +**Common Authentication Issues:** -.. toctree:: - :maxdepth: 1 +- **Institutional credentials not working**: Ensure your institutional credentials are + up-to-date and correctly linked to your Chameleon account +- **Account linking problems**: Contact the :doc:`Help Desk ` to verify your + identity and manually relink accounts if needed +- **Browser issues**: Clear your browser cache and cookies, then try logging in again +- **Password reset problems**: Use the password reset links provided in the portal - federation/federation_migration +**Getting Help:** + +For persistent login issues, contact our :doc:`Help Desk ` with details about: +- Which authentication method you're trying to use +- Any error messages you're seeing +- Your institutional affiliation (if using federated login) + +.. note:: + Users who had legacy Chameleon accounts (created before November 2020) successfully + completed migration to federated identity in 2021. All current authentication uses + federated identity providers. diff --git a/source/user/help.rst b/source/user/help.rst index 5a881f5..6a5fc7a 100644 --- a/source/user/help.rst +++ b/source/user/help.rst @@ -1,18 +1,32 @@ .. _help: -============= -Getting help -============= +============ +Getting Help +============ -Mailing lists -============= +Chameleon offers several support channels to assist users with their +questions and issues. Depending on the nature of your inquiry, you can choose +from the following options: -You can communicate with other Chameleon users by sending email to -users@chameleoncloud.org. We also use this mailing list to communicate minor -announcements or provide early access to some new hardware or features. All -Chameleon users are registered by default: we recommend that you stay -registered. However, if you really want to opt-out, you can do so via `your user -profile `_. +Community Forum +=============== + +.. note:: + For immediate assistance or urgent issues, use the Help Desk rather + than the community forum. The Help Desk ensures your request receives prompt + attention from Chameleon staff. + +Join the `Chameleon Community Forums `_ to connect +with other users, ask questions, and share experiences. The forum provides a space for: + +* User discussions and community support +* Non-urgent questions about using Chameleon +* Sharing best practices and experimental approaches +* Announcements of new features and capabilities + +The forum is actively monitored by Chameleon staff and experienced community members +who provide assistance on a best-effort basis. For immediate help or urgent issues, +use the Help Desk instead. .. _outages-page: @@ -45,17 +59,10 @@ To create a new help ticket, click the `+Create a new ticket `_ button and fill in the form. A system administrator will respond to your ticket within 3 business days. -.. note:: - An alternative way of asking for help is sending an email to the `Chameleon - users mailing list `_, especially when the - Help Desk is down or you think it's something worth sharing with all - Chameleon users. A system administrator will reply to your email and, if - necessary, create a ticket for you. - .. _webinars-page: Webinars ======== The `Webinars `_ page provides a -list of upcoming webinars for Chameleon user training. +list of upcoming webinars for Chameleon user training. \ No newline at end of file diff --git a/source/user/pi_eligibility.rst b/source/user/pi_eligibility.rst index 8d77156..29dab8e 100644 --- a/source/user/pi_eligibility.rst +++ b/source/user/pi_eligibility.rst @@ -1,8 +1,11 @@ .. _pi-eligibility: -=============== -PI eligibility -=============== +============== +PI Eligibility +============== + +Overview +======== Chameleon PIs carry significant responsibility for the users on their projects; we therefore limit PI eligibility to individuals from the following groups: @@ -14,3 +17,29 @@ Chameleon PIs carry significant responsibility for the users on their projects; - **State educational offices or organizations and local school districts** may submit allocation requests intended to broaden the impact, accelerate the pace, and increase the effectiveness of improvements in science, mathematics, and engineering education in both K-12 and post-secondary levels. A teacher or educator at an accredited public or private K-12 school is eligible to apply for an allocation as PI. We do occasionally provide case-by-case exceptions to this guideline in well-justified cases. + +How to Request PI Eligibility +============================= + +If you meet the eligibility criteria above, you can request PI status on Chameleon by following these steps: + +1. **Complete your user profile**: Ensure your `Chameleon profile `_ + is complete with your institutional affiliation and contact information. + +2. **Request PI eligibility**: In your profile page, click the "Edit Profile" button, then check the + "Request PI Eligibility" checkbox and save your profile. + +3. **Wait for review**: Chameleon PI status requests are typically reviewed and approved within + one business day. + +4. **Create your first project**: Once approved, you can create a new project by going to the + `Projects Dashboard `_ and clicking + "Create a Project." + +.. tip:: + **Students**: If you don't meet the PI eligibility criteria (most graduate students do not), + ask your faculty advisor or research supervisor to request PI status and add you to their project. + This is the most common approach for student researchers. + +For questions about PI eligibility or if you believe you have a special case that warrants +consideration, contact our :doc:`Help Desk `. diff --git a/source/user/profile.rst b/source/user/profile.rst index 378faa1..3c02849 100644 --- a/source/user/profile.rst +++ b/source/user/profile.rst @@ -1,8 +1,8 @@ .. _profile-page: -============= -User profile -============= +============ +User Profile +============ The `Profile `_ page in the Dashboard allows you to manage your biographical information and membership to diff --git a/source/user/project.rst b/source/user/project.rst index 61425ad..7ec13bc 100644 --- a/source/user/project.rst +++ b/source/user/project.rst @@ -1,17 +1,40 @@ .. _project-management: ================== -Project management +Project Management ================== -Project management tasks, such as adding users to your project or requesting a -renewal, is performed through the portal at https://chameleoncloud.org. After +Overview +======== + +Projects are the fundamental organizing unit for research on Chameleon. Each project provides: + +- **Resource allocations** measured in Service Units (SUs) for computational time +- **User management** with role-based access control (PI, Manager, Member) +- **Resource isolation** with dedicated security groups, networks, and storage +- **Usage tracking** and billing management for fair resource sharing + +All project management tasks are performed through the `Chameleon portal `_. After you have `registered `_ and verified your email address, you may `login to the portal `_. Once logged in, you should be at *Dashboard* page automatically. If not, you can access your *Dashboard* via the dropdown list on top right of the screen. +**Quick Navigation** + +- :ref:`Dashboard overview ` - View projects and tickets +- :ref:`Creating a project ` - Start a new research project +- :ref:`Managing users ` - Add/remove team members and set roles +- :ref:`Allocation management ` - Request renewals and recharges +- :ref:`Publications tracking ` - Maintain research output records + +**Related Documentation** + +- :doc:`PI eligibility requirements ` - Criteria for creating projects +- :doc:`User authentication ` - Login and account configuration +- :doc:`Getting help ` - Support channels and community resources + .. _dashboard-page: Dashboard @@ -60,7 +83,7 @@ Creating a Project To create a project, click the *+Create a Project* button. After filling out and submit the request form, a system administrator will review your request and -notify you once your project get approved. Project durations are six months with +notify you once your project gets approved. Project durations are six months with a default allocation of 20,000 :ref:`service-units`. .. figure:: project/createproject.png @@ -71,19 +94,19 @@ a default allocation of 20,000 :ref:`service-units`. .. _service-units: Service Units -------------- +~~~~~~~~~~~~~~ One Service Unit (SU) is equivalent to one hour of usage of one allocatable resource (physical hosts, network segments, or floating IPs). For example, a reservation for 5 Skylake compute nodes for 8 hours would use 40 SUs. However, for certain types of resources, more SUs will be charged. For more details about -allocation charges, please see `here +allocation charges, see `here `_. .. _project-details: Project Details ---------------- +~~~~~~~~~~~~~~~ Clicking on a project from either the :ref:`dashboard-page` main page or the :ref:`projects-page` page will allow you to manage one of your approved @@ -115,7 +138,9 @@ project start and end dates, current *Service Unit* usage and request a allocation row, and then click *Recharge/Extend Allocation*. When requesting renewal or recharge of the allocations, we may ask you to update your :ref:`publications dashboard `, so -keeping it up to date now can save you time later! +keeping it up to date now can save you time later! For questions about allocation +management, visit our `Community Forum `_ or +contact the :doc:`Help Desk `. .. _view-charge: @@ -141,9 +166,9 @@ Manage Publications -------------------- To add publications to a project, click the *Add Publications* button in the -:ref:`project-details` page. Please enter the publications in BibTex format. All +:ref:`project-details` page. Enter the publications in BibTex format. All regular BibTex publication types are supported. If you can provide a link, -please enter as *note* or *howpublished* using the url package. +enter as *note* or *howpublished* using the url package. To manage the publications you have entered, use the *Publications Dashboard*. @@ -237,8 +262,8 @@ To add or remove users of a *Project*, use the *Project Members* section in the You may add a user to your project by filling out their username or email address and clicking the *Add user* button. While each user has their own Chameleon User account independent of your project, they may be added to one or -more projects. Being a user of a *Project* **does not** require a :ref:`PI -eligibility `. +more projects. Being a user of a *Project* **does not** require PI eligibility +(see our :doc:`PI eligibility guide ` for details on project creation requirements). You may remove a user from your project by locating the user in the user list; clicking the *gear* button at the end of the row; and clicking *Remove user*.