From 02aff58cc98fe4cf187977ebaa41675d2bcf194d Mon Sep 17 00:00:00 2001 From: "max@fused.io" Date: Tue, 11 Feb 2025 10:05:20 +0100 Subject: [PATCH] Updating Package Management to use `mount/` --- .../Package_Management_debug_catona.py | 126 ++++++++++++++++++ .../Package_Management_debug_catona/README.MD | 11 ++ .../Package_Management_debug_catona/meta.json | 94 +++++++++++++ 3 files changed, 231 insertions(+) create mode 100644 community/max/Package_Management_debug_catona/Package_Management_debug_catona.py create mode 100644 community/max/Package_Management_debug_catona/README.MD create mode 100644 community/max/Package_Management_debug_catona/meta.json diff --git a/community/max/Package_Management_debug_catona/Package_Management_debug_catona.py b/community/max/Package_Management_debug_catona/Package_Management_debug_catona.py new file mode 100644 index 0000000..7a91e32 --- /dev/null +++ b/community/max/Package_Management_debug_catona/Package_Management_debug_catona.py @@ -0,0 +1,126 @@ +import streamlit as st +import time # This should be on top if you want the spinner to happen +import fused +st.title("Install a Package") +is_loggedin=False +try: + name = fused.api.whoami()['name'] + st.success(f'You are successfully logged in as {name}.') + is_loggedin=True +except: + st.error('You are currently not logged in to Fused.') + +if 'button1' not in st.session_state: + st.session_state.button1 = True +if 'button2' not in st.session_state: + st.session_state.button2 = True +if 'expanded' not in st.session_state: + st.session_state.expanded = True + +def click_button(button_arg): + st.session_state[button_arg] = False + +@fused.udf +def list_folders(path="/mnt/cache/envs/"): + import os + import pandas as pd + if os.path.exists(path): + folders = [f for f in os.listdir(path)] + return pd.DataFrame(folders, columns=['Environments']) + else: + return pd.DataFrame({'error': ["Path does not exist"]}) + +@fused.udf +def pip_freeze(env='testxenv', mnt_path="/mnt/cache/envs/"): + import pandas as pd + utils = fused.load('https://github.com/fusedio/udfs/tree/d0f4a86/public/common/').utils + r = utils.run_cmd( + f"{mnt_path}{env}/bin/python -m pip freeze", communicate=True + ) + df = pd.DataFrame({'status': [str(i.decode('utf-8')) for i in r]}) + return df + +@fused.udf +def udf(name='numpy', env='testxenv', mnt_path="/mnt/cache/envs/", packages_path="/lib/python3.11/site-packages"): + import pandas as pd + import loguru + utils = fused.load('https://github.com/fusedio/udfs/tree/d0f4a86/public/common/').utils + # install_module = utils.install_module + def install_module( + name, + env="testxenv", + mnt_path="/mnt/cache/envs/", + packages_path="/lib/python3.11/site-packages", + ): + import os + import sys + path = f"{mnt_path}{env}{packages_path}" + sys.path.append(path) + if not os.path.exists(path): + utils.run_cmd(f"python3.11 -m venv {mnt_path}{env}", communicate=True) + return utils.run_cmd( + f"{mnt_path}{env}/bin/python -m pip install {name}", communicate=True + ) + + r = install_module(name=name, env=env, mnt_path=mnt_path) + loguru.logger.info(r) + # print(r) + df = pd.DataFrame({'status': [str(i.decode('utf-8')) for i in r]}) + df['status'] = df['status'] + return df + + +if is_loggedin: + # User Inputs + name = st.text_input("Module Name", value="numpy") + env = st.text_input("Environment", value="testxenv") + mnt_path = st.text_input("Mount Path", value="/mount/envs/") + # with st.expander(f"See all Environment options in {mnt_path}", expanded=False): + place_holder0 = st.empty() + x = place_holder0.button(f"Environments Details",on_click=click_button, args=['button2']) + if x: + with place_holder0.status(f"Environments in `{mnt_path}`", expanded=st.session_state.expanded): + st.session_state.expanded=True + time.sleep(0.01) + from datetime import datetime + cache_id = datetime.now() + st.write(fused.run(list_folders, path=mnt_path, cache_id=cache_id)) + st.markdown(f'**List of packages in `{env}`**') + try: + result = fused.run(pip_freeze(env=env, mnt_path=mnt_path, cache_id=cache_id)) + success =st.success(f'{result.values[0][0]}') + except: + success =st.warning(f'{mnt_path}{env} is empty or not accessable.') + # mode = st.radio("Execution Mode", ('Real-time', 'Batch')) + place_holder = st.empty() + place_holder.button('Submit', on_click=click_button, disabled= not st.session_state.button1, args=['button1']) + if not st.session_state.button1: + st.session_state.button1=True + mode='Batch' + if mode == 'Batch': + with place_holder.status(f"Install {name}", expanded=True): + time.sleep(0.01) + result = udf(name=name, env=env, mnt_path=mnt_path).run_remote() + st.session_state.button1=True + job_url = fused.options.base_url.replace('server/v1',f'job_status/{result.job_id}') + st.markdown(f"[job_status]({job_url})") + else: + with place_holder.status(f"Install {name} (realtime)", expanded=True): + time.sleep(0.01) + result = fused.run(udf(name=name, env=env, mnt_path=mnt_path, cache_id=cache_id)) + st.session_state.button1=True + success = result.values[0][0] + error = result.values[1][0] + if success: + st.success(success) + if error: + st.error(error) + + st.markdown("To use this package add this inside your udf:") + + st.code(f""" + import sys; + sys.path.append(f"/mount/envs/{env}/lib/python3.11/site-packages/") + """) + st.session_state.button1=True + diff --git a/community/max/Package_Management_debug_catona/README.MD b/community/max/Package_Management_debug_catona/README.MD new file mode 100644 index 0000000..9de40b4 --- /dev/null +++ b/community/max/Package_Management_debug_catona/README.MD @@ -0,0 +1,11 @@ + +

UDF preview image

+ + +### Install a Package +This example shows how to install a python package in Fused. +To use this package add this inside your udf: + +import sys; +env_name='testxenv'; +sys.path.append(f"/mnt/cache/envs/{env_name}/lib/python3.13/site-packages") diff --git a/community/max/Package_Management_debug_catona/meta.json b/community/max/Package_Management_debug_catona/meta.json new file mode 100644 index 0000000..06cfa3e --- /dev/null +++ b/community/max/Package_Management_debug_catona/meta.json @@ -0,0 +1,94 @@ +{ + "version": "0.0.3", + "job_config": { + "version": "0.0.3", + "name": null, + "steps": [ + { + "type": "udf", + "udf": { + "type": "geopandas_v2", + "name": "Package_Management_debug_catona", + "entrypoint": "udf", + "parameters": {}, + "metadata": { + "fused:vizConfig": { + "tileLayer": { + "@@type": "TileLayer", + "minZoom": 0, + "maxZoom": 19, + "tileSize": 256 + }, + "rasterLayer": { + "@@type": "BitmapLayer", + "pickable": true + }, + "vectorLayer": { + "@@type": "GeoJsonLayer", + "stroked": true, + "filled": false, + "pickable": true, + "lineWidthMinPixels": 1, + "pointRadiusMinPixels": 1, + "getLineColor": { + "@@function": "colorContinuous", + "attr": "value", + "domain": [ + 0, + 10 + ], + "colors": "Peach", + "nullColor": [ + 184, + 184, + 184 + ] + }, + "getFillColor": [ + 208, + 208, + 208, + 40 + ] + } + }, + "fused:udfType": "app", + "fused:slug": "Package_Management_debug_catona", + "fused:name": "Package_Management_debug_catona", + "fused:id": null, + "fused:gitUrl": "https://github.com/fusedlabs/fusedudfs/tree/4b60f4acdf3ea8a69cfb7b850c7a9101bca58ffd/Package_Management/", + "fused:gitShortUrl": "https://github.com/fusedlabs/fusedudfs/tree/4b60f4a/Package_Management/", + "fused:gitPath": "Package_Management", + "fused:gitRef": "4b60f4acdf3ea8a69cfb7b850c7a9101bca58ffd", + "fused:gitAuthorNames": [ + "Sina Kashuk", + null + ], + "fused:gitAuthorUsernames": [ + "kashuk", + "fused-io-staging[bot]" + ], + "fused:gitAuthorUrls": [ + "https://github.com/kashuk", + "https://github.com/apps/fused-io-staging" + ], + "fused:gitAuthorAvatarUrls": [ + "https://avatars.githubusercontent.com/u/12432685?v=4", + "https://avatars.githubusercontent.com/in/885821?v=4" + ], + "fused:gitLastModified": "2025-02-10T10:07:24.714Z", + "fused:explorerTab": "team", + "fused:gitRepo": "fusedlabs/fusedudfs", + "fused:gitPullRequestBranch": "max-patch-2cc0de-Updating-path-2nd-try-", + "fused:gitPullRequestLink": "https://github.com/fusedlabs/fusedudfs/pull/546", + "fused:assetUrl": "https://miro.medium.com/v2/resize:fit:1160/format:webp/0*Kt5_0uGLlCFAgbt6.png", + "fused:description": "### Install a Package\nThis example shows how to install a python package in Fused.\nTo use this package add this inside your udf:\n\nimport sys;\nenv_name='testxenv';\nsys.path.append(f\"/mnt/cache/envs/{env_name}/lib/python3.13/site-packages\")\n" + }, + "source": "Package_Management_debug_catona.py", + "headers": [] + } + } + ], + "metadata": null + } +} \ No newline at end of file