Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
282 changes: 282 additions & 0 deletions template/v4/v4.6/Dockerfile

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"python.terminal.activateEnvironment": false,
"python.defaultInterpreterPath": "/opt/conda/bin/python"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{
"extensions.autoUpdate": false
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
https://open-vsx.org/api/ms-toolsai/jupyter/2024.5.0/file/ms-toolsai.jupyter-2024.5.0.vsix
https://open-vsx.org/api/ms-python/python/2026.2.0/file/ms-python.python-2026.2.0.vsix
https://open-vsx.org/api/amazonwebservices/aws-toolkit-vscode/3.101.0/file/amazonwebservices.aws-toolkit-vscode-3.101.0.vsix
https://open-vsx.org/api/amazonwebservices/amazon-q-vscode/2.0.0/file/amazonwebservices.amazon-q-vscode-2.0.0.vsix
4 changes: 4 additions & 0 deletions template/v4/v4.6/dirs/etc/code-editor/extensions.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
https://open-vsx.org/api/ms-toolsai/jupyter/2024.5.0/file/ms-toolsai.jupyter-2024.5.0.vsix
https://open-vsx.org/api/ms-python/python/2023.20.0/file/ms-python.python-2023.20.0.vsix
https://open-vsx.org/api/amazonwebservices/aws-toolkit-vscode/3.101.0/file/amazonwebservices.aws-toolkit-vscode-3.101.0.vsix
https://open-vsx.org/api/amazonwebservices/amazon-q-vscode/2.0.0/file/amazonwebservices.amazon-q-vscode-2.0.0.vsix
6 changes: 6 additions & 0 deletions template/v4/v4.6/dirs/etc/conda/.condarc
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
envs_dirs:
- ~/.conda/envs
- /opt/conda/envs
pkgs_dirs:
- ~/.conda/pkgs
- /opt/conda/pkgs
31 changes: 31 additions & 0 deletions template/v4/v4.6/dirs/etc/jupyter/jupyter_server_config.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
# Default Jupyter server config
# Note: those config can be overridden by user-level configs.

c.ServerApp.terminado_settings = {"shell_command": ["/bin/bash"]}
c.ServerApp.tornado_settings = {"compress_response": True}

# Do not delete files to trash. Instead, permanently delete files.
c.FileContentsManager.delete_to_trash = False

# Allow deleting non-empty directory via file browser. Related documentation:
# https://github.com/jupyter-server/jupyter_server/blob/main/jupyter_server/services/contents/filemanager.py#L125-L129
c.FileContentsManager.always_delete_dir = True

# Enable `allow_hidden` by default, so hidden files are accessible via Jupyter server
# Related documentation: https://jupyterlab.readthedocs.io/en/stable/user/files.html#displaying-hidden-files
c.ContentsManager.allow_hidden = True

# This will set the LanguageServerManager.extra_node_roots setting if amazon_sagemaker_sql_editor exists in the
# environment. Ignore otherwise, don't fail the JL server start
# Related documentation: https://jupyterlab-lsp.readthedocs.io/en/v3.4.0/Configuring.html
try:
import os

module = __import__("amazon_sagemaker_sql_editor")
module_location = os.path.dirname(module.__file__)
c.LanguageServerManager.extra_node_roots = [f"{module_location}/sql-language-server"]
except:
pass

# Set Kiro as the default AI persona
c.PersonaManager.default_persona_id = "jupyter-ai-personas::jupyter_ai_acp_client::KiroAcpPersona"
114 changes: 114 additions & 0 deletions template/v4/v4.6/dirs/etc/patches/apply_patches.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
#!/bin/bash

set -eux

# Function to compare version numbers
# Returns 0 if version1 >= version2, 1 otherwise
version_gte() {
local version1="$1"
local version2="$2"

# Handle empty versions
[ -z "$version1" ] && return 1
[ -z "$version2" ] && return 0

# If versions are identical, return true
[ "$version1" = "$version2" ] && return 0

# Use sort -V (version sort) to compare versions
# Check if version1 comes after version2 in version sort
local sorted=$(printf '%s\n%s\n' "$version1" "$version2" | sort -V)
local first_line=$(echo "$sorted" | head -n1)

# If version2 comes first in sort, then version1 >= version2
[ "$first_line" = "$version2" ]
}

get_package_version() {
local package_name="$1"

# Try to get version using pip show
local pkg_version=$(pip show "$package_name" 2>/dev/null | grep "Version:" | cut -d' ' -f2)

if [ -z "$pkg_version" ]; then
# Try using conda list as fallback
pkg_version=$(conda list "$package_name" 2>/dev/null | grep "^$package_name " | awk '{print $2}' | head -n1)
fi

echo "$pkg_version"
}

# NOTE: Consider removing these patches entirely if all non-deprecating SMD versions
# have package versions larger than the specified thresholds, as the patches would
# no longer be needed.
should_skip_patch() {
local patch_file="$1"
local patch_basename=$(basename "$patch_file")

# Check if patch filename contains "fix-ipython-display"
if [[ "$patch_basename" == *"fix-ipython-display"* ]]; then
# Skip this patch if hdijupyterutils >= 0.23.0
local hdijupyterutils_version=$(get_package_version "hdijupyterutils")
if [ -n "$hdijupyterutils_version" ]; then
if version_gte "$hdijupyterutils_version" "0.23.0"; then
echo "Skipping $patch_basename: hdijupyterutils version $hdijupyterutils_version >= 0.23"
return 0
fi
fi
fi

# Check if patch filename contains "fix-boto3-endpoints"
if [[ "$patch_basename" == *"fix-boto3-endpoints"* ]]; then
# Skip this patch if botocore >= 1.37.17
local botocore_version=$(get_package_version "botocore")
if [ -n "$botocore_version" ]; then
if version_gte "$botocore_version" "1.37.17"; then
echo "Skipping $patch_basename: botocore version $botocore_version >= 1.37.17"
return 0
fi
fi
fi

return 1
}

# Check if parameter is provided
if [ $# -ne 1 ]; then
echo "Usage: $0 [smus|smus-code-editor]"
exit 1
fi

# Validate parameter
case "$1" in
"smus")
bash "/etc/patches/smus-script/replace-job-with-schedule.sh"
PATCH_DIR="/etc/patches/smus"
;;
"smus-code-editor")
PATCH_DIR="/etc/patches/smus-code-editor"
;;
*)
echo "Error: Parameter must be either 'smus' or 'smus-code-editor'"
exit 1
;;
esac

# Check if patch directory exists
if [ ! -d "$PATCH_DIR" ]; then
echo "Error: Patch directory $PATCH_DIR does not exist"
exit 1
fi

# Patch files can be generated via "diff -u /path/to/original_file /path/to/new_file > XXX_bad_package.patch"
# See https://www.thegeekstuff.com/2014/12/patch-command-examples/
for PATCHFILE in "$PATCH_DIR"/*.patch; do
[ -f "$PATCHFILE" ] || continue

# Check if this patch should be skipped due to version constraints
if should_skip_patch "$PATCHFILE"; then
continue
fi

echo "Applying $PATCHFILE"
(cd "/opt/conda" && patch --strip=3 < "$PATCHFILE")
done
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
--- /opt/conda/lib/python3.12/site-packages/sparkmagic/livyclientlib/command.py
+++ /opt/conda/lib/python3.12/site-packages/sparkmagic/livyclientlib/command.py
@@ -54,6 +54,7 @@
try:
session.wait_for_idle()
data = {"code": self.code}
+ data["kind"] = session.kind
response = session.http_client.post_statement(session.id, data)
statement_id = response["id"]
output = self._get_statement_output(session, statement_id)

Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
--- /opt/conda/lib/python3.12/site-packages/sparkmagic/utils/utils.py
+++ /opt/conda/lib/python3.12/site-packages/sparkmagic/utils/utils.py
@@ -94,15 +94,7 @@


def get_sessions_info_html(info_sessions, current_session_id):
- html = (
- """<table>
-<tr><th>ID</th><th>YARN Application ID</th><th>Kind</th><th>State</th><th>Spark UI</th><th>Driver log</th><th>User</th><th>Current session?</th></tr>"""
- + "".join(
- [session.get_row_html(current_session_id) for session in info_sessions]
- )
- + "</table>"
- )
-
+ html = ""
return html
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
--- /opt/conda/lib/python3.12/site-packages/hdijupyterutils/ipythondisplay.py
+++ /opt/conda/lib/python3.12/site-packages/hdijupyterutils/ipythondisplay.py
@@ -1,4 +1,4 @@
-from IPython.core.display import display, HTML
+from IPython.display import display, HTML
from IPython import get_ipython
import sys
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
--- /opt/conda/lib/python3.12/site-packages/botocore/data/endpoints.json
+++ /opt/conda/lib/python3.12/site-packages/botocore/data/endpoints.json
@@ -5404,6 +5404,9 @@
"ap-northeast-3" : {
"hostname" : "datazone.ap-northeast-3.api.aws"
},
+ "ap-south-1" : {
+ "hostname" : "datazone.ap-south-1.api.aws"
+ },
"ap-south-2" : {
"hostname" : "datazone.ap-south-2.api.aws"
},
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
#!/bin/bash
# this is to replace sagemaker unfied studio jupyter scheduler label from "Job" to "Scheudule" to align with overall experience
set -ex

BASE_PATH="/opt/conda/share/jupyter/labextensions/@jupyterlab/scheduler/static"

# Function to perform replacement
replace_string() {
local search="$1"
local replace="$2"
echo "Replacing '$search' with '$replace'..."
grep -l -i -r "$search" "$BASE_PATH" | xargs sed -i "s/$search/$replace/g"
}

# List of replacements
replace_string "\"Schedule\"" "\" \""
replace_string "Create Job\"" "Create Schedule\""
replace_string "Job name\"" "Schedule name\""
replace_string "Run job with input folder\"" "Run schedule with input folder\""
replace_string "\"The scheduled job will have access to all files under" "\"The schedule will have access to all files under"
replace_string "Jobs\"" "Schedules\""
replace_string "Job Definitions\"" "Schedule Definitions\""
replace_string "Job definitions" "Schedule definitions"
replace_string "Job definition" "Schedule definition"
replace_string "job definitions" "schedule definitions"
replace_string "Create Job" "Create Schedule"
replace_string "Job Detail\"" "Schedule Detail\""
replace_string "Job ID\"" "Schedule ID\""
replace_string "Your job" "Your schedule"
replace_string "Job Definition\"" "Schedule Definition\""
replace_string "job definition" "schedule definition"
replace_string "Run Job\"" "Run Schedule\""
replace_string "Creating job" "Creating schedule"
replace_string "Reload Job\"" "Reload Schedule\""
replace_string "Delete Job\"" "Delete Schedule\""
replace_string "Download Job Files\"" "Download Schedule Files\""
replace_string "No jobs associated with this schedule definition" "No schedules associated with this schedule definition"
replace_string "There are no jobs. Jobs run" "There are no schedules. Schedules run"
replace_string "Create a job" "Create a schedule"
replace_string "create a job" "create a schedule"
replace_string "Stop Job\"" "Stop Schedule\""

echo "All replacements completed!"
10 changes: 10 additions & 0 deletions template/v4/v4.6/dirs/etc/patches/smus/0001-sparkmagic.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
--- /opt/conda/lib/python3.12/site-packages/sparkmagic/livyclientlib/command.py
+++ /opt/conda/lib/python3.12/site-packages/sparkmagic/livyclientlib/command.py
@@ -54,6 +54,7 @@
try:
session.wait_for_idle()
data = {"code": self.code}
+ data["kind"] = session.kind
response = session.http_client.post_statement(session.id, data)
statement_id = response["id"]
output = self._get_statement_output(session, statement_id)
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
--- /opt/conda/lib/python3.12/site-packages/sparkmagic/utils/utils.py
+++ /opt/conda/lib/python3.12/site-packages/sparkmagic/utils/utils.py
@@ -94,15 +94,7 @@


def get_sessions_info_html(info_sessions, current_session_id):
- html = (
- """<table>
-<tr><th>ID</th><th>YARN Application ID</th><th>Kind</th><th>State</th><th>Spark UI</th><th>Driver log</th><th>User</th><th>Current session?</th></tr>"""
- + "".join(
- [session.get_row_html(current_session_id) for session in info_sessions]
- )
- + "</table>"
- )
-
+ html = ""
return html
68 changes: 68 additions & 0 deletions template/v4/v4.6/dirs/etc/patches/smus/0005-fix-logger.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
--- /opt/conda/lib/python3.12/site-packages/hdijupyterutils/log.py
+++ /opt/conda/lib/python3.12/site-packages/hdijupyterutils/log.py
@@ -2,7 +2,7 @@

from __future__ import print_function
import logging
-import logging.config
+import importlib

from .constants import LOGGING_CONFIG_CLASS_NAME

@@ -11,13 +11,56 @@
"""Logger for magics. A small wrapper class around the configured logger described in the configuration file"""

def __init__(self, logger_name, logging_config, caller_name):
- logging.config.dictConfig(logging_config)
+ self._setup_logger(logger_name, logging_config)

assert caller_name is not None
self._caller_name = caller_name
self.logger_name = logger_name
self._getLogger()

+ def _setup_logger(self, logger_name, logging_config):
+ """Set up logger manually using handlers and formatters from logging_config"""
+ config = logging_config
+
+ # Create logger
+ logger = logging.getLogger(logger_name)
+
+ # Set up formatters
+ formatters = {}
+ for formatter_name, formatter_config in config["formatters"].items():
+ formatters[formatter_name] = logging.Formatter(
+ fmt=formatter_config["format"],
+ datefmt=formatter_config["datefmt"]
+ )
+
+ # Check which handlers are needed but not already present
+ existing_handler_types = {type(handler).__name__ for handler in logger.handlers}
+
+ # Set up handlers only if they don't already exist
+ for handler_name in config["handlers"]:
+ handler_config = config["handlers"][handler_name]
+
+ # Get handler class
+ handler_class_name = handler_config["class"]
+ module_name, class_name = handler_class_name.rsplit(".", 1)
+ module = importlib.import_module(module_name)
+ handler_class = getattr(module, class_name)
+
+ # Check if this type of handler already exists
+ if handler_class.__name__ not in existing_handler_types:
+ # Create handler with config (excluding class and formatter)
+ handler_kwargs = {k: v for k, v in handler_config.items()
+ if k not in ["class", "formatter"]}
+ handler = handler_class(**handler_kwargs)
+
+ # Set formatter if specified
+ if "formatter" in handler_config:
+ formatter_name = handler_config["formatter"]
+ handler.setFormatter(formatters[formatter_name])
+
+ # Add handler to logger
+ logger.addHandler(handler)
+
def debug(self, message):
self.logger.debug(self._transform_log_message(message))
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
from __future__ import absolute_import

import utils.logger
2 changes: 2 additions & 0 deletions template/v4/v4.6/dirs/etc/sagemaker-inference-server/serve
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
#!/bin/bash
python /etc/sagemaker-inference-server/serve.py
25 changes: 25 additions & 0 deletions template/v4/v4.6/dirs/etc/sagemaker-inference-server/serve.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
from __future__ import absolute_import

"""
TODO: when adding support for more serving frameworks, move the below logic into a condition statement.
We also need to define the right environment variable for signify what serving framework to use.

Ex.

inference_server = None
serving_framework = os.getenv("SAGEMAKER_INFERENCE_FRAMEWORK", None)

if serving_framework == "FastAPI":
inference_server = FastApiServer()
elif serving_framework == "Flask":
inference_server = FlaskServer()
else:
inference_server = TornadoServer()

inference_server.serve()

"""
from tornado_server.server import TornadoServer

inference_server = TornadoServer()
inference_server.serve()
Loading