Skip to content

Commit 7c718a0

Browse files
abhijeet-dhumalabhijeet-dhumal
committed
Update
1 parent fd60367 commit 7c718a0

File tree

1 file changed

+176
-0
lines changed

1 file changed

+176
-0
lines changed

‎.github/workflows/odh-kfto-sdk-notebooks-sync.yaml

+176
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
# The aim of this GitHub workflow is to update the pipfile to sync with Codeflare-SDK release.
2+
name: Sync ODH-notebooks with Kubeflow-Training SDK release
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
upstream-notebooks-repository-organization:
7+
required: true
8+
description: "Owner of target upstream notebooks repository used to open a PR against"
9+
default: "opendatahub-io"
10+
notebooks-target-branch:
11+
required: true
12+
description: "Target branch of upstream repository"
13+
default: "main"
14+
python-version:
15+
required: true
16+
description: "Provide the python version to be used for the notebooks"
17+
default: "3.11"
18+
notebooks-repository-organization:
19+
required: true
20+
description: "Owner of origin notebooks repository used to open a PR"
21+
notebooks-repository-name:
22+
required: true
23+
description: "Name of origin notebooks repository used to open a PR"
24+
training-sdk-release-version:
25+
required: true
26+
description: "Provide version of the kubeflow-training-sdk release"
27+
notebooks-repository-organization-owner-user-email:
28+
required: true
29+
description: "Provide notebooks repository organization owner's user-email used to open a PR"
30+
notebooks-repository-organization-owner-user-name:
31+
required: true
32+
description: "Provide notebooks repository organization owner's user-name used to open a PR"
33+
34+
env:
35+
BRANCH_NAME: ${{ github.event.inputs.notebooks-target-branch }}
36+
PYTHON_VERSION: ${{ github.event.inputs.python-version }}
37+
TRAINING_SDK_RELEASE_VERSION: ${{ github.event.inputs.training-sdk-release-version }}
38+
UPDATER_BRANCH: odh-sync-updater-${{ github.run_id }}
39+
UPSTREAM_OWNER: ${{ github.event.inputs.upstream-notebooks-repository-organization }}
40+
UPSTREAM_REPO_NAME: notebooks
41+
REPO_OWNER: ${{ github.event.inputs.notebooks-repository-organization }}
42+
REPO_OWNER_USER_EMAIL: ${{ github.event.inputs.notebooks-repository-organization-owner-user-email }}
43+
REPO_OWNER_USER_NAME: ${{ github.event.inputs.notebooks-repository-organization-owner-user-name }}
44+
REPO_NAME: ${{ github.event.inputs.notebooks-repository-name }}
45+
GITHUB_TOKEN: ${{ secrets.KUBEFLOW_TRAINING_ACCOUNT_TOKEN }} # add KUBEFLOW_TRAINING_ACCOUNT_TOKEN named secret in your notebooks repo to be used here
46+
MINIMUM_SUPPORTED_PYTHON_VERSION: 3.9
47+
48+
jobs:
49+
build:
50+
runs-on: ubuntu-latest
51+
steps:
52+
- name: Clone repository and Sync
53+
run: |
54+
git clone https://x-access-token:${GITHUB_TOKEN}@github.com/$REPO_OWNER/$REPO_NAME.git $REPO_NAME
55+
cd $REPO_NAME
56+
git remote add upstream https://github.com/$UPSTREAM_OWNER/$UPSTREAM_REPO_NAME.git
57+
git config --global user.email $REPO_OWNER_USER_EMAIL
58+
git config --global user.name $REPO_OWNER_USER_NAME
59+
git remote -v
60+
git checkout $BRANCH_NAME
61+
git config pull.rebase true
62+
git pull upstream $BRANCH_NAME && git push -f origin $BRANCH_NAME
63+
64+
- name: Setup Python environment
65+
uses: actions/setup-python@v4
66+
with:
67+
python-version: ${{ env.PYTHON_VERSION }}
68+
cache: 'pipenv'
69+
70+
- name: Install pipenv and pip-versions
71+
run: pip install pipenv==2024.4.0 pip-versions
72+
73+
- name: Update Pipfiles in accordance with Codeflare-SDK latest release
74+
run: |
75+
package_name=kubeflow-training
76+
available_python_versions=("$PYTHON_VERSION") # add space separated python versions according to 'python-versions' specified in 'Setup Python Environment' step
77+
install_package_using_pipenv(){
78+
# args allow custom names for Pipfile and Pipfile.lock
79+
if [ $# -eq 2 ]; then
80+
mv "${1}" Pipfile
81+
mv "${2}" Pipfile.lock
82+
fi
83+
# replace existing version of cf-sdk with new version in Pipfile
84+
sed -i "s/$package_name = \"==[^\"]*\"/$package_name = \"~=${TRAINING_SDK_RELEASE_VERSION}\"/" Pipfile
85+
# Lock dependencies, ensuring pre-release are included and clear previous state
86+
if ! pipenv lock --verbose --pre --clear ; then
87+
echo "Failed to lock dependencies"
88+
exit 1
89+
fi
90+
# remove virtual env and clear cache
91+
if ! pipenv --rm --clear ; then
92+
echo "Failed to remove virtual environment"
93+
exit 1
94+
fi
95+
if [ $# -eq 2 ]; then
96+
mv Pipfile "${1}"
97+
mv Pipfile.lock "${2}"
98+
fi
99+
}
100+
# Get the list of available versions for the package
101+
if ! versions=$(pipenv run pip-versions list $package_name);then
102+
echo "Failed to retrieve versions for $package_name"
103+
exit 1
104+
fi
105+
# Check if the desired version exists in the list
106+
if echo "$versions" | grep -q "${TRAINING_SDK_RELEASE_VERSION}"; then
107+
echo "Version ${TRAINING_SDK_RELEASE_VERSION} is available for $package_name"
108+
directories+=($(grep --exclude-dir=.git --exclude-dir=.github --exclude-dir=intel --exclude-dir=tensorflow --exclude-dir=rocm-tensorflow --include="Pipfile*" -rl "$package_name = \"==[0-9.]*\"" | xargs dirname | sort | uniq))
109+
counter=0
110+
total=${#directories[@]}
111+
echo -----------
112+
echo ${#total[@]}
113+
for dir in "${directories[@]}"; do
114+
counter=$((counter+1))
115+
echo "--Processing directory $counter '$dir' of total $total"
116+
cd "$dir"
117+
minimum_supported_python_version_major=$(echo "${MINIMUM_SUPPORTED_PYTHON_VERSION}" | awk -F '.' '{print $1}') #integer of MINIMUM_SUPPORTED_PYTHON_VERSION env variable
118+
minimum_supported_python_version_minor=$(echo "${MINIMUM_SUPPORTED_PYTHON_VERSION}" | awk -F '.' '{print $2}') #decimal of MINIMUM_SUPPORTED_PYTHON_VERSION env variable
119+
if ! [ -f "Pipfile" ]; then
120+
if [ -f "Pipfile.cpu" ]; then
121+
pipfile_python_version=$(grep -E '^python_version' ./Pipfile.cpu | cut -d '"' -f 2) # extracted from pipfile.cpu
122+
fi
123+
else
124+
pipfile_python_version=$(grep -E '^python_version' ./Pipfile | cut -d '"' -f 2) # extracted from pipfile
125+
fi
126+
pipfile_python_version_major=$(echo "$pipfile_python_version" | awk -F '.' '{print $1}')
127+
pipfile_python_version_minor=$(echo "$pipfile_python_version" | awk -F '.' '{print $2}')
128+
if [[ " ${available_python_versions[@]} " =~ " ${pipfile_python_version} " && "$pipfile_python_version_major" -ge "$minimum_supported_python_version_major" && "$pipfile_python_version_minor" -ge "$minimum_supported_python_version_minor" ]]; then
129+
if ! [ -f "Pipfile" ]; then
130+
if [ -f "Pipfile.cpu" ]; then
131+
install_package_using_pipenv Pipfile.cpu Pipfile.lock.cpu
132+
fi
133+
if [ -f "Pipfile.gpu" ]; then
134+
install_package_using_pipenv Pipfile.gpu Pipfile.lock.gpu
135+
fi
136+
else
137+
#install specified package
138+
install_package_using_pipenv
139+
fi
140+
else
141+
echo "Skipped installation of ${package_name} with version ${TRAINING_SDK_RELEASE_VERSION} in $dir"
142+
fi
143+
cd -
144+
echo "$((total-counter)) directories remaining.."
145+
done
146+
# Run a script to install micropipenv and format the file changes correctly
147+
cd ./notebooks && bash ./ci/generate_code.sh && cd -
148+
else
149+
versions_list=$(echo "$versions" | tr '\n' ' ' | sed 's/, $//')
150+
versions="${versions_list%,}"
151+
echo "Version '${TRAINING_SDK_RELEASE_VERSION}' is not available for $package_name"
152+
echo "Available versions for $package_name: $versions"
153+
exit 1
154+
fi
155+
156+
- name: Push changes
157+
run: |
158+
cd $REPO_NAME
159+
git add . && git status && git checkout -b ${{ env.UPDATER_BRANCH }} && \
160+
git commit -am "Updated notebooks via ${{ env.UPDATER_BRANCH }} GitHub action" --signoff &&
161+
git remote set-url origin https://x-access-token:${GITHUB_TOKEN}@github.com/$REPO_OWNER/$REPO_NAME.git
162+
git push origin ${{ env.UPDATER_BRANCH }}
163+
164+
- name: Create Pull Request
165+
run: |
166+
gh pr create --repo $UPSTREAM_OWNER/$UPSTREAM_REPO_NAME \
167+
--title "$pr_title" \
168+
--body "$pr_body" \
169+
--head $REPO_OWNER:$UPDATER_BRANCH \
170+
--base $BRANCH_NAME
171+
env:
172+
pr_title: "[Kubeflow-Training Action] Update notebook's pipfile to sync with Kubeflow-Training SDK release ${{ env.TRAINING_SDK_RELEASE_VERSION }}"
173+
pr_body: |
174+
:rocket: This is an automated Pull Request generated by [odh-kfto-sdk-notebooks-sync.yml](https://github.com/opendatahub-io/training-operator/tree/dev/.github/workflows/odh-kfto-sdk-notebooks-sync.yml) workflow.
175+
176+
This PR updates the `Pipfile` to sync with latest Kubeflow-Training SDK release.

0 commit comments

Comments
 (0)