Skip to content

add scripts to build and package starter projects for offline registries #179

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Aug 10, 2023
Merged

add scripts to build and package starter projects for offline registries #179

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
143 changes: 109 additions & 34 deletions build-tools/dl_starter_projects.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,17 @@
#!/bin/bash

if [[ -z "$@" ]]
then
echo "No starter projects specified."
exit 0
fi

# Path of stacks directory in the registry
STACKS_DIR=/registry/stacks
# List of starter projects to use offline
offline_starter_projects=( "$@" )
# When no starter projects are specifed,
# all starter projects will be downloaded
download_all_starter_projects=false

if [[ -z "$@" ]]
then
download_all_starter_projects=true
fi

# Downloads a starter project from a remote git repository and packages it as a zip archive
# to be used as an offline resource.
Expand All @@ -34,14 +36,21 @@ download_git_starter_project() {

if [ "${revision}" != "null" ]
then
cd $local_path && git checkout $revision && cd -
cd $local_path
git checkout $revision
cd -
fi

if [ "${subDir}" != "null" ]
then
cd $local_path/$subDir && zip -q ${local_path}.zip * .[^.]* && cd -
cd $local_path/$subDir
zip -q ../${name}-offline.zip * .[^.]*
cd -
else
cd $local_path && rm -rf ./.git && zip -q ${local_path}.zip * .[^.]* && cd -
cd $local_path
rm -rf ./.git
zip -q ../${name}-offline.zip * .[^.]*
cd -
fi

rm -rf $local_path
Expand All @@ -58,12 +67,62 @@ download_zip_starter_project() {
curl -L $remote_url -o ${local_path}.zip
}

# Read stacks list
read -r -a stacks <<< "$(ls ${STACKS_DIR} | tr '\n' ' ')"
download_specific() {
for starter_project in ${offline_starter_projects[@]}
do
for stack in ${stacks[@]}
do
stack_root=$STACKS_DIR/$stack
stack_devfile=$stack_root/devfile.yaml
# Read version list for stack
read -r -a versions <<< "$(ls ${STACKS_DIR}/${stack} | grep -e '[0-9].[0-9].[0-9]' | tr '\n' ' ')"
# If multi version stack
if [[ ${#versions[@]} -gt 0 ]]
then
for version in ${versions[@]}
do
stack_root=$STACKS_DIR/$stack/$version
stack_devfile=$stack_root/devfile.yaml
# If the specified starter project is found
if [ ! -z "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\")" $stack_devfile)" ]
then
# Starter project has a git remote
if [ "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\").git" $stack_devfile)" != "null" ]
then
echo "Downloading ${starter_project} starter project in stack ${stack} version ${version}.."
download_git_starter_project $stack_root $starter_project
echo "Downloading ${starter_project} starter project in stack ${stack} version ${version}..done!"
# Starter project has a zip remote
elif [ "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\").zip" $stack_devfile)" != "null" ]
then
echo "Downloading ${starter_project} starter project in stack ${stack} version ${version}.."
download_zip_starter_project $stack_root $starter_project
echo "Downloading ${starter_project} starter project in stack ${stack} version ${version}..done!"
fi
fi
done
# If not multi version stack & the specified starter project is found
elif [ ! -z "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\")" $stack_devfile)" ]
then
# Starter project has a git remote
if [ "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\").git" $stack_devfile)" != "null" ]
then
echo "Downloading ${starter_project} starter project in stack ${stack}.."
download_git_starter_project $stack_root $starter_project
echo "Downloading ${starter_project} starter project in stack ${stack}..done!"
# Starter project has a zip remote
elif [ "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\").zip" $stack_devfile)" != "null" ]
then
echo "Downloading ${starter_project} starter project in stack ${stack}.."
download_zip_starter_project $stack_root $starter_project
echo "Downloading ${starter_project} starter project in stack ${stack}..done!"
fi
fi
done
done
}

echo "Downloading offline starter projects.."
for starter_project in ${offline_starter_projects[@]}
do
download_all() {
for stack in ${stacks[@]}
do
stack_root=$STACKS_DIR/$stack
Expand All @@ -77,9 +136,10 @@ do
do
stack_root=$STACKS_DIR/$stack/$version
stack_devfile=$stack_root/devfile.yaml
# If the specified starter project is found
if [ ! -z "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\")" $stack_devfile)" ]
then
starter_projects="$(yq e ".starterProjects[].name" $stack_devfile)"

for starter_project in $starter_projects
do
# Starter project has a git remote
if [ "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\").git" $stack_devfile)" != "null" ]
then
Expand All @@ -93,25 +153,40 @@ do
download_zip_starter_project $stack_root $starter_project
echo "Downloading ${starter_project} starter project in stack ${stack} version ${version}..done!"
fi
done
done
# If not multi version stack
else
starter_projects="$(yq e ".starterProjects[].name" $stack_devfile)"
for starter_project in $starter_projects
do
# Starter project has a git remote
if [ "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\").git" $stack_devfile)" != "null" ]
then
echo "Downloading ${starter_project} starter project in stack ${stack}.."
download_git_starter_project $stack_root $starter_project
echo "Downloading ${starter_project} starter project in stack ${stack}..done!"
# Starter project has a zip remote
elif [ "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\").zip" $stack_devfile)" != "null" ]
then
echo "Downloading ${starter_project} starter project in stack ${stack}.."
download_zip_starter_project $stack_root $starter_project
echo "Downloading ${starter_project} starter project in stack ${stack}..done!"
fi
done
# If not multi version stack & the specified starter project is found
elif [ ! -z "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\")" $stack_devfile)" ]
then
# Starter project has a git remote
if [ "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\").git" $stack_devfile)" != "null" ]
then
echo "Downloading ${starter_project} starter project in stack ${stack}.."
download_git_starter_project $stack_root $starter_project
echo "Downloading ${starter_project} starter project in stack ${stack}..done!"
# Starter project has a zip remote
elif [ "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\").zip" $stack_devfile)" != "null" ]
then
echo "Downloading ${starter_project} starter project in stack ${stack}.."
download_zip_starter_project $stack_root $starter_project
echo "Downloading ${starter_project} starter project in stack ${stack}..done!"
fi
fi
echo
done
done
}

# Read stacks list
read -r -a stacks <<< "$(ls ${STACKS_DIR} | tr '\n' ' ')"

echo "Downloading offline starter projects.."
if [ "$download_all_starter_projects" = true ]
then
download_all
else
download_specific
fi
echo "Downloading offline starter projects..done!"
147 changes: 147 additions & 0 deletions build-tools/update_devfiles_offline.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,147 @@
#!/bin/bash

# This script will modify the devfiles in each stack to use offline resources.
# Stack devfiles will not be modified if there are no starterProjects or if the
# staterProjects have already been modified.
# - starterProjects will be commented out and replaced with a zip block with
# the location to the offline project in the stack root

# Path of stacks directory in the registry
STACKS_DIR=/registry/stacks
# Automated comment used to check whether the devfile has already been modified
MODIFIED_MESSAGE="# AUTOMATED MODIFICATION -"

cleanup() {
rm -f "$devfile.tmp"
rm -f "$offline_starter_projects"
}

trap cleanup EXIT

comment_out_starter_project() {
stack_root=$1
name=$2
devfile=$3

has_starterProjects=$(yq e '.starterProjects' $devfile 2> /dev/null)
if [[ $has_starterProjects != null ]]
then
# the first line of the diff needs to be removed; example of the diff:
# 44a45,52
# > starterProjects:
# > - name: go-starter
# ...
#
# 's/^../#/' replaces the first 2 characters of each line with #
diff="$(diff -b <(yq e 'del(.starterProjects)' $devfile) $devfile | tail -n +2)"
starter_projects="$(printf '%s\n' "$diff" | sed 's/^../#/')"

# comment out the starter projects
yq e 'del(.starterProjects)' "$devfile" > "$devfile.tmp"
echo "# Commented out original starter projects" >> "$devfile.tmp"
echo "$starter_projects" >> "$devfile.tmp"
mv "$devfile.tmp" "$devfile"
fi
}

# Updates a devfile to use an offline starter project
new_starter_projects() {
stack_root=$1
name=$2
devfile=$3
offline_starter_projects=$4

project="
- name: ${name}
zip:
location: ${name}-offline.zip"

echo -n "$project" >> $offline_starter_projects
}

# Read stacks list
read -r -a stacks <<< "$(ls ${STACKS_DIR} | tr '\n' ' ')"

echo "Updating devfiles.."
for stack in ${stacks[@]}
do
stack_root=$STACKS_DIR/$stack
stack_devfile=$stack_root/devfile.yaml
# Exit early on failure to avoid bad overwriting of the devfile
offline_starter_projects=$(mktemp) || exit 1
# Overwrite the temp file for each stack
echo -n "" > $offline_starter_projects
# Read version list for stack
read -r -a versions <<< "$(ls ${STACKS_DIR}/${stack} | grep -e '[0-9].[0-9].[0-9]' | tr '\n' ' ')"

# If multi version stack
if [[ ${#versions[@]} -gt 0 ]]
then
for version in ${versions[@]}
do
stack_root=$STACKS_DIR/$stack/$version
stack_devfile=$stack_root/devfile.yaml
starter_projects="$(yq e ".starterProjects[].name" $stack_devfile)"
echo -n "starterProjects:" > $offline_starter_projects

if [[ $starter_projects == "" ]]
then
echo "Skipping stack ${stack} version ${version}: no starter projects found."
continue
fi

for starter_project in $starter_projects
do
if ! grep -q "$MODIFIED_MESSAGE" "$stack_devfile";
then
echo "Updating the ${starter_project} starter project in stack ${stack} version ${version}.."
new_starter_projects $stack_root $starter_project $stack_devfile $offline_starter_projects
comment_out_starter_project $stack_root $starter_project $stack_devfile
echo "Updating the ${starter_project} starter project in stack ${stack} version ${version}..done!"
else
echo "The ${starter_project} starter project in stack ${stack} version ${version} has already been modified."
fi
done

# Only write to the devfile if starter projects have been commented out
has_starterProjects=$(yq e '.starterProjects' $stack_devfile 2> /dev/null)
if [[ $has_starterProjects == null ]] && ! grep -q "$MODIFIED_MESSAGE" "$stack_devfile"
then
echo "${MODIFIED_MESSAGE} Updated starterProjects to use offline versions" >> $stack_devfile
cat $offline_starter_projects >> $stack_devfile
fi
done
# If not multi version stack
else
starter_projects="$(yq e ".starterProjects[].name" $stack_devfile)"
echo -n "starterProjects:" > $offline_starter_projects

if [[ $starter_projects == "" ]]
then
echo "Skipping stack ${stack}: no starter projects found."
continue
fi

for starter_project in $starter_projects
do
if ! grep -q "$MODIFIED_MESSAGE" "$stack_devfile";
then
echo "Updating the ${starter_project} starter project in stack ${stack}.."
new_starter_projects $stack_root $starter_project $stack_devfile $offline_starter_projects
comment_out_starter_project $stack_root $starter_project $stack_devfile
echo "Updating the ${starter_project} starter project in stack ${stack}..done!"
else
echo "The ${starter_project} starter project in stack ${stack} has already been modified."
fi
done

# Only write to the devfile if starter projects have been commented out
has_starterProjects=$(yq e '.starterProjects' $stack_devfile 2> /dev/null)
if [[ $has_starterProjects == null ]] && ! grep -q "$MODIFIED_MESSAGE" "$stack_devfile"
then
echo "${MODIFIED_MESSAGE} Updated starterProjects to use offline versions" >> $stack_devfile
cat $offline_starter_projects >> $stack_devfile
fi
fi
done
echo "Updating devfiles....done!"