Implement index server

.github/workflows/ci.yaml

@@ -30,4 +30,4 @@ jobs:
       uses: actions/checkout@v2
 
     - name: Check if registry image build is working
-      run: cd oci-devfile-registry-metadata && ./build.sh
+      run: cd index/server && ./build.sh

deploy/kubernetes/registry.yaml

@@ -20,7 +20,7 @@ spec:
           claimName: devfile-registry-storage
       containers:
       - name: devfile-registry-metadata
-        image: quay.io/devfile/metadata-server:latest
+        image: quay.io/devfile/devfile-index-base:latest
         ports:
         - containerPort: 8080
         resources:
@@ -32,13 +32,13 @@ spec:
             cpu: "250m"
         livenessProbe:
           httpGet:
-            path: /
+            path: /health
             port: 8080
           initialDelaySeconds: 3
           periodSeconds: 3
         readinessProbe:
           httpGet:
-            path: /
+            path: /health
             port: 8080
           initialDelaySeconds: 3
           periodSeconds: 3

index/server/.vscode/launch.json

@@ -0,0 +1,17 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Launch",
+            "type": "go",
+            "request": "launch",
+            "mode": "auto",
+            "program": "${fileDirname}",
+            "env": {},
+            "args": []
+        }
+    ]
+}

oci-devfile-registry-metadata/Dockerfile → index/server/Dockerfile

@@ -1,29 +1,36 @@
+# Builder image
+FROM golang:alpine3.11 AS builder
+WORKDIR /tools
+COPY . .
+RUN CGO_ENABLED=0 go build -mod=vendor -o index-server main.go
+
+# Application image
 FROM nginx:stable-alpine
 
 # Install and configure dependencies
 RUN apk add --no-cache bash git curl coreutils
-RUN wget https://github.com/deislabs/oras/releases/download/v0.8.1/oras_0.8.1_linux_amd64.tar.gz && \
-  mkdir -p oras-install/ && \
-  tar -zxf oras_0.8.1_*.tar.gz -C oras-install/ && \
-  mv oras-install/oras /usr/local/bin/ && \
-  rm -rf oras_0.8.1_*.tar.gz oras-install/
 
 COPY nginx.conf /etc/nginx/nginx.conf
 COPY entrypoint.sh /
 
+# Load index server
+COPY --from=builder /tools/index-server /registry/index-server
+RUN chgrp -R 0 /registry && \
+    chmod -R g=u /registry
+
 # Create a non-root user to run the nginx server as
 RUN set -x ; \
-  adduser -u 82 -D -S -G root www-data && exit 0 ; exit 1
+    adduser -u 82 -D -S -G root www-data && exit 0 ; exit 1
 
 RUN touch /var/run/nginx.pid
 RUN mkdir -p /www/data
 
 # Modify the permissions on the necessary files to allow the container to properly run as a non-root UID
 RUN chown -R www-data:root /var/run/nginx.pid && \
-  chown -R www-data:root /var/cache/nginx && \
-  chown www-data:root /etc/nginx/conf.d /etc/nginx/nginx.conf
+    chown -R www-data:root /var/cache/nginx && \
+    chown www-data:root /etc/nginx/conf.d /etc/nginx/nginx.conf
 RUN chmod g+rwx /var/run/nginx.pid && \
-  chmod -R g+rwx /var/cache/nginx && chmod -R g+rwx /etc/nginx && chmod -R g+rwx /www/data
+    chmod -R g+rwx /var/cache/nginx && chmod -R g+rwx /etc/nginx && chmod -R g+rwx /www/data
 
 USER www-data
 
@@ -33,4 +40,3 @@ ENV DEVFILE_INDEX /index.json
 
 EXPOSE 8080
 ENTRYPOINT ["/entrypoint.sh"]
-CMD ["nginx", "-g", "daemon off;"]

index/server/build.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+# Build the index container for the registry
+buildfolder="$(basename "$(dirname "$0")")"
+docker build -t devfile-index-base:latest $buildfolder

index/server/entrypoint.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+# Check if devfile stacks and index.json exist
+if [ ! -d "$DEVFILE_STACKS" ]; then
+    echo "The container does not contain any devfile stacks in $DEVFILE_STACKS. Exiting..."
+    exit 1
+fi
+if [ ! -e "$DEVFILE_INDEX" ]; then
+    echo "The container does not contain an index.json at $DEVFILE_INDEX. Exiting..."
+    exit 1
+fi
+
+# Start the nginx server
+nginx -g "daemon off;" &
+
+# Start the index server
+/registry/index-server
+
+fg %1

index/server/go.mod

@@ -0,0 +1,13 @@
+module github.com/devfile/registry-support/index/server
+
+go 1.14
+
+require (
+	github.com/containerd/containerd v1.4.1
+	github.com/deislabs/oras v0.8.1
+	github.com/devfile/registry-support/index/generator v0.0.0-20201027142749-62d041d56b76
+	github.com/gin-gonic/gin v1.6.3
+	github.com/opencontainers/image-spec v1.0.1
+	gotest.tools/v3 v3.0.3 // indirect
+	k8s.io/apimachinery v0.19.4
+)

index/server/main.go

@@ -0,0 +1,147 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"time"
+
+	"github.com/deislabs/oras/pkg/content"
+	"github.com/deislabs/oras/pkg/oras"
+	indexSchema "github.com/devfile/registry-support/index/generator/schema"
+
+	"github.com/containerd/containerd/remotes/docker"
+	"github.com/gin-gonic/gin"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+)
+
+const (
+	devfileName            = "devfile.yaml"
+	devfileConfigMediaType = "application/vnd.devfileio.devfile.config.v2+json"
+	devfileMediaType       = "application/vnd.devfileio.devfile.layer.v1"
+	scheme                 = "http"
+	registryService        = "localhost:5000"
+)
+
+var (
+	stacksPath = os.Getenv("DEVFILE_STACKS")
+	indexPath  = os.Getenv("DEVFILE_INDEX")
+)
+
+func main() {
+	// Wait until registry is up and running
+	err := wait.PollImmediate(time.Millisecond, time.Second*30, func() (bool, error) {
+		resp, err := http.Get(scheme + "://" + registryService)
+		if err != nil {
+			log.Println(err.Error())
+			return false, nil
+		}
+
+		if resp.StatusCode == http.StatusOK {
+			log.Println("Registry is up and running")
+			return true, nil
+		}
+
+		log.Println("Waiting for registry to start...")
+		return false, nil
+	})
+	if err != nil {
+		log.Fatal(err.Error())
+	}
+
+	// Load index file
+	bytes, err := ioutil.ReadFile(indexPath)
+	if err != nil {
+		log.Fatalf("failed to read index file: %v", err)
+	}
+
+	// TODO: add code block to parse index.json by using common library
+	// Issue: https://github.com/devfile/api/issues/223
+	var index []indexSchema.Schema
+	err = json.Unmarshal(bytes, &index)
+	if err != nil {
+		log.Fatalf("failed to unmarshal index file: %v", err)
+	}
+
+	// Before starting the server, push the devfile artifacts to the registry
+	for _, devfileIndex := range index {
+		err := pushStackToRegistry(devfileIndex)
+		if err != nil {
+			log.Fatal(err.Error())
+		}
+	}
+
+	// Start the server and serve requests and index.json
+	router := gin.Default()
+
+	router.GET("/health", func(c *gin.Context) {
+		c.JSON(http.StatusOK, gin.H{
+			"message": "the server is up and running",
+		})
+	})
+
+	router.Static("/stacks", stacksPath)
+	router.StaticFile("/index.json", indexPath)
+	router.StaticFile("/", indexPath)
+
+	router.Run(":7070")
+}
+
+// pushStackToRegistry pushes the given devfile stack to the OCI registry
+func pushStackToRegistry(devfileIndex indexSchema.Schema) error {
+	// Load the devfile into memory and set up the pushing resource (file name, file content, media type, ref)
+	devfileContent, err := ioutil.ReadFile(filepath.Join(stacksPath, devfileIndex.Name, devfileName))
+	if err != nil {
+		return err
+	}
+	ref := path.Join(registryService, "/", devfileIndex.Links["self"])
+
+	ctx := context.Background()
+	resolver := docker.NewResolver(docker.ResolverOptions{PlainHTTP: true})
+
+	// Add the devfile (and its custom media type) to the memory store
+	memoryStore := content.NewMemoryStore()
+	desc := memoryStore.Add(devfileName, devfileMediaType, devfileContent)
+	pushContents := []ocispec.Descriptor{desc}
+
+	log.Printf("Pushing %s to %s...\n", devfileName, ref)
+	desc, err = oras.Push(ctx, resolver, ref, memoryStore, pushContents, oras.WithConfigMediaType(devfileConfigMediaType))
+	if err != nil {
+		return fmt.Errorf("failed to push %s to %s: %v", devfileName, ref, err)
+	}
+	log.Printf("Pushed to %s with digest %s\n", ref, desc.Digest)
+	return nil
+}
+
+// pullStackFromRegistry pulls the given devfile stack from the OCI registry
+func pullStackFromRegistry(devfileIndex indexSchema.Schema) ([]byte, error) {
+	// Pull the devfile from registry and save to disk
+	ref := path.Join(registryService, "/", devfileIndex.Links["self"])
+
+	ctx := context.Background()
+	resolver := docker.NewResolver(docker.ResolverOptions{PlainHTTP: true})
+
+	// Initialize memory store
+	memoryStore := content.NewMemoryStore()
+	allowedMediaTypes := []string{devfileMediaType}
+
+	log.Printf("Pulling %s from %s...\n", devfileName, ref)
+	desc, _, err := oras.Pull(ctx, resolver, ref, memoryStore, oras.WithAllowedMediaTypes(allowedMediaTypes))
+	if err != nil {
+		return nil, fmt.Errorf("failed to pull %s from %s: %v", devfileName, ref, err)
+	}
+	_, bytes, ok := memoryStore.GetByName(devfileName)
+	if !ok {
+		return nil, fmt.Errorf("failed to load %s to memory", devfileName)
+	}
+
+	log.Printf("Pulled from %s with digest %s\n", ref, desc.Digest)
+	return bytes, nil
+}

oci-devfile-registry-metadata/nginx.conf → index/server/nginx.conf

@@ -31,13 +31,10 @@ http {
 
     # required to avoid HTTP 411: see Issue #1486 (https://github.com/moby/moby/issues/1486)
     chunked_transfer_encoding on;
-
+    
     location / {
-        # Temporary until the new metadata server has been delivered
-        # ToDo: Update to point to the new server when ready
-        default_type application/json;
-        root /www/data;
-        index index.json;
+      # Forward requests to index server
+      proxy_pass http://localhost:7070;
     }
 
     location /v2 {      

oci-devfile-registry-metadata/push.sh → index/server/push.sh

@@ -1,4 +1,4 @@
 #!/bin/sh
 IMAGE_TAG=$1
 docker tag devfile-index-base:latest $IMAGE_TAG
-docker push $1
+docker push $IMAGE_TAG