[Starlingx-discuss] StarlingX platform-integ-apps docker images

[Yang, Bin]

Here is the script which I use to sync the registry for starlingx provision. FYI



On Fri, Sep 27, 2019 at 12:32:10PM -0500, cristopher.j.lemus.contreras at intel.com wrote:
> List of docker images required for "platform-integ-apps":
> 
> BUILD_ID="20190927T013000Z"
> 
> rabbitmq:3.7-management
> k8s.gcr.io/kube-proxy:v1.15.3
> k8s.gcr.io/kube-apiserver:v1.15.3
> k8s.gcr.io/kube-scheduler:v1.15.3
> k8s.gcr.io/kube-controller-manager:v1.15.3
> quay.io/airshipit/armada:8a1638098f88d92bf799ef4934abe569789b885e-ubuntu_bionic
> quay.io/calico/node:v3.6.2
> quay.io/calico/cni:v3.6.2
> quay.io/calico/kube-controllers:v3.6.2
> rabbitmq:3.7.13-management
> rabbitmq:3.7.13
> nfvpe/multus:v3.2
> gcr.io/kubernetes-helm/tiller:v2.13.1
> quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.23.0
> k8s.gcr.io/coredns:1.3.1
> openstackhelm/mariadb:10.2.18
> quay.io/external_storage/rbd-provisioner:v2.1.1-k8s1.11
> quay.io/stackanetes/kubernetes-entrypoint:v0.3.1
> mariadb:10.2.13
> memcached:1.5.5
> k8s.gcr.io/pause:3.1
> quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.9.0
> nginx:1.13.3
> gcr.io/google_containers/defaultbackend:1.0
> 
> 
> _______________________________________________
> Starlingx-discuss mailing list
> Starlingx-discuss at lists.starlingx.io
> http://lists.starlingx.io/cgi-bin/mailman/listinfo/starlingx-discuss
-------------- next part --------------
#!/bin/bash -e

cd `dirname $0`

echo -n "Lock ..."
exec 100>.lock
flock -w 3 -x 100
echo "Done"
echo $$ > pid.sync

echo "update start: `date`" | tee -a log

: ${OPENSTACK_HELM_CHARTS_URL:=http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/latest_build/outputs/helm-charts/helm-charts-stx-openstack-centos-stable-latest.tgz}
: ${BUILD_INFO_URL:=http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/latest_build/outputs/BUILD_INFO.txt}

REGISTRY="localhost:5000"

function check_image()
{
  local reg="$1"
  local img=`echo $2 | cut -d':' -f 1`
  local tag=`echo $2 | cut -d':' -f 2`
  if echo $tag | grep latest; then
    return 1 #always sync it
  fi
  curl --silent $reg/v2/$img/tags/list | grep -q -w "$tag"
}

function fixup_docker_images()
{
  grep --no-filename -o "{{.*_registry[[:space:]]*}}/[^[:space:]]*" $1 -r \
    | sed "s/[\'\"]//g" | sed "s/{{[[:space:]]*//g" | sed "s/[[:space:]]*}}//g" \
    | sed "s/quay_registry/quay.io/g" \
    | sed "s/k8s_registry/k8s.gcr.io/g" \
    | sed "s/gcr_registry/gcr.io/g" \
    | sed "s/docker_registry/docker.io/g"
}

function get_docker_images()
{
  grep -E -w -o "docker.io/.*|quay.io/.*|gcr.io/.*|k8s.gcr.io/.*" $1 -r --no-filename --include=*.yml --include=*.yaml | sed "s/[\'\"]//g" | sort | uniq
}

function sync_images()
{
  local reg
  local img
  local limg

  while read img; do
    reg=`echo $img | cut -d '/' -f 1`
    limg=`echo $img | cut -d '/' -f 2-`

    echo -n "check img: $img ..."
    if check_image $REGISTRY $limg; then
      echo "OK"
      continue
    fi
    echo

#    echo "img: $img"
#    echo "local img: $limg"
    if ! sudo docker pull $img; then
      if echo $img | grep 'docker.io/openstackhelm/placement:'; then
        echo "ignore $img ..."
        continue;
      fi
    fi
    sudo docker tag $img $REGISTRY/$limg
    sudo docker push $REGISTRY/$limg
  done
}

function cleanup_images()
{
  sudo docker image prune
  sudo docker exec -t registry-srv registry garbage-collect /etc/docker/registry/config.yml --delete-untagged=true
}

function list_images()
{
  for repo in $(curl --silent $REGISTRY/v2/_catalog | jq -r .repositories[]); do
    for tag in $(curl --silent $REGISTRY/v2/${repo}/tags/list | jq -r .tags[]); do
      echo $repo:$tag
    done
  done
}

rm -rf tmp/
mkdir tmp/

wget -O build_info.txt "$BUILD_INFO_URL"

cat > docker_images.txt <<__EOF__
k8s.gcr.io/kube-apiserver:v1.15.3
k8s.gcr.io/kube-controller-manager:v1.15.3
k8s.gcr.io/kube-scheduler:v1.15.3
k8s.gcr.io/kube-proxy:v1.15.3
k8s.gcr.io/pause:3.1
k8s.gcr.io/etcd:3.3.10
k8s.gcr.io/coredns:1.3.1
__EOF__

mkdir tmp/openstack
wget -O tmp/openstack/latest.tgz "$OPENSTACK_HELM_CHARTS_URL"
tar xzvf tmp/openstack/latest.tgz -C tmp/openstack
find tmp/openstack -name "*.tgz" -exec tar xzvf {} -C tmp/openstack/charts/ \;

git clone https://opendev.org/starlingx/config.git tmp/config
git clone https://opendev.org/starlingx/ansible-playbooks.git tmp/ansible-playbooks

fixup_docker_images tmp/ansible-playbooks > tmp/fixup.yml
get_docker_images tmp/ >> docker_images.txt

cat docker_images.txt | sync_images

cleanup_images

#list_images > docker_images.txt

echo `date` > latest.txt
cp -f latest.txt www/
cp -f build_info.txt www/
cp -f docker_images.txt www/latest_docker_images.txt

echo "update end: `date`" | tee -a log