gatherallfunctions

#/bin/bash

#This script has a few for loops that are run multiple times that could probably be made into a function in the future where parameters are passed in a more resonable way. For now, this works... It takes about an hour to run bbecause helm pull is so slow. Google had a distributed pull tool that helped make those much faster, but bitnami charts are still VERY slow... we should explore some sort of distributed curl or wget tool in the future... I think there is a library written in Go we map eventually be able to use?

#Start your engines
#start=`date +%s`

function SETVERS {
#set global variables
imagestoreurl=https://test.com
chartstoreurl=test.com
bucket=pics
pathtocharts=$(pwd)

#set helm chart stable variables
indexfilestable=/tmp/stable/index.yaml
stablecharts=$pathtocharts'/stable'
mkdir $stablecharts

#set helm chart incubator variables
indexfileincubator=/tmp/incubator/index.yaml
incubatorcharts=$pathtocharts'/incubator'
mkdir $incubatorcharts

#set bitnami helm chart variables & since bitnami takes so long, in the future we'll only want to get the delta's so we want to keep track of the old list as well as the new list. For now, we don't do that, but setup for follow on work.
indexfilebitnami=/tmp/bitnami-charts/index.yaml
bitnamicharts=$pathtocharts'/bitnami-charts'
rm -rf $bitnamicharts'-old'
mv $bitnamicharts $bitnamicharts'-old'
mkdir $bitnamicharts

#set rook-ceph helm chart variables.
indexfilerookceph=/tmp/rookceph-charts/index.yaml
rookcephcharts=$pathtocharts'/rookceph-charts'
rm -rf $rookcephcharts'-old'
mv $rookcephcharts $rookcephcharts'-old'
mkdir $rookcephcharts


#set ranchestable helm chart variables.
indexfilerancherstable=/tmp/rancherstable-charts/index.yaml
rancherstablecharts=$pathtocharts'/rancherstable-charts'
rm -rf $rancherstablecharts'-old'
mv $rancherstablecharts $rancherstablecharts'-old'
mkdir $rancherstablecharts

#set rancherlatest helm chart variables.
indexfilerancherlatest=/tmp/rancherlatest-charts/index.yaml
rancherlatestcharts=$pathtocharts'/rancherlatest-charts'
rm -rf $rancherlatestcharts'-old'
mv $rancherlatestcharts $rancherlatestcharts'-old'
mkdir $rancherlatestcharts

#set vmware helm chart variables & since vmware takes so long, in the future we'll only want to get the delta's so we want to keep track of the old list as well as the new list. For now, we don't do that, but setup for follow on work.
indexfilevmware=/tmp/vmware-charts/index.yaml
vmwarecharts=$pathtocharts'/vmware-charts'
rm -rf $vmwarecharts'-old'
mv $vmwarecharts $vmwarecharts'-old'
mkdir $vmwarecharts

}

function HELMUPDATE {

#move latest verison of helm to usr bin
sudo mv /usr/bin/helm /usr/bin/helm_old
sudo cp $pathtocharts/tool/helm /usr/bin/helm

#update help
helm repo update

}

#This tool is much faster than helm for gathering all the stable and incubator helm charts. Helm pull takes forever. Its worth it and is easy to configure (see readme)

#function GETKUBECHARTS {
#gsutil -m cp -R gs://kubernetes-charts .
#gsutil -m cp -R gs://kubernetes-charts-incubator .
#ls -d kubernetes-charts/* > all.stable.charts.txt
#ls -d kubernetes-charts-incubator/* > all.incubator.charts.txt
#}

function GETKUBESTABLE {
cd $stablecharts
helm repo add stable https://kubernetes-charts.storage.googleapis.com
helm repo update

#This command finds every version of every helm chart and puts it into a file where we can actually do something with it
helm search repo stable --versions | cut -c -47 | grep -v NAME | awk '{$1=$1};1' | sed 's/ / --version /g' > fetch-stable.txt

#This commnd pulls all of the Rook charts contained within the index file
curl -L  https://kubernetes-charts.storage.googleapis.com/index.yaml | grep -v icon: | grep -v home | grep tgz | cut -c 7- |xargs -P 10 -n 1 wget

#curl down the rookcephindex file
curl -o index.yaml -L https://kubernetes-charts.storage.googleapis.com/index.yaml

#get the list of all the charts for the same type of for loop as we do with stable and incubator
mv fetch-stable.txt ../
cd ../
echo $(pwd)
ls -d $stablecharts/* > all.stable.charts.txt
}

function GETKUBEINCUBATOR {
cd $incubatorcharts
helm repo add incubator https://kubernetes-charts-incubator.storage.googleapis.com
helm repo update

#This command finds every version of every helm chart and puts it into a file where we can actually do something with it
helm search repo incubator --versions | cut -c -47 | grep -v NAME | awk '{$1=$1};1' | sed 's/ / --version /g' > fetch-incubator.txt

#This commnd pulls all of the Rook charts contained within the index file
curl -L  https://kubernetes-charts-incubator.storage.googleapis.com/index.yaml | grep -v icon: | grep -v home | grep tgz | cut -c 7- |xargs -P 10 -n 1 wget

#curl down the rookcephindex file
curl -o index.yaml -L https://kubernetes-charts-incubator.storage.googleapis.com/index.yaml

#get the list of all the charts for the same type of for loop as we do with stable and incubator
mv fetch-incubator.txt ../
cd ../
echo $(pwd)
ls -d $incubatorcharts/* >> all.stable.charts.txt
}

function GETKUBECHARTDEPS {
#Loop through stable and template out all charts to be able to easily grab image and tags

cat all.stable.charts.txt |xargs -P 10 -n 1 helm template | grep image: >>imagelist.txt
rm all.stable.charts.txt

#Loop through incubator and template out all charts to be able to easily grab image and tags

cat all.incubator.charts.txt |xargs -P 10 -n 1 helm template | grep image: >>imagelist.txt
rm all.incubator.charts.txt

}

#end fetch kubecharts charts

function GETROOKCHARTS {
cd $rookcephcharts
helm repo add rook-release https://charts.rook.io/release
helm repo update

#This command finds every version of every helm chart and puts it into a file where we can actually do something with it
helm search repo rook-release --versions | cut -c -47 | grep -v NAME | awk '{$1=$1};1' | sed 's/ / --version /g' > fetch-rookceph.txt

#This commnd pulls all of the Rook charts contained within the index file
curl -L  https://charts.rook.io/release/index.yaml | grep -v icon: | grep -v home | grep tgz | cut -c 7- |xargs -P 10 -n 1 wget

#curl down the rookcephindex file
curl -o index.yaml -L https://charts.rook.io/release/index.yaml

#get the list of all the charts for the same type of for loop as we do with stable and incubator
mv fetch-rookceph.txt ../
cd ../
echo $(pwd)
ls -d $rookcephcharts/* > rookceph.charts.txt

}

function GETROOKCHARTDEPS { 
#Loop through rookceph and template out all charts to be able to easily grab image and tags

cat rookceph.txt |xargs -P 10 -n 1 helm template | grep image: >>imagelist.txt
rm rookceph.charts.txt

}
#end fetch rookceph charts


#Rancher Stable

function GETRANCHERSTABLECHARTS {
cd $rancherstablecharts
helm repo add rancher-stable https://releases.rancher.com/server-charts/stable
helm repo update

#This command finds every version of every helm chart and puts it into a file where we can actually do something with it
helm search repo rancher-stable  --versions | cut -c -47 | grep -v NAME | awk '{$1=$1};1' | sed 's/ / --version /g' > fetch-rancherstable.txt

#This commnd pulls all of the Rancher Stable charts contained within the index file
curl -L  https://releases.rancher.com/server-charts/stable/index.yaml | grep -v icon: | grep -v home | grep tgz | cut -c 7- | sed 's/^/https:\/\/releases.rancher.com\/server-charts\/stable\//'  |xargs -P 10 -n 1 wget

#curl down the rookcephindex file
curl -o index.yaml -L https://releases.rancher.com/server-charts/stable/index.yaml 

#get the list of all the charts for the same type of for loop as we do with stable and incubator
mv fetch-rancherstable.txt ../
cd ../
echo $(pwd)
ls -d $rancherstablecharts/* > rancherstable.charts.txt
}

function GETRANCHERSTABLECHARTDEPS {
#Loop through rookceph and template out all charts to be able to easily grab image and tags

cat rancherstable.charts.txt |xargs -P 10 -n 1 helm template | grep image: >>imagelist.txt
rm rancherstable.charts.txt

}
#end fetch rancherstable charts


#Rancher Latest

function GETRANCHERLATESTCHARTS {
cd $rancherlatestcharts
helm repo add rancher-latest https://releases.rancher.com/server-charts/latest
helm repo update

#This command finds every version of every helm chart and puts it into a file where we can actually do something with it
helm search repo rancher-latest  --versions | cut -c -47 | grep -v NAME | awk '{$1=$1};1' | sed 's/ / --version /g' > fetch-rancherlatest.txt

#This commnd pulls all of the Rancher latest charts contained within the index file
curl -L  https://releases.rancher.com/server-charts/latest/index.yaml | grep -v icon: | grep -v home | grep tgz | cut -c 7- | sed 's/^/https:\/\/releases.rancher.com\/server-charts\/latest\//'  |xargs -P 10 -n 1 wget

#curl down the rancherlatestindex file
curl -o index.yaml -L https://releases.rancher.com/server-charts/latest/index.yaml

#get the list of all the charts for the same type of for loop as we do with stable and incubator
mv fetch-rancherlatest.txt ../
cd ../
echo $(pwd)
ls -d $rancherlatestcharts/* > rancherlatest.charts.txt

}

function GETRANCHERLATESTCHARTDEPS {
#Loop through rancher-latest and template out all charts to be able to easily grab image and tags

cat rancherlatest.charts.txt |xargs -P 10 -n 1 helm template | grep image: >>imagelist.txt
rm rancherlatest.charts.txt

}
#end fetch rancherlatest charts



#bitnami charts gathering here

function GETBITNAMICHARTS {
cd $bitnamicharts
helm repo add bitnami https://charts.bitnami.com/bitnami
helm repo update


#This command finds every version of every helm chart and puts it into a file where we can actually do something with it
helm search repo bitnami --versions | cut -c -47 | grep -v NAME | awk '{$1=$1};1' | sed 's/ / --version /g' > fetch-bitnami.txt


#This loop goes through and pulls all the helm charts. It should be made faster in the future with some sort of distributed pull tool. 
curl -L https://charts.bitnami.com/bitnami/index.yaml | grep -v icon: | grep -v home | grep tgz | cut -c 7- |xargs -P 10 -n 1 wget

#curl down the bitnami index file
curl -o index.yaml -L https://charts.bitnami.com/bitnami/index.yaml

#get the list of all the charts for the same type of for loop as we do with stable and incubator
mv fetch-bitnami.txt ../
cd ../
echo $(pwd)
ls -d $bitnamicharts/* > bitnami.charts.txt
}

function GETBITNAMICHARTDEPS {
#Loop through bitnami and template out all charts to be able to easily grab image and tags

cat bitnami.charts.txt |xargs -P 10 -n 1 helm template | grep image: >>imagelist.txt
rm bitnami.charts.txt

}
#end fetch bitnami charts

#vmware charts gathering here
function GETVMWARECHARTS {
cd $vmwarecharts
helm repo add vmware https://vmware-tanzu.github.io/helm-charts
helm repo update

#This command finds every version of every helm chart and puts it into a file where we can actually do something with it
helm search repo vmware/ --versions | cut -c -25 | grep -v NAME | awk '{$1=$1};1' | sed 's/ / --version /g' > fetch-vmware.txt

#This commnd pulls all of the VMWare  charts contained within the index file
curl -L https://vmware-tanzu.github.io/helm-charts/index.yaml | grep -v icon: | grep -v home | grep tgz | cut -c 7- |xargs -P 10 -n 1 wget

#curl down the vmware index file
curl -o index.yaml -L https://vmware-tanzu.github.io/helm-charts/index.yaml
sed -i "s/\*//g" index.yaml

#get the list of all the charts for the same type of for loop as we do with stable and incubator
mv fetch-vmware.txt ../
cd ../
echo $(pwd)
ls $vmwarecharts > vmware.charts.txt
}

function GETVMWARECHARTDEPS {
#Loop through vmware and template out all charts to be able to easily grab image and tags
for f in `cat vmware.charts.txt`;
do
echo
echo
echo
echo

mkdir /tmp/$f-final
#helm template --output-dir /tmp/$f-final $vmwarecharts/$f
helm template --namespace velero --set configuration.provider=aws --set configuration.backupStorageLocation.name=aws --set configuration.backupStorageLocation.bucket=bucket --set configuration.backupStorageLocation.config.region=region --set configuration.volumeSnapshotLocation.name=whatever --set configuration.volumeSnapshotLocation.config.region=region --set image.repository=velero/velero --set image.tag=v1.2.0 --set image.pullPolicy=IfNotPresent --set initContainers[0].name=velero-plugin-for-aws --set initContainers[0].image=velero/velero-plugin-for-aws:v1.0.0 --set initContainers[0].volumeMounts[0].mountPath=/target --output-dir /tmp/$f-final $vmwarecharts/$f
grep -hR image: /tmp/$f-final >>./imagelist.txt
cp $vmwarecharts/$f .
tar xvf $f
grep -hR image: |grep -v Values |grep -v Binary |grep -v .list |grep -v .txt |sed 's/^.*\(image.*\).*$/\1/' |sed 's/image://' |sed 's/\"//g' |sed 's/\#//g' |sed 's/ //g' |grep -v "'" |sort |uniq >>./imagelist.txt
rm -rf $f
rm -rf /tmp/$f-final
rm -rf /tmp/$f
done
rm vmware.charts.txt
}
#end fetch vmware charts

function CLEANUP {
#Clean up container list
cat imagelist.txt |sed 's/^.*\(image.*\).*$/\1/' |sed 's/image://' |sed 's/\"//g' |grep -v "'" |sed 's/^ *//;s/ *$//;s/  */ /;' |sort |uniq  >~/closed-env-container-images.txt
rm imagelist.txt


#move chart files to tmp foler for processing

mv stable /tmp/
mv incubator /tmp/
mv bitnami-charts /tmp/
mv vmware-charts /tmp/
mv rookceph-charts /tmp/
mv rancherlatest-charts /tmp/
mv rancherstable-charts /tmp/
}
#end moving fetched helm charts to /tmp/

#get the pics using a series of for loops

function GETCHARTPICS {
cat /tmp/rancherlatest-charts/index.yaml | grep icon | sed 's/    icon: //g' > iconlist; uniq iconlist output.txt; cat -n output.txt | sed 's/^.......//' > imagelist_rancherlatest.txt; rm output.txt iconlist
cat /tmp/rancherstable-charts/index.yaml | grep icon | sed 's/    icon: //g' > iconlist; uniq iconlist output.txt; cat -n output.txt | sed 's/^.......//' > imagelist_rancherstable.txt; rm output.txt iconlist
cat /tmp/stable/index.yaml | grep icon | sed 's/    icon: //g' > iconlist; uniq iconlist output.txt; cat -n output.txt | sed 's/^.......//' > imagelist_stable.txt; rm output.txt iconlist
cat /tmp/rookceph-charts/index.yaml | grep icon | sed 's/    icon: //g' > iconlist; uniq iconlist output.txt; cat -n output.txt | sed 's/^.......//' > imagelist_rookceph.txt; rm output.txt iconlist
cat /tmp/rancherlatest-charts/index.yaml | grep icon | sed 's/    icon: //g' > iconlist; uniq iconlist output.txt; cat -n output.txt | sed 's/^.......//' > imagelist_rancherlatest.txt; rm output.txt iconlist
cat /tmp/rancherstable-charts/index.yaml | grep icon | sed 's/    icon: //g' > iconlist; uniq iconlist output.txt; cat -n output.txt | sed 's/^.......//' > imagelist_rancherstable.txt; rm output.txt iconlist
cat /tmp/incubator/index.yaml | grep icon | sed 's/    icon: //g' >> iconlist; uniq iconlist output.txt; cat -n output.txt | sed 's/^.......//' > imagelist_incubator.txt; rm output.txt iconlist
cat /tmp/bitnami-charts/index.yaml | grep icon | sed 's/    icon: //g' >> iconlist; uniq iconlist output.txt; cat -n output.txt | sed 's/^.......//' > imagelist_bitnami.txt; rm output.txt iconlist
cat /tmp/vmware-charts/index.yaml | grep icon | sed 's/    icon: //g' >> iconlist; uniq iconlist output.txt; cat -n output.txt | sed 's/^.......//'  > imagelist_vmware.txt; rm output.txt iconlist
rm -rf /tmp/chartpics/
mkdir -p /tmp/chartpics/ 

#grab stable pictures and icons
for f in `cat imagelist_stable.txt`;
do
d=`echo $f |sed 's/https\:\/\///g' | sed 's/\//-/g'`
j=$(echo $f |  sed 's;/;\\/;g')
sed -i "/icon: $j/c\    icon: $imagestoreurl/$bucket/$d" $indexfilestable
echo go get image "### $(echo $f| sed 's|.*/||') ###";
curl -o /tmp/chartpics/$(echo $f | sed 's/https\:\/\///g' | sed 's/\//-/g') $f &
done

#grab rancher-latest pictures and icons
for f in `cat imagelist_rancherlatest.txt`;
do
d=`echo $f |sed 's/https\:\/\///g' | sed 's/\//-/g'`
j=$(echo $f |  sed 's;/;\\/;g')
sed -i "/icon: $j/c\    icon: $imagestoreurl/$bucket/$d" $indexfilerancherlatest
echo go get image "### $(echo $f| sed 's|.*/||') ###";
curl -o /tmp/chartpics/$(echo $f | sed 's/https\:\/\///g' | sed 's/\//-/g') $f &
done

#grab rancher-stable pictures and icons
for f in `cat imagelist_rancherstable.txt`;
do
d=`echo $f |sed 's/https\:\/\///g' | sed 's/\//-/g'`
j=$(echo $f |  sed 's;/;\\/;g')
sed -i "/icon: $j/c\    icon: $imagestoreurl/$bucket/$d" $indexfilerancherstable
echo go get image "### $(echo $f| sed 's|.*/||') ###";
curl -o /tmp/chartpics/$(echo $f | sed 's/https\:\/\///g' | sed 's/\//-/g') $f &
done

#grab rookceph pictures and icons
for f in `cat imagelist_rookceph.txt`;
do
d=`echo $f |sed 's/https\:\/\///g' | sed 's/\//-/g'`
j=$(echo $f |  sed 's;/;\\/;g')
sed -i "/icon: $j/c\    icon: $imagestoreurl/$bucket/$d" $indexfilerookceph
echo go get image "### $(echo $f| sed 's|.*/||') ###";
curl -o /tmp/chartpics/$(echo $f | sed 's/https\:\/\///g' | sed 's/\//-/g') $f &
done

#grab incubator pictures and icons
for f in `cat imagelist_incubator.txt`;
do
d=`echo $f |sed 's/https\:\/\///g' | sed 's/\//-/g'`
j=$(echo $f |  sed 's;/;\\/;g')
sed -i "/icon: $j/c\    icon: $imagestoreurl/$bucket/$d" $indexfileincubator
echo go get image "### $(echo $f| sed 's|.*/||') ###";
curl -o /tmp/chartpics/$(echo $f | sed 's/https\:\/\///g' | sed 's/\//-/g') $f
done

#Grab bitnami pictures and icons
for f in `cat imagelist_bitnami.txt`;
do
d=`echo $f |sed 's/https\:\/\///g' | sed 's/\//-/g'`
j=$(echo $f |  sed 's;/;\\/;g')
sed -i "/icon: $j/c\    icon: $imagestoreurl/$bucket/$d" $indexfilebitnami
echo go get image "### $(echo $f| sed 's|.*/||') ###";
curl -o /tmp/chartpics/$(echo $f | sed 's/https\:\/\///g' | sed 's/\//-/g') $f
done

#Grab vmware pictures and icons
for f in `cat imagelist_vmware.txt`;
do
d=`echo $f |sed 's/https\:\/\///g' | sed 's/\//-/g'`
j=$(echo $f |  sed 's;/;\\/;g')
sed -i "/icon: $j/c\    icon: $imagestoreurl/$bucket/$d" $indexfilevmware
echo go get image "### $(echo $f| sed 's|.*/||') ###";
curl -o /tmp/chartpics/$(echo $f | sed 's/https\:\/\///g' | sed 's/\//-/g') $f
done

}
# end of the get the pictures section


#this section is for fixing the index files, right now I am not confident it does bitnami right but kubernetes and kubernetes incubator are correct
function FIXINDEXFILES {

sed -i "s/kubernetes-charts.storage.googleapis.com/$chartstoreurl\/stable/g"  $indexfilestable
sed -i "s/kubernetes-charts-incubator.storage.googleapis.com/$chartstoreurl\/incubator/g"  $indexfileincubator
sed -i "s/charts.rook.io/$chartstoreurl\/rookceph/g" $indexfilerookceph
sed -i "s/charts.bitnami.com/$chartstoreurl/g" $indexfilebitnami
sed -i "s/bitnami.com/$chartstoreurl/g" $indexfilebitnami
sed -i "s/rancher-/\https:\/\/$chartstoreurl\/rancherstable\//g" $indexfilerancherstable
sed -i "s/rancher-/\https:\/\/$chartstoreurl\/rancherlatest\//g" $indexfilerancherlatest
sed -i "s/vmware-tanzu.github.io\/helm-charts/$chartstoreurl\/vmware/g" $indexfilevmware
sed -i "s/github.com\/vmware-tanzu\/helm-charts\/releases\/download\/.*\//$chartstoreurl\/vmware\//g" $indexfilevmware
}
# end of indexfile fixting seciton

#create the tar file and clean up list files
function CREATETAR {
cd /tmp

#i think this is just a duplicate command of above now
#sed -i "s/charts.bitnami.com/$chartstoreurl/g" bitnami-charts/index.yaml
rm incubator/*-.tgz.1
mv stable/*-.tgz.1
mv incubator kubernetes-charts-incubator
mv stable kubernetes-charts
tar -cf ~/helm-charts.tar  kubernetes-charts-incubator kubernetes-charts chartpics bitnami-charts vmware-charts rancherstable-charts rookceph-charts rancherlatest-charts
rm -rf kubernetes-charts-incubator kubernetes-charts chartpics bitnami-charts vmware-charts rancherstable-charts rookceph-charts rancherlatest-charts

#end of clean up and tar creation

echo
echo

cd $pathtocharts
cd ../
rm -rf ./charts
cd $pathtocharts
rm stable.charts.txt incubator.charts.txt imagelist_incubator.txt  imagelist_stable.txt fetch-bitnami.txt imagelist_bitnami.txt fetch-vmware.txt imagelist_vmware.txt imagelist_rookceph.txt imagelist_rancherlatest.txt imagelist_rancherstable.txt fetch-rancherlatest.txt fetch-rancherstable.txt fetch-rookceph.txt fetch-stable.txt fetch-incubator.txt
}

# transfer new container list after copying and finding the new unique images from last run
function TOTHEBUCKETS {

# create date variable
d=`date +%Y-%m-%d-%H-%M`

# move new container list to tmp folder
mv ~/closed-env-container-images.txt /tmp/full-list

# copy yesterdays image list from minio and s3 bucket
mc cp minio/helmcharts/minio-closed-env-container-images.txt ~/yester-minio-list
aws s3 cp s3://rancher-container-list/s3-closed-env-container-images.txt ~/yester-s3-list

# append todays list to yesterdays list
cat /tmp/full-list >> ~/yester-minio-list
cat /tmp/full-list >> ~/yester-s3-list

# copy old list from minio temp and s3 full_container_list bucket
mc mv minio/helmcharts/full_container_list/temp-minio-old.txt ~/
aws s3 mv s3://rancher-container-list/full_container_list/temp-s3-old.txt ~/

# append old list to yesterdays list
cat ~/temp-minio-old.txt >> ~/yester-minio-list
cat ~/temp-s3-old.txt >> ~/yester-s3-list

# copy then move new full list to full_container_list buckets
mc cp /tmp/full-list minio/helmcharts/full_container_list/temp-minio-old.txt
aws s3 mv /tmp/full-list s3://rancher-container-list/full_container_list/temp-s3-old.txt

# sort out new unique list and rename back to original
cat ~/yester-minio-list | sort | uniq -u > ~/minio-closed-env-container-images.txt
cat ~/yester-s3-list | sort | uniq -u > ~/s3-closed-env-container-images.txt

# rename and move yesterdays list in minio and s3 to old_container_lists bucket
mc mv minio/helmcharts/minio-closed-env-container-images.txt minio/helmcharts/old_container_lists/minio-closed-env-container-images-$d
aws s3 mv s3://rancher-container-list/s3-closed-env-container-images.txt s3://rancher-container-list/old_container_lists/s3-closed-env-container-images-$d

# move new unique list back to minio and s3 bucket
mc cp ~/minio-closed-env-container-images.txt minio/helmcharts
aws s3 cp ~/s3-closed-env-container-images.txt s3://rancher-container-list

# move old helm-charts.tar to minio and s3 old_charts bucket
mc mv minio/helmcharts/helm-charts.tar minio/helmcharts/old_charts/helm-charts-$d.tar
aws s3 mv s3://rancher-container-list/helm-charts.tar s3://rancher-container-list/old_charts/helm-charts-$d.tar
	
# move new helm-charts.tar
mc cp ~/helm-charts.tar minio/helmcharts
aws s3 cp ~/helm-charts.tar s3://rancher-container-list

# remove any charts older than 7 days
mc rm --recursive --force --older-than 7d minio/helmcharts/old_charts/

# remove any container lists older than 30 days
mc rm --recursive --force --older-than 30d minio/helmcharts/old_container_lists/

# cleanup
rm -rf ~/helm-charts.tar ~/minio-closed-env-container-images.txt ~/s3-closed-env-container-images.txt ~/yester-minio-list ~/yester-s3-list ~/temp-minio-old.txt ~/temp-s3-old.txt

# show updated buckets
mc ls minio/helmcharts
aws s3 ls s3://rancher-container-list

echo happy dance
}

#aws s3 cp s3://rancher-container-list/closed-env-container-images.txt3 s3://rancher-container-list/closed-env-container-images.txt4
#aws s3 cp s3://rancher-container-list/closed-env-container-images.txt2 s3://rancher-container-list/closed-env-container-images.txt3
#aws s3 cp s3://rancher-container-list/closed-env-container-images.txt1 s3://rancher-container-list/closed-env-container-images.txt2
#aws s3 cp s3://rancher-container-list/closed-env-container-images.txt s3://rancher-container-list/closed-env-container-images.txt1
#aws s3 cp ~/closed-env-container-images.txt  s3://rancher-container-list/

#aws s3 cp s3://rancher-container-list/ /tmp/ --recursive

#grep -Fxvf /tmp/closed-env-container-images.txt4 /tmp/closed-env-container-images.txt >new-closed-env-container-images.txt
#grep -Fxvf /tmp/closed-env-container-images.txt3 /tmp/closed-env-container-images.txt >>new-closed-env-container-images.txt
#grep -Fxvf /tmp/closed-env-container-images.txt2 /tmp/closed-env-container-images.txt >>new-closed-env-container-images.txt
#grep -Fxvf /tmp/closed-env-container-images.txt1 /tmp/closed-env-container-images.txt >>new-closed-env-container-images.txt

#cat new-closed-env-container-images.txt |sort |uniq  >uniqnew-closed-env-container-images.txt

#rm /tmp/closed-env-container-images.txt
#rm /tmp/closed-env-container-images.txt1
#rm /tmp/closed-env-container-images.txt2
#rm /tmp/closed-env-container-images.txt3
#rm /tmp/closed-env-container-images.txt4

#aws s3 cp uniqnew-closed-env-container-images.txt s3://rancher-container-list/
#rm uniqnew-closed-env-container-images.txt
#rm new-closed-env-container-images.txt