build: Refactor the Minio sync logic and separate the helm chart release
continuous-integration/drone/push Build is passing
Details
continuous-integration/drone/push Build is passing
Details
This commit is contained in:
parent
2eef222b49
commit
5e19dbd9d8
|
@ -1,4 +1,17 @@
|
|||
.git/**/*
|
||||
.git
|
||||
|
||||
node_modules/**/*
|
||||
node_modules
|
||||
|
||||
static/**/*
|
||||
static
|
||||
|
||||
scripts/**/*
|
||||
scripts
|
||||
|
||||
chart/**/*
|
||||
chart
|
||||
|
||||
kube/**/*
|
||||
kube
|
||||
|
|
62
.drone.yml
62
.drone.yml
|
@ -1,10 +1,10 @@
|
|||
---
|
||||
# ----------------------------------------------
|
||||
# -- Build the site and push it to the registry
|
||||
# -- Update the helm chart
|
||||
# ----------------------------------------------
|
||||
kind: pipeline
|
||||
type: kubernetes
|
||||
name: Build badhouseplants.net
|
||||
name: Publish the helm chart
|
||||
|
||||
trigger:
|
||||
event:
|
||||
|
@ -21,8 +21,20 @@ steps:
|
|||
- helm package chart -d chart-package
|
||||
- helm repo add --username allanger --password $GITEA_TOKEN badhouseplants-net https://git.badhouseplants.net/api/packages/badhouseplants/helm
|
||||
- helm cm-push "./chart-package/$(ls chart-package)" badhouseplants-net
|
||||
---
|
||||
# ----------------------------------------------
|
||||
# -- Build the site and push it to the registry
|
||||
# ----------------------------------------------
|
||||
kind: pipeline
|
||||
type: kubernetes
|
||||
name: Build badhouseplants.net
|
||||
|
||||
- name: Init git submodules with themes
|
||||
trigger:
|
||||
event:
|
||||
- push
|
||||
|
||||
steps:
|
||||
- name: Download submodules with themes
|
||||
image: alpine/git
|
||||
commands:
|
||||
- git submodule update --init --recursive
|
||||
|
@ -61,6 +73,42 @@ steps:
|
|||
- ${DRONE_COMMIT_SHA}
|
||||
- latest
|
||||
|
||||
- name: Download media files
|
||||
image: alpine/git
|
||||
commands:
|
||||
- git lfs pull
|
||||
|
||||
- name: Sync pictures from lfs to Minio
|
||||
image: rclone/rclone:latest
|
||||
when:
|
||||
branch:
|
||||
exclude:
|
||||
- main
|
||||
environment:
|
||||
RCLONE_CONFIG_CONTENT:
|
||||
from_secret: RCLONE_CONFIG_CONTENT_PRIVATE
|
||||
RCLONE_CONFIG: /tmp/rclone.conf
|
||||
commands:
|
||||
- echo "$RCLONE_CONFIG_CONTENT" > $RCLONE_CONFIG
|
||||
- apk update
|
||||
- apk add git make
|
||||
- make upload_static
|
||||
|
||||
- name: Sync pictures to the main Minio bucket
|
||||
image: rclone/rclone:latest
|
||||
when:
|
||||
branch:
|
||||
- main
|
||||
environment:
|
||||
RCLONE_CONFIG_CONTENT:
|
||||
from_secret: RCLONE_CONFIG_CONTENT_PRIVATE
|
||||
RCLONE_CONFIG: /tmp/rclone.conf
|
||||
commands:
|
||||
- echo "$RCLONE_CONFIG_CONTENT" > $RCLONE_CONFIG
|
||||
- apk update
|
||||
- apk add git make
|
||||
- make sync_static_with_main
|
||||
|
||||
- name: Deploy a preview ApplicationSet
|
||||
image: alpine/k8s:1.24.10
|
||||
when:
|
||||
|
@ -130,12 +178,16 @@ steps:
|
|||
- argocd app sync -l app=badhouseplants -l branch=$DRONE_BRANCH
|
||||
- argocd app wait -l app=badhouseplants -l branch=$DRONE_BRANCH
|
||||
|
||||
- name: Cleanup container registry
|
||||
image: alpine
|
||||
- name: Cleanup everything
|
||||
image: rclone/rclone:latest
|
||||
environment:
|
||||
RCLONE_CONFIG_CONTENT:
|
||||
from_secret: RCLONE_CONFIG_CONTENT_PRIVATE
|
||||
RCLONE_CONFIG: /tmp/rclone.conf
|
||||
GITEA_TOKEN:
|
||||
from_secret: GITEA_TOKEN
|
||||
commands:
|
||||
- echo "$RCLONE_CONFIG_CONTENT" > $RCLONE_CONFIG
|
||||
- apk update
|
||||
- apk add curl jq perl git
|
||||
- ./scripts/cleanup.pl
|
||||
|
|
|
@ -0,0 +1 @@
|
|||
static/* filter=lfs diff=lfs merge=lfs -text
|
|
@ -1,5 +1,5 @@
|
|||
node_modules
|
||||
static
|
||||
#static
|
||||
content/cv/index.pdf
|
||||
public
|
||||
resources
|
||||
|
|
|
@ -1,4 +1,5 @@
|
|||
FROM git.badhouseplants.net/badhouseplants/hugo-builder
|
||||
WORKDIR /src
|
||||
COPY . /src
|
||||
RUN ls -la /src
|
||||
ENTRYPOINT ["hugo"]
|
||||
|
|
8
Makefile
8
Makefile
|
@ -1,9 +1,15 @@
|
|||
upload_static:
|
||||
rclone copy -P static badhouseplants-minio:/badhouseplants-static
|
||||
rclone copy -P static badhouseplants-minio:/badhouseplants-net/$(shell git rev-parse HEAD)/
|
||||
|
||||
sync_static_with_main:
|
||||
rclone sync -P badhouseplants-minio:/badhouseplants-net/$(shell git rev-parse HEAD)/ badhouseplants-minio:/badhouseplants-net-main/
|
||||
|
||||
get_static:
|
||||
rclone copy -P badhouseplants-public:/badhouseplants-static static
|
||||
|
||||
init_static:
|
||||
rclone copy -P badhouseplants-public:/badhouseplants-static badhouseplants-minio:/badhouseplants-net/$(shell git rev-parse HEAD)/
|
||||
|
||||
submodules:
|
||||
git submodule update --init --recursive
|
||||
|
||||
|
|
|
@ -2,7 +2,7 @@ apiVersion: v2
|
|||
name: badhouseplants-net
|
||||
description: A Helm chart for Kubernetes
|
||||
type: application
|
||||
version: 0.5.8
|
||||
version: 0.6.0
|
||||
appVersion: "4.20.0"
|
||||
dependencies:
|
||||
- name: remark42
|
||||
|
|
|
@ -35,7 +35,7 @@ spec:
|
|||
- sh
|
||||
args:
|
||||
- "-c"
|
||||
- "rclone copy -P badhouseplants-public:/badhouseplants-static /static"
|
||||
- {{ .Values.rclone.command }}
|
||||
volumeMounts:
|
||||
- name: rclone-config
|
||||
mountPath: "/tmp"
|
||||
|
|
|
@ -17,6 +17,7 @@ nginx:
|
|||
tag: latest
|
||||
|
||||
rclone:
|
||||
command: "rclone copy -P badhouseplants-public:/badhouseplants-net-main /static"
|
||||
container:
|
||||
name: rclone
|
||||
resources: {}
|
||||
|
|
|
@ -0,0 +1,26 @@
|
|||
---
|
||||
title: "Design a Scalable System"
|
||||
date: 2023-03-06T21:18:23+01:00
|
||||
draft: true
|
||||
ShowToc: true
|
||||
cover:
|
||||
image: "cover.png"
|
||||
caption: "Design a Scalable System"
|
||||
relative: false
|
||||
responsiveImages: false
|
||||
---
|
||||
|
||||
Well, after I've posted my argo serie, I've found out that I couldn't really make myself understood. So now I want to talk more not about the way of implementation, but rather about the consequinces of different implementations. And maybe I will e able to finally make a point about why I don't like Terraform and why I think that ArgoCD is mostly mis-used by almost any SRE team I know.
|
||||
|
||||
But first I'll try to describe how I see myself as a part of a team, the team as a part of a bigger team, and all the teams across different companies as links in the bigger chain.
|
||||
|
||||
This is how I used to see development teams before:
|
||||
|
||||
![Chain](/posts/design-a-scalable-system/chain-1.png)
|
||||
|
||||
The whole team is using something as a service, for example `AWS`, the whole team is working together and producing something that is passed to a customer. But apparently this approach is only applicable to small teams, and I think it's working just fine. But there is a problem. Teams tend to grow without an understanding that they are growing, hence they keep acting like they're small but in the same time they don't change the workflow, and brick-by-brick they are building something that eventually is something unscalable at first and later unmaintanble.
|
||||
|
||||
Example of an evolution like this:
|
||||
|
||||
Let's take a regular Java(backend) + JS(frontend) web application
|
||||
|
|
@ -34,3 +34,5 @@
|
|||
auth:
|
||||
anonymous: true
|
||||
secretKey: $ARGO_REMARK42_SECRET
|
||||
rclone:
|
||||
command: "rclone copy -P badhouseplants-public:/badhouseplants-net/$ARGO_APP_IMAGE_TAG /static"
|
||||
|
|
|
@ -30,13 +30,21 @@ my $commits = "git fetch && git log --format=format:%H --all";
|
|||
my @commits_out = `$commits`;
|
||||
chomp @commits_out;
|
||||
push @commits_out, 'latest';
|
||||
# --------------------------------------
|
||||
# -- Rclone variables
|
||||
# -------------------------------------
|
||||
my $dirs = "rclone lsf badhouseplants-minio:/badhouseplants-net";
|
||||
my @dirs_out = `$dirs`;
|
||||
chomp @dirs_out;
|
||||
# ---------------------------------------
|
||||
# -- Compare builds to commits
|
||||
# -- And remove obsolete imgages from
|
||||
# -- registry
|
||||
# ---------------------------------------
|
||||
print "Cleaning up the container registry\n";
|
||||
foreach my $line (@builds_out)
|
||||
{
|
||||
print "Checking if $line is in @commits_out\n\n";
|
||||
if ( ! grep( /^$line$/, @commits_out ) ) {
|
||||
my $cmd = "curl -X 'DELETE' -s \"$gitea_api/$line\" -H 'accept: application/json' -u $gitea_user:$gitea_token || true";
|
||||
print "Removing ${line}\n\n";
|
||||
|
@ -45,3 +53,17 @@ foreach my $line (@builds_out)
|
|||
}
|
||||
}
|
||||
|
||||
print "Cleaning up Minio\n";
|
||||
foreach my $line (@dirs_out)
|
||||
{
|
||||
print "Checking if $line is in @commits_out\n\n";
|
||||
chop($line);
|
||||
if ( ! grep( /^$line$/, @commits_out ) ) {
|
||||
my $cmd = "rclone purge badhouseplants-minio:/badhouseplants-net/$line";
|
||||
print "Removing ${line}\n\n";
|
||||
my $output = `$cmd`;
|
||||
print "$output \n";
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
|
|
@ -0,0 +1,10 @@
|
|||
#!/usr/bin/env bash
|
||||
set -e
|
||||
|
||||
#=== 'prev-commit' solution by o_O Tync
|
||||
#commit_hash=$(git rev-parse --verify HEAD)
|
||||
commit=$(git log -1 --pretty="%H%n%ci") # hash \n date
|
||||
commit_date=$(echo "$commit" | head -2 | tail -1) # 2010-12-28 05:16:23 +0300
|
||||
commit_hash=$(echo "$commit" | head -1)
|
||||
echo "$(git rev-parse HEAD)" > /tmp/test
|
||||
echo "prev_commit='$commit_hash'\ndate='$commit_date'\nbranch='$branch'\n" > /tmp/test.txt
|
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
static/dont-use-argocd-for-infrastructure/3-clusters-and-gitea.png (Stored with Git LFS)
Normal file
BIN
static/dont-use-argocd-for-infrastructure/3-clusters-and-gitea.png (Stored with Git LFS)
Normal file
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Loading…
Reference in New Issue