From 0dd2a034ce2716c70b7e91787b8c4a06bb16eef7 Mon Sep 17 00:00:00 2001 From: Eray Arslan Date: Sat, 24 Feb 2024 23:44:07 +0300 Subject: [PATCH] chore: run integration on multi node cluster --- .github/workflows/dcp.yml | 2 +- .github/workflows/integration.yml | 12 +- .gitignore | 3 +- Makefile | 7 +- docker-compose.yml | 13 -- scripts/cluster.sh | 7 + scripts/create_cluster.sh | 230 +++++++++++++++++++++++++++ scripts/delete_cluster.sh | 13 ++ test/couchbase/Dockerfile | 9 -- test/couchbase/configure.sh | 41 ----- test/integration/Dockerfile | 10 -- test/integration/config.yml | 10 +- test/integration/integration_test.go | 7 +- 13 files changed, 273 insertions(+), 91 deletions(-) delete mode 100644 docker-compose.yml create mode 100644 scripts/cluster.sh create mode 100644 scripts/create_cluster.sh create mode 100644 scripts/delete_cluster.sh delete mode 100644 test/couchbase/Dockerfile delete mode 100644 test/couchbase/configure.sh delete mode 100644 test/integration/Dockerfile diff --git a/.github/workflows/dcp.yml b/.github/workflows/dcp.yml index 01d9558..48a7b07 100644 --- a/.github/workflows/dcp.yml +++ b/.github/workflows/dcp.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - version: [ "5.0.1", "5.1.3", "5.5.6", "6.0.5", "6.5.2", "6.6.6", "7.0.5", "7.1.6", "7.2.3" ] + version: [ "5.0.1", "5.1.3", "5.5.6", "6.0.5", "6.5.2", "6.6.6", "7.0.5", "7.1.6", "7.2.4" ] steps: - name: Checkout uses: actions/checkout@v3 diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index f56cadb..300ecc3 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -10,6 +10,9 @@ on: jobs: integration: runs-on: ubuntu-latest + strategy: + matrix: + version: [ "7.1.6", "7.2.4" ] steps: - name: Checkout uses: actions/checkout@v3 @@ -22,11 +25,16 @@ jobs: go-version: '1.20' - uses: actions/checkout@v3 - - name: Start containers - run: make compose + - name: Start cluster + run: make create-cluster version=${{ matrix.version }} - name: Integration Test run: go test -v test/integration/integration_test.go env: INPUT_PUBLISH: false GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - uses: actions/checkout@v3 + - name: Delete cluster + if: always() + run: make delete-cluster diff --git a/.gitignore b/.gitignore index 0ea7bdb..dd7c59e 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,5 @@ # Dependency directories (remove the comment below to include it) # vendor/ -.idea \ No newline at end of file +.idea +test/integration/hosts \ No newline at end of file diff --git a/Makefile b/Makefile index 92a68cc..5070f44 100644 --- a/Makefile +++ b/Makefile @@ -19,8 +19,11 @@ lint: test: go test ./... -bench . -compose: - docker compose up --wait --build --force-recreate --remove-orphans +create-cluster: + bash scripts/create_cluster.sh --version $(version) + +delete-cluster: + bash scripts/delete_cluster.sh docker-build: docker build --progress=plain -t docker.io/trendyoltech/dcp . \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 6ecbec3..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,13 +0,0 @@ -version: "3.8" -services: - couchbase: - build: - context: "test/couchbase" - ports: - - "8091:8091" - - "11210:11210" - healthcheck: - test: [ "CMD", "curl", "-f", "http://user:123456@localhost:8091/pools/default/buckets/dcp-test" ] - interval: 2s - timeout: 3s - retries: 60 \ No newline at end of file diff --git a/scripts/cluster.sh b/scripts/cluster.sh new file mode 100644 index 0000000..3cbfc98 --- /dev/null +++ b/scripts/cluster.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +sh create_cluster.sh + +read -r -p "press enter to delete cluster _ " + +sh delete_cluster.sh \ No newline at end of file diff --git a/scripts/create_cluster.sh b/scripts/create_cluster.sh new file mode 100644 index 0000000..8ef90fc --- /dev/null +++ b/scripts/create_cluster.sh @@ -0,0 +1,230 @@ +#!/bin/bash + +POSITIONAL_ARGS=() + +VERSION=7.2.4 +NAME_PREFIX="couchbase_node_" +SUBNET="13.37.11.0/24" +TOTAL_NODE=3 +REST_PORT=8091 +MEMORY_AS_MB=512 +USERNAME=user +PASSWORD=123456 +HOSTS_PATH="" + +while [[ $# -gt 0 ]]; do + case $1 in + -v|--version) + VERSION="$2" + shift + shift;; + -np|--name-prefix) + NAME_PREFIX="$2" + shift + shift;; + -s|--subnet) + SUBNET="$2" + shift + shift;; + -tn|--total-node) + TOTAL_NODE="$2" + shift + shift;; + -rp|--rest-port) + REST_PORT="$2" + shift + shift;; + -mam|--memory-as-mb) + MEMORY_AS_MB="$2" + shift + shift;; + -u|--username) + USERNAME="$2" + shift + shift;; + -p|--password) + PASSWORD="$2" + shift + shift;; + -hp|--hosts-path) + HOSTS_PATH="$2" + shift + shift;; + -*|--*) + echo "Unknown option $1" + exit 1;; + *) + POSITIONAL_ARGS+=("$1") + shift;; + esac +done + +set -- "${POSITIONAL_ARGS[@]}" + +echo "NAME_PREFIX = $NAME_PREFIX" +echo "VERSION = $VERSION" +echo "SUBNET = $SUBNET" +echo "TOTAL_NODE = $TOTAL_NODE" +echo "REST_PORT = $REST_PORT" +echo "MEMORY_AS_MB = $MEMORY_AS_MB" +echo "USERNAME = $USERNAME" +echo "PASSWORD = $PASSWORD" +echo "HOSTS_PATH = $HOSTS_PATH" +echo "---------------------------" + +MASTER="" +KNOWN_NODES="" + +echo "creating network" +docker network create couchbase --subnet=$SUBNET + +check_db() { + URL="http://$1:$REST_PORT/pools" + curl --silent --max-time 5 "$URL" > /dev/null + echo $? +} + +get_buckets() { + URL="http://$1:$REST_PORT/pools/default/buckets" + RESPONSE=$(curl --silent --max-time 5 "$URL" -X 'GET' -u "$USERNAME:$PASSWORD") + echo "$RESPONSE" +} + +get_rebalance_logs() { + URL="http://$1:$REST_PORT/logs/rebalanceReport" + RESPONSE=$(curl --silent --max-time 5 "$URL" -X 'GET' -u "$USERNAME:$PASSWORD") + echo "$RESPONSE" +} + +# url, data +do_post() { + RESPONSE=$(curl -s -w ",status_code_%{http_code}" "$1" -X 'POST' -u "$USERNAME:$PASSWORD" --data "$2") + if [[ "$RESPONSE" == *"status_code_200" ]] || [[ "$RESPONSE" == *"status_code_202" ]]; then + echo "$RESPONSE" + else + echo "error when POST $1 with $2 = $RESPONSE" + fi +} + +install_travel_sample() { + URL="http://$1:$REST_PORT/sampleBuckets/install" + RESPONSE=$(do_post "$URL" '["travel-sample"]') + echo "install_travel_sample > $RESPONSE" +} + +node_init() { + URL="http://$1:$REST_PORT/nodeInit" + RESPONSE=$(do_post "$URL" "hostname=$1&dataPath=%2Fopt%2Fcouchbase%2Fvar%2Flib%2Fcouchbase%2Fdata&indexPath=%2Fopt%2Fcouchbase%2Fvar%2Flib%2Fcouchbase%2Fdata&eventingPath=%2Fopt%2Fcouchbase%2Fvar%2Flib%2Fcouchbase%2Fdata&javaHome=&analyticsPath=%2Fopt%2Fcouchbase%2Fvar%2Flib%2Fcouchbase%2Fdata") + echo "node_init > $RESPONSE" +} + +create_cluster() { + URL="http://$1:$REST_PORT/clusterInit" + RESPONSE=$(do_post "$URL" "hostname=$1&dataPath=%2Fopt%2Fcouchbase%2Fvar%2Flib%2Fcouchbase%2Fdata&indexPath=%2Fopt%2Fcouchbase%2Fvar%2Flib%2Fcouchbase%2Fdata&eventingPath=%2Fopt%2Fcouchbase%2Fvar%2Flib%2Fcouchbase%2Fdata&sendStats=false&services=kv&analyticsPath=%2Fopt%2Fcouchbase%2Fvar%2Flib%2Fcouchbase%2Fdata&javaHome=&clusterName=home&memoryQuota=$MEMORY_AS_MB&afamily=ipv4&afamilyOnly=false&nodeEncryption=off&username=$USERNAME&password=$PASSWORD&port=SAME") + echo "create_cluster > $RESPONSE" +} + +join_cluster() { + URL="http://$2:$REST_PORT/node/controller/doJoinCluster" + RESPONSE=$(do_post "$URL" "hostname=$1&user=$USERNAME&password=$PASSWORD&newNodeHostname=$2&services=kv") + echo "join_cluster > $RESPONSE" +} + +rebalance() { + URL="http://$1:$REST_PORT/controller/rebalance" + RESPONSE=$(do_post "$URL" "knownNodes=$2&ejectedNodes=") + echo "rebalance > $RESPONSE" +} + +echo "creating nodes" +for (( i=1; i<=TOTAL_NODE; i++ )) +do + NAME="$NAME_PREFIX$i" + docker run -d --pull missing --name $NAME --net couchbase "couchbase:$VERSION" +done + +echo "waiting nodes" +for (( i=1; i<=TOTAL_NODE; i++ )) +do + NAME="$NAME_PREFIX$i" + IP=$(docker inspect "$NAME" -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}') + KNOWN_NODES="${KNOWN_NODES}ns_1%40${IP}" + if [ "$i" != "$TOTAL_NODE" ]; then + KNOWN_NODES="${KNOWN_NODES}%2C" + fi + + echo "$NAME IP=$IP" + + until [[ $(check_db "$IP") = 0 ]]; do + >&2 echo "waiting $NAME to be available" + sleep 5 + done + + echo "$NAME ready" + + if [ "$i" == 1 ]; then + echo "creating cluster" + create_cluster "$IP" + MASTER="$IP" + else + echo "$NAME joining cluster" + node_init "$IP" + join_cluster "$MASTER" "$IP" + fi +done + +echo "rebalancing" + +rebalance "$MASTER" "$KNOWN_NODES" + +until [[ $(get_rebalance_logs "$MASTER") == *"Rebalance completed"* ]]; do + >&2 echo "waiting rebalance to be done" + sleep 5 +done + +echo "rebalance done" + +echo "installing travel-sample" + +install_travel_sample "$MASTER" + +until [[ $(get_buckets "$MASTER") == *"\"itemCount\":63288"* ]]; do + >&2 echo "waiting travel-sample to be installed" + sleep 15 +done + +echo "travel-sample installed" + +URL="http://$MASTER:$REST_PORT" + +echo "---------------------------" +echo "username $USERNAME" +echo "password $PASSWORD" +echo "---------------------------" +echo "ui $URL" +echo "---------------------------" +RAW="" +HOSTS="hosts:" +HOSTS_AS_GO_ARR="[]string{" +for (( i=1; i<=TOTAL_NODE; i++ )) +do + NAME="$NAME_PREFIX$i" + IP=$(docker inspect "$NAME" -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}') + RAW="$RAW$IP:$REST_PORT" + HOSTS="$HOSTS\n - $IP:$REST_PORT" + HOSTS_AS_GO_ARR="${HOSTS_AS_GO_ARR}\"$IP:$REST_PORT\"" + if [ "$i" != "$TOTAL_NODE" ]; then + RAW="$RAW\n" + HOSTS_AS_GO_ARR="${HOSTS_AS_GO_ARR}, " + fi +done +HOSTS_AS_GO_ARR="${HOSTS_AS_GO_ARR}}" +echo "$HOSTS" +echo "---------------------------" +echo "$HOSTS_AS_GO_ARR" +echo "---------------------------" + +if [ "$HOSTS_PATH" != "" ]; then + echo "writing hosts to $HOSTS_PATH" + echo -e "$RAW" >> "$HOSTS_PATH" +fi \ No newline at end of file diff --git a/scripts/delete_cluster.sh b/scripts/delete_cluster.sh new file mode 100644 index 0000000..4963bf5 --- /dev/null +++ b/scripts/delete_cluster.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +echo "deleting nodes" + +NODES=$(docker network inspect couchbase --format '{{range .Containers}}{{printf "%s\n" .Name}}{{end}}') + +echo "$NODES" | while read -r NAME ; do + docker rm -f "$NAME" +done + +echo "deleting network" + +docker network rm couchbase \ No newline at end of file diff --git a/test/couchbase/Dockerfile b/test/couchbase/Dockerfile deleted file mode 100644 index d2e6670..0000000 --- a/test/couchbase/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM couchbase:7.2.3 - -ADD configure.sh /configure.sh -RUN chmod +x /configure.sh -RUN echo " image starting" - -EXPOSE 8091 8092 8093 8094 8095 8096 11207 11210 11211 - -CMD ["/configure.sh"] diff --git a/test/couchbase/configure.sh b/test/couchbase/configure.sh deleted file mode 100644 index 7f8a842..0000000 --- a/test/couchbase/configure.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -# Enables job control -set -m - -# Enables error propagation -set -e - -# Run the server and send it to the background -/entrypoint.sh couchbase-server & - -# Check if couchbase server is up -check_db() { - curl --silent http://127.0.0.1:8091/pools > /dev/null - echo $? -} - -# Variable used in echo -i=1 -# Echo with -log() { - echo "[$i] [$(date +"%T")] $@" - i=`expr $i + 1` -} - -# Wait until it's ready -until [[ $(check_db) = 0 ]]; do - >&2 log "Waiting for Couchbase Server to be available ..." - sleep 1 -done - -couchbase-cli cluster-init -c localhost --cluster-name Cluster --cluster-username user \ - --cluster-password 123456 --services data --cluster-ramsize 1024 - -couchbase-cli bucket-create -c couchbase --username user --password 123456 --bucket dcp-test --bucket-type couchbase --bucket-ramsize 1024 - -cbimport json -c couchbase://127.0.0.1 -u user -p 123456 --bucket-quota 1024 -b dcp-test -d file://opt/couchbase/samples/travel-sample.zip -f sample - -echo "couchbase-dev started" - -fg 1 diff --git a/test/integration/Dockerfile b/test/integration/Dockerfile deleted file mode 100644 index cf4ed80..0000000 --- a/test/integration/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM golang:1.20-alpine as builder - -WORKDIR /project - -RUN apk add build-base - -COPY .. . - -RUN go mod tidy -RUN go test -v ./... \ No newline at end of file diff --git a/test/integration/config.yml b/test/integration/config.yml index 55dedd0..889e311 100644 --- a/test/integration/config.yml +++ b/test/integration/config.yml @@ -1,12 +1,8 @@ hosts: - - localhost:8091 + - 13.37.11.2:8091 username: user password: 123456 -bucketName: dcp-test -rollbackMitigation: - disabled: true -checkpoint: - type: manual +bucketName: travel-sample dcp: group: name: groupName @@ -16,4 +12,4 @@ metadata: type: couchbase readOnly: true config: - bucket: dcp-test \ No newline at end of file + bucket: travel-sample \ No newline at end of file diff --git a/test/integration/integration_test.go b/test/integration/integration_test.go index c9d3a1f..88443e4 100644 --- a/test/integration/integration_test.go +++ b/test/integration/integration_test.go @@ -30,7 +30,6 @@ func listener(ctx *models.ListenerContext) { } func TestCouchbase(t *testing.T) { - time.Sleep(10 * time.Second) newDcp, err := dcp.NewDcp("config.yml", listener) if err != nil { log.Fatalf("couldn't create dcp err: %v", err) @@ -44,11 +43,11 @@ func TestCouchbase(t *testing.T) { }() go func() { - ctx, _ := context.WithTimeout(context.Background(), 20*time.Second) + ctx, _ := context.WithTimeout(context.Background(), 30*time.Second) for { select { case <-ctx.Done(): - t.Fatalf("could not process events in 20 seconds") + t.Fatalf("could not process events in 30 seconds") default: if mutationCount == 31591 { newDcp.Close() @@ -59,7 +58,5 @@ func TestCouchbase(t *testing.T) { } }() - time.Sleep(time.Second) - wg.Wait() }