diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index f56cadb..960794f 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -22,11 +22,15 @@ jobs: go-version: '1.20' - uses: actions/checkout@v3 - - name: Start containers - run: make compose + - name: Start cluster + run: make create-cluster - name: Integration Test run: go test -v test/integration/integration_test.go env: INPUT_PUBLISH: false GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - uses: actions/checkout@v3 + - name: Delete cluster + run: make delete-cluster diff --git a/Makefile b/Makefile index 92a68cc..2155908 100644 --- a/Makefile +++ b/Makefile @@ -19,8 +19,11 @@ lint: test: go test ./... -bench . -compose: - docker compose up --wait --build --force-recreate --remove-orphans +create-cluster: + bash scripts/create_cluster.sh + +delete-cluster: + bash scripts/delete_cluster.sh docker-build: docker build --progress=plain -t docker.io/trendyoltech/dcp . \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 6ecbec3..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,13 +0,0 @@ -version: "3.8" -services: - couchbase: - build: - context: "test/couchbase" - ports: - - "8091:8091" - - "11210:11210" - healthcheck: - test: [ "CMD", "curl", "-f", "http://user:123456@localhost:8091/pools/default/buckets/dcp-test" ] - interval: 2s - timeout: 3s - retries: 60 \ No newline at end of file diff --git a/scripts/cluster.sh b/scripts/cluster.sh new file mode 100644 index 0000000..3cbfc98 --- /dev/null +++ b/scripts/cluster.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +sh create_cluster.sh + +read -r -p "press enter to delete cluster _ " + +sh delete_cluster.sh \ No newline at end of file diff --git a/scripts/create_cluster.sh b/scripts/create_cluster.sh new file mode 100644 index 0000000..7244598 --- /dev/null +++ b/scripts/create_cluster.sh @@ -0,0 +1,148 @@ +#!/bin/bash + +VERSION=7.2.4 +TOTAL_NODE=12 +REST_PORT=8091 +MEMORY_AS_MB=512 +USERNAME=user +PASSWORD=123456 +MASTER="" +KNOWN_NODES="" +NAME_PREFIX="couchbase_node_" +SUBNET="13.37.11.0/24" + +echo "creating network" +docker network create couchbase --subnet=$SUBNET + +check_db() { + URL="http://$1:$REST_PORT/pools" + curl --silent --max-time 5 "$URL" > /dev/null + echo $? +} + +install_travel_sample() { + URL="http://$1:$REST_PORT/sampleBuckets/install" + RESPONSE=$(curl --silent -o /dev/null -w "%{http_code}" "$URL" -X 'POST' -u "$USERNAME:$PASSWORD" --data '["travel-sample"]') + echo "install_travel_sample > $RESPONSE" +} + +get_buckets() { + URL="http://$1:$REST_PORT/pools/default/buckets" + RESPONSE=$(curl --silent --max-time 5 "$URL" -X 'GET' -u "$USERNAME:$PASSWORD") + echo "$RESPONSE" +} + +get_rebalance_logs() { + URL="http://$1:$REST_PORT/logs/rebalanceReport" + RESPONSE=$(curl --silent --max-time 5 "$URL" -X 'GET' -u "$USERNAME:$PASSWORD") + echo "$RESPONSE" +} + +node_init() { + URL="http://$1:$REST_PORT/nodeInit" + RESPONSE=$(curl --silent -o /dev/null -w "%{http_code}" "$URL" -X 'POST' --data "hostname=$1&dataPath=%2Fopt%2Fcouchbase%2Fvar%2Flib%2Fcouchbase%2Fdata&indexPath=%2Fopt%2Fcouchbase%2Fvar%2Flib%2Fcouchbase%2Fdata&eventingPath=%2Fopt%2Fcouchbase%2Fvar%2Flib%2Fcouchbase%2Fdata&javaHome=&analyticsPath=%2Fopt%2Fcouchbase%2Fvar%2Flib%2Fcouchbase%2Fdata") + echo "node_init > $RESPONSE" +} + +create_cluster() { + URL="http://$1:$REST_PORT/clusterInit" + RESPONSE=$(curl --silent -o /dev/null -w "%{http_code}" "$URL" -X 'POST' --data "hostname=$1&dataPath=%2Fopt%2Fcouchbase%2Fvar%2Flib%2Fcouchbase%2Fdata&indexPath=%2Fopt%2Fcouchbase%2Fvar%2Flib%2Fcouchbase%2Fdata&eventingPath=%2Fopt%2Fcouchbase%2Fvar%2Flib%2Fcouchbase%2Fdata&sendStats=false&services=kv&analyticsPath=%2Fopt%2Fcouchbase%2Fvar%2Flib%2Fcouchbase%2Fdata&javaHome=&clusterName=home&memoryQuota=$MEMORY_AS_MB&afamily=ipv4&afamilyOnly=false&nodeEncryption=off&username=$USERNAME&password=$PASSWORD&port=SAME") + echo "create_cluster > $RESPONSE" +} + +join_cluster() { + URL="http://$2:$REST_PORT/node/controller/doJoinCluster" + RESPONSE=$(curl --silent -o /dev/null -w "%{http_code}" "$URL" -X 'POST' --data "hostname=$1&user=$USERNAME&password=$PASSWORD&newNodeHostname=$2&services=kv") + echo "join_cluster > $RESPONSE" +} + +rebalance() { + URL="http://$1:$REST_PORT/controller/rebalance" + RESPONSE=$(curl --silent -o /dev/null -w "%{http_code}" "$URL" -X 'POST' -u "$USERNAME:$PASSWORD" --data "knownNodes=$2&ejectedNodes=") + echo "rebalance > $RESPONSE" +} + +echo "creating nodes" +for (( i=1; i<=TOTAL_NODE; i++ )) +do + NAME="$NAME_PREFIX$i" + docker run -d --name $NAME --net couchbase "couchbase:$VERSION" +done + +echo "waiting nodes" +for (( i=1; i<=TOTAL_NODE; i++ )) +do + NAME="$NAME_PREFIX$i" + IP=$(docker inspect $NAME -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}') + KNOWN_NODES="${KNOWN_NODES}ns_1%40${IP}" + if [ "$i" != $TOTAL_NODE ]; then + KNOWN_NODES="${KNOWN_NODES}%2C" + fi + + echo "$NAME has IP=$IP" + + until [[ $(check_db "$IP") = 0 ]]; do + >&2 echo "waiting $NAME to be available" + sleep 5 + done + + echo "$NAME ready" + + if [ "$i" == 1 ]; then + echo "creating cluster" + create_cluster "$IP" + MASTER="$IP" + else + echo "$NAME joining cluster" + node_init "$IP" + join_cluster "$MASTER" "$IP" + fi +done + +echo "rebalancing" + +rebalance "$MASTER" "$KNOWN_NODES" + +until [[ $(get_rebalance_logs "$MASTER") == *"Rebalance completed"* ]]; do + >&2 echo "waiting rebalance to be done" + sleep 5 +done + +echo "rebalance done" + +echo "installing travel-sample" + +install_travel_sample "$MASTER" + +until [[ $(get_buckets "$MASTER") == *"\"itemCount\":63288"* ]]; do + >&2 echo "waiting travel-sample to be installed" + sleep 10 +done + +echo "travel-sample installed" + +URL="http://$MASTER:$REST_PORT" + +echo "---------------------------" +echo "username $USERNAME" +echo "password $PASSWORD" +echo "---------------------------" +echo "ui $URL" +echo "---------------------------" +HOSTS="hosts:" +HOSTS_AS_GO_ARR="[]string{" +for (( i=1; i<=TOTAL_NODE; i++ )) +do + NAME="$NAME_PREFIX$i" + IP=$(docker inspect $NAME -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}') + HOSTS="$HOSTS\n - $IP:$REST_PORT" + HOSTS_AS_GO_ARR="${HOSTS_AS_GO_ARR}\"$IP:$REST_PORT\"" + if [ "$i" != $TOTAL_NODE ]; then + HOSTS_AS_GO_ARR="${HOSTS_AS_GO_ARR}, " + fi +done +HOSTS_AS_GO_ARR="${HOSTS_AS_GO_ARR}}" +echo "$HOSTS" +echo "---------------------------" +echo "$HOSTS_AS_GO_ARR" +echo "---------------------------" \ No newline at end of file diff --git a/scripts/delete_cluster.sh b/scripts/delete_cluster.sh new file mode 100644 index 0000000..eb6b4fe --- /dev/null +++ b/scripts/delete_cluster.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +TOTAL_NODE=12 +NAME_PREFIX="couchbase_node_" + +echo "deleting nodes" + +for (( i=1; i<=TOTAL_NODE; i++ )) +do + NAME="$NAME_PREFIX$i" + docker rm -f $NAME +done + +echo "deleting network" + +docker network rm couchbase \ No newline at end of file diff --git a/test/couchbase/Dockerfile b/test/couchbase/Dockerfile deleted file mode 100644 index d2e6670..0000000 --- a/test/couchbase/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM couchbase:7.2.3 - -ADD configure.sh /configure.sh -RUN chmod +x /configure.sh -RUN echo " image starting" - -EXPOSE 8091 8092 8093 8094 8095 8096 11207 11210 11211 - -CMD ["/configure.sh"] diff --git a/test/couchbase/configure.sh b/test/couchbase/configure.sh deleted file mode 100644 index 7f8a842..0000000 --- a/test/couchbase/configure.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -# Enables job control -set -m - -# Enables error propagation -set -e - -# Run the server and send it to the background -/entrypoint.sh couchbase-server & - -# Check if couchbase server is up -check_db() { - curl --silent http://127.0.0.1:8091/pools > /dev/null - echo $? -} - -# Variable used in echo -i=1 -# Echo with -log() { - echo "[$i] [$(date +"%T")] $@" - i=`expr $i + 1` -} - -# Wait until it's ready -until [[ $(check_db) = 0 ]]; do - >&2 log "Waiting for Couchbase Server to be available ..." - sleep 1 -done - -couchbase-cli cluster-init -c localhost --cluster-name Cluster --cluster-username user \ - --cluster-password 123456 --services data --cluster-ramsize 1024 - -couchbase-cli bucket-create -c couchbase --username user --password 123456 --bucket dcp-test --bucket-type couchbase --bucket-ramsize 1024 - -cbimport json -c couchbase://127.0.0.1 -u user -p 123456 --bucket-quota 1024 -b dcp-test -d file://opt/couchbase/samples/travel-sample.zip -f sample - -echo "couchbase-dev started" - -fg 1 diff --git a/test/integration/Dockerfile b/test/integration/Dockerfile deleted file mode 100644 index cf4ed80..0000000 --- a/test/integration/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM golang:1.20-alpine as builder - -WORKDIR /project - -RUN apk add build-base - -COPY .. . - -RUN go mod tidy -RUN go test -v ./... \ No newline at end of file diff --git a/test/integration/config.yml b/test/integration/config.yml index 55dedd0..b59d4db 100644 --- a/test/integration/config.yml +++ b/test/integration/config.yml @@ -1,12 +1,19 @@ hosts: - - localhost:8091 + - 13.37.11.2:8091 + - 13.37.11.3:8091 + - 13.37.11.4:8091 + - 13.37.11.5:8091 + - 13.37.11.6:8091 + - 13.37.11.7:8091 + - 13.37.11.8:8091 + - 13.37.11.9:8091 + - 13.37.11.10:8091 + - 13.37.11.11:8091 + - 13.37.11.12:8091 + - 13.37.11.13:8091 username: user password: 123456 -bucketName: dcp-test -rollbackMitigation: - disabled: true -checkpoint: - type: manual +bucketName: travel-sample dcp: group: name: groupName @@ -16,4 +23,4 @@ metadata: type: couchbase readOnly: true config: - bucket: dcp-test \ No newline at end of file + bucket: travel-sample \ No newline at end of file