diff --git a/deployments/analytics-datadog/docker-compose.yml b/deployments/analytics-datadog/docker-compose.yml index f3dd3924..d65ec433 100644 --- a/deployments/analytics-datadog/docker-compose.yml +++ b/deployments/analytics-datadog/docker-compose.yml @@ -16,7 +16,7 @@ services: - .env tyk-pump-datadog: - image: tykio/tyk-pump-docker-pub:${PUMP_VERSION:-v1.10.0} + image: tykio/tyk-pump-docker-pub:${PUMP_VERSION:-v1.11.0} networks: - tyk volumes: diff --git a/deployments/analytics-kibana/docker-compose.yml b/deployments/analytics-kibana/docker-compose.yml index 297ab0f7..a1dba860 100644 --- a/deployments/analytics-kibana/docker-compose.yml +++ b/deployments/analytics-kibana/docker-compose.yml @@ -17,7 +17,7 @@ services: volumes: - elasticsearch-data:/usr/share/elasticsearch/data tyk-pump-elasticsearch: - image: tykio/tyk-pump-docker-pub:${PUMP_VERSION:-v1.10.0} + image: tykio/tyk-pump-docker-pub:${PUMP_VERSION:-v1.11.0} networks: - tyk volumes: diff --git a/deployments/analytics-splunk/docker-compose.yml b/deployments/analytics-splunk/docker-compose.yml index 7350f6de..cd6e43a6 100644 --- a/deployments/analytics-splunk/docker-compose.yml +++ b/deployments/analytics-splunk/docker-compose.yml @@ -15,7 +15,7 @@ services: - splunk-data:/opt/splunk/var - splunk-data:/opt/splunk/etc tyk-splunk-pump: - image: tykio/tyk-pump-docker-pub:${PUMP_VERSION:-v1.10.0} + image: tykio/tyk-pump-docker-pub:${PUMP_VERSION:-v1.11.0} networks: - tyk volumes: diff --git a/deployments/cicd/Dockerfile b/deployments/cicd/Dockerfile new file mode 100644 index 00000000..3a0f4955 --- /dev/null +++ b/deployments/cicd/Dockerfile @@ -0,0 +1,11 @@ +FROM jenkins/jenkins:2.319.2 + +USER root + +RUN apt-get update && \ + curl -O https://packagecloud.io/install/repositories/tyk/tyk-sync/script.deb.sh && \ + chmod +x script.deb.sh && \ + ./script.deb.sh && \ + apt-get install -y tyk-sync=1.5.1 + +USER jenkins \ No newline at end of file diff --git a/deployments/cicd/data/jenkins/Jenkinsfile b/deployments/cicd/data/jenkins/Jenkinsfile index a5cfba9f..fcd50eb5 100644 --- a/deployments/cicd/data/jenkins/Jenkinsfile +++ b/deployments/cicd/data/jenkins/Jenkinsfile @@ -14,7 +14,7 @@ pipeline { script { if (fileExists('.tyk.json')) { echo "Deploying from ${env.BRANCH_NAME}" - sh "./tyk-sync sync -d http://tyk2-dashboard:3000 -s ${env.tyk2_dashboard_credentials} -p ." + sh "/opt/tyk-sync/tyk-sync sync -d http://tyk2-dashboard:3000 -s ${env.tyk2_dashboard_credentials} -p ." } else { echo 'No files to deploy' } diff --git a/deployments/cicd/docker-compose.yml b/deployments/cicd/docker-compose.yml index 0d296a08..047ec8ce 100644 --- a/deployments/cicd/docker-compose.yml +++ b/deployments/cicd/docker-compose.yml @@ -1,7 +1,8 @@ --- services: jenkins: - image: jenkins/jenkins:2.319.2-lts-alpine + build: ./deployments/cicd + image: tyk-demo-jenkins-tyk-sync:v1.5.1 ports: - 8070:8080 networks: diff --git a/deployments/cicd/scripts/dump-tyk.sh b/deployments/cicd/scripts/dump-tyk.sh index f61da76c..4a381dc2 100755 --- a/deployments/cicd/scripts/dump-tyk.sh +++ b/deployments/cicd/scripts/dump-tyk.sh @@ -7,5 +7,5 @@ gitea_tyk_data_repo_path=$(cat .context-data/gitea-tyk-data-repo-path) docker run --rm \ --network tyk-demo_tyk \ -v $gitea_tyk_data_repo_path:/opt/tyk-sync/data \ - tykio/tyk-sync:v1.4.3 \ + tykio/tyk-sync:v1.5.1 \ dump -d http://tyk-dashboard:3000 -s $dashboard_user_api_credentials -t data diff --git a/deployments/load-balancer-nginx/bootstrap.sh b/deployments/load-balancer-nginx/bootstrap.sh index 943ae946..607f5dd5 100755 --- a/deployments/load-balancer-nginx/bootstrap.sh +++ b/deployments/load-balancer-nginx/bootstrap.sh @@ -6,13 +6,13 @@ deployment="Load Balancer" log_start_deployment bootstrap_progress -# log_message "Restart Gateways to load latest certificates" -# docker restart tyk-demo-tyk-gateway-3-1 tyk-demo-tyk-gateway-4-1 1>/dev/null 2>>logs/bootstrap.log -# if [ "$?" != 0 ]; then -# echo "Error when restart Gateways to load latest certificates" -# exit 1 -# fi -# log_ok +log_message "Restart Gateways to load latest certificates" +eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-gateway-3 tyk-gateway-4 1>/dev/null 2>>logs/bootstrap.log +if [ "$?" != 0 ]; then + echo "Error when restart Gateways to load latest certificates" + exit 1 +fi +log_ok log_message "Restart nginx to reset load balancer" docker restart tyk-demo-nginx-1 1>/dev/null 2>>logs/bootstrap.log diff --git a/deployments/load-balancer-nginx/docker-compose.yml b/deployments/load-balancer-nginx/docker-compose.yml index b32e02cd..89daa2cb 100755 --- a/deployments/load-balancer-nginx/docker-compose.yml +++ b/deployments/load-balancer-nginx/docker-compose.yml @@ -1,7 +1,7 @@ --- services: tyk-gateway-3: - image: tykio/tyk-gateway:${GATEWAY_VERSION:-v5.4.0} + image: tykio/tyk-gateway:${GATEWAY_VERSION:-v5.5.0} networks: - tyk environment: @@ -19,7 +19,7 @@ services: - .env volumes: - ./deployments/tyk/volumes/tyk-gateway/tyk.conf:/opt/tyk-gateway/tyk.conf - - ./deployments/tyk/volumes/tyk-gateway/certs:/opt/tyk-gateway/certs + - tyk-gateway-certs:/opt/tyk-gateway/certs - ./deployments/tyk/volumes/tyk-gateway/middleware:/opt/tyk-gateway/middleware - ./deployments/tyk/volumes/tyk-gateway/plugins:/opt/tyk-gateway/plugins - ./deployments/tyk/volumes/tyk-gateway/templates/error_401.json:/opt/tyk-gateway/templates/error_401.json @@ -28,7 +28,7 @@ services: - tyk-redis - tyk-dashboard tyk-gateway-4: - image: tykio/tyk-gateway:${GATEWAY_VERSION:-v5.4.0} + image: tykio/tyk-gateway:${GATEWAY_VERSION:-v5.5.0} networks: - tyk environment: @@ -46,7 +46,7 @@ services: - .env volumes: - ./deployments/tyk/volumes/tyk-gateway/tyk.conf:/opt/tyk-gateway/tyk.conf - - ./deployments/tyk/volumes/tyk-gateway/certs:/opt/tyk-gateway/certs + - tyk-gateway-certs:/opt/tyk-gateway/certs - ./deployments/tyk/volumes/tyk-gateway/middleware:/opt/tyk-gateway/middleware - ./deployments/tyk/volumes/tyk-gateway/plugins:/opt/tyk-gateway/plugins - ./deployments/tyk/volumes/tyk-gateway/templates/error_401.json:/opt/tyk-gateway/templates/error_401.json diff --git a/deployments/mdcb/docker-compose.yml b/deployments/mdcb/docker-compose.yml index 5d298073..e6d92c5b 100644 --- a/deployments/mdcb/docker-compose.yml +++ b/deployments/mdcb/docker-compose.yml @@ -1,7 +1,7 @@ --- services: tyk-mdcb: - image: tykio/tyk-mdcb-docker:${MDCB_VERSION:-v2.6.0} + image: tykio/tyk-mdcb-docker:${MDCB_VERSION:-v2.7.0} ports: - 9091:9091 networks: @@ -15,7 +15,7 @@ services: - tyk-redis - tyk-mongo tyk-worker-gateway: - image: tykio/tyk-gateway:${GATEWAY_WORKER_VERSION:-v5.4.0} + image: tykio/tyk-gateway:${GATEWAY_WORKER_VERSION:-v5.5.0} ports: - 8090:8080 networks: @@ -28,7 +28,7 @@ services: - TYK_GW_OPENTELEMETRY_ENDPOINT=${OPENTELEMETRY_ENDPOINT:-false} volumes: - ./deployments/mdcb/volumes/tyk-worker-gateway/tyk.conf:/opt/tyk-gateway/tyk.conf - - ./deployments/tyk/volumes/tyk-gateway/certs:/opt/tyk-gateway/certs + - tyk-gateway-certs:/opt/tyk-gateway/certs - ./deployments/tyk/volumes/tyk-gateway/middleware:/opt/tyk-gateway/middleware - ./deployments/tyk/volumes/tyk-gateway/plugins:/opt/tyk-gateway/plugins depends_on: diff --git a/deployments/portal/docker-compose.yml b/deployments/portal/docker-compose.yml index cc1b77e6..59e8f735 100644 --- a/deployments/portal/docker-compose.yml +++ b/deployments/portal/docker-compose.yml @@ -1,7 +1,7 @@ --- services: tyk-portal: - image: tykio/portal:${PORTAL_VERSION:-v1.9.0} + image: tykio/portal:${PORTAL_VERSION:-v1.10.0} networks: - tyk env_file: diff --git a/deployments/slo-prometheus-grafana/docker-compose.yml b/deployments/slo-prometheus-grafana/docker-compose.yml index cbf5ba06..29949aed 100755 --- a/deployments/slo-prometheus-grafana/docker-compose.yml +++ b/deployments/slo-prometheus-grafana/docker-compose.yml @@ -1,7 +1,7 @@ --- services: tyk-slo-pump: - image: tykio/tyk-pump-docker-pub:${PUMP_VERSION:-v1.10.0} + image: tykio/tyk-pump-docker-pub:${PUMP_VERSION:-v1.11.0} ports: - 8091:8083 - 8092:8092 diff --git a/deployments/sso/docker-compose.yml b/deployments/sso/docker-compose.yml index 2fd936c7..fcd674fe 100644 --- a/deployments/sso/docker-compose.yml +++ b/deployments/sso/docker-compose.yml @@ -1,14 +1,14 @@ --- services: tyk-dashboard-sso: - image: tykio/tyk-dashboard:${DASHBOARD_SSO_VERSION:-v5.4.0} + image: tykio/tyk-dashboard:${DASHBOARD_SSO_VERSION:-v5.5.0} ports: - 3001:3000 networks: - tyk volumes: - ./deployments/tyk/volumes/tyk-dashboard/tyk_analytics.conf:/opt/tyk-dashboard/tyk_analytics.conf - - ./deployments/tyk/volumes/tyk-dashboard/private-key.pem:/opt/tyk-dashboard/private-key.pem + - tyk-dashboard-certs:/opt/tyk-dashboard/certs environment: - TYK_DB_LICENSEKEY=${DASHBOARD_LICENCE:?Please set DASHBOARD_LICENCE in .env} - TYK_DB_SSOCUSTOMLOGINURL=http://localhost:3010/auth/tyk-dashboard/openid-connect diff --git a/deployments/tyk/bootstrap.sh b/deployments/tyk/bootstrap.sh index 5b48aaaa..1a51afb5 100755 --- a/deployments/tyk/bootstrap.sh +++ b/deployments/tyk/bootstrap.sh @@ -53,9 +53,6 @@ bootstrap_progress # Certificates -log_message "Wait for services to be ready before beginning to bootstrap" -wait_for_liveness - log_message "Checking for existing OpenSSL container" OPENSSL_CONTAINER_NAME="tyk-demo-openssl" if [ "$(docker ps -a --format '{{.Names}}' | grep -w "$OPENSSL_CONTAINER_NAME" | wc -l)" -gt 0 ]; then @@ -67,16 +64,19 @@ fi bootstrap_progress log_message "Creating temporary container $OPENSSL_CONTAINER_NAME for OpenSSL usage" -docker run -d --name $OPENSSL_CONTAINER_NAME alpine:3.20.1 tail -f /dev/null > /dev/null 2>&1 +docker run -d --name $OPENSSL_CONTAINER_NAME \ + -v tyk-demo_tyk-gateway-certs:/tyk-gateway-certs \ + -v tyk-demo_tyk-dashboard-certs:/tyk-dashboard-certs \ + alpine:3.20.1 tail -f /dev/null >/dev/null 2>&1 log_ok bootstrap_progress log_message "Install OpenSSL into container $OPENSSL_CONTAINER_NAME" -docker exec -d $OPENSSL_CONTAINER_NAME apk add --no-cache openssl +docker exec $OPENSSL_CONTAINER_NAME apk add --no-cache openssl >/dev/null 2>>logs/bootstrap.log # Wait for the installation to complete while true; do # Check if OpenSSL is installed by trying to get its version - if docker exec $OPENSSL_CONTAINER_NAME openssl version > /dev/null 2>&1; then + if docker exec $OPENSSL_CONTAINER_NAME openssl version >/dev/null 2>&1; then log_message " OpenSSL has been successfully installed" break else @@ -87,107 +87,53 @@ done log_message "OpenSSL version used for generating certs: $(docker exec $OPENSSL_CONTAINER_NAME openssl version)" -log_message "Removing any pre-existing certs" -rm deployments/tyk/volumes/tyk-dashboard/certs/*.pem 1> /dev/null 2>> logs/bootstrap.log -rm deployments/tyk/volumes/tyk-gateway/certs/*.pem 1> /dev/null 2>> logs/bootstrap.log -log_ok -bootstrap_progress - log_message "Generating self-signed certificate for TLS connections to tyk-gateway-2.localhost" -docker exec -d $OPENSSL_CONTAINER_NAME sh -c "openssl req -x509 -newkey rsa:4096 -subj \"/CN=tyk-gateway-2.localhost\" -keyout /tmp/tls-private-key.pem -out /tmp/tls-certificate.pem -days 365 -nodes" >>logs/bootstrap.log +docker exec $OPENSSL_CONTAINER_NAME sh -c "openssl req -x509 -newkey rsa:4096 -subj \"/CN=tyk-gateway-2.localhost\" -keyout /tyk-gateway-certs/tls-private-key.pem -out /tyk-gateway-certs/tls-certificate.pem -days 365 -nodes" >/dev/null 2>&1 if [ "$?" -ne "0" ]; then echo "ERROR: Could not generate self-signed certificate" exit 1 fi -while true; do - docker exec $OPENSSL_CONTAINER_NAME sh -c "[ -s /tmp/tls-certificate.pem ]" - if [ $? -eq 0 ]; then - log_ok - bootstrap_progress - break; - else - log_message " Waiting for /tmp/tls-certificate.pem to be ready" - bootstrap_progress - sleep 2 - fi -done +log_ok +bootstrap_progress +wait_for_file "/tyk-gateway-certs/tls-certificate.pem" "$OPENSSL_CONTAINER_NAME" +wait_for_file "/tyk-gateway-certs/tls-private-key.pem" "$OPENSSL_CONTAINER_NAME" log_message "Generating private key for secure messaging and signing" -docker exec -d $OPENSSL_CONTAINER_NAME sh -c "openssl genrsa -out /tmp/private-key.pem 2048" >>logs/bootstrap.log +docker exec $OPENSSL_CONTAINER_NAME sh -c "openssl genrsa -out /tyk-dashboard-certs/private-key.pem 2048" >/dev/null 2>>logs/bootstrap.log if [ "$?" -ne "0" ]; then echo "ERROR: Could not generate private key" exit 1 fi -while true; do - docker exec $OPENSSL_CONTAINER_NAME sh -c "[ -s /tmp/private-key.pem ]" - if [ $? -eq 0 ]; then - log_ok - bootstrap_progress - break; - else - log_message " Waiting for /tmp/private-key.pem to be ready" - bootstrap_progress - sleep 2 - fi -done +log_ok +bootstrap_progress +wait_for_file "/tyk-dashboard-certs/private-key.pem" "$OPENSSL_CONTAINER_NAME" log_message "Generating public key for secure messaging and signing" -docker exec -d $OPENSSL_CONTAINER_NAME sh -c "openssl rsa -in /tmp/private-key.pem -pubout -out /tmp/public-key.pem" >>logs/bootstrap.log +docker exec $OPENSSL_CONTAINER_NAME sh -c "openssl rsa -in /tyk-dashboard-certs/private-key.pem -pubout -out /tyk-gateway-certs/public-key.pem" >/dev/null 2>>logs/bootstrap.log if [ "$?" -ne "0" ]; then echo "ERROR: Could not generate public key" exit 1 fi -while true; do - docker exec $OPENSSL_CONTAINER_NAME sh -c "[ -s /tmp/public-key.pem ]" - if [ $? -eq 0 ]; then - log_ok - bootstrap_progress - break; - else - log_message " Waiting for /tmp/public-key.pem to be ready" - bootstrap_progress - sleep 2 - fi -done - -log_message "Copying private-key.pem to dashboard volume mount" -docker cp $OPENSSL_CONTAINER_NAME:/tmp/private-key.pem deployments/tyk/volumes/tyk-dashboard/certs >>logs/bootstrap.log -if [ "$?" != "0" ]; then - echo "ERROR: Could not copy private-key.pem to dashboard volume mount" - exit 1 -fi log_ok bootstrap_progress +wait_for_file "/tyk-gateway-certs/public-key.pem" "$OPENSSL_CONTAINER_NAME" -log_message "Copying public-key.pem to gateway volume mount" -docker cp $OPENSSL_CONTAINER_NAME:/tmp/public-key.pem deployments/tyk/volumes/tyk-gateway/certs >>logs/bootstrap.log +log_message "Setting read permissions on certificate volumes" +docker exec $OPENSSL_CONTAINER_NAME chmod -R a+r /tyk-gateway-certs >/dev/null 2>>logs/bootstrap.log if [ "$?" != "0" ]; then - echo "ERROR: Could not copy public-key.pem to gateway volume mount" + echo "ERROR: Could not set read permissions on /tyk-gateway-certs volume" exit 1 fi -log_ok -bootstrap_progress - -log_message "Copying tls-certificate.pem to gateway volume mount" -docker cp $OPENSSL_CONTAINER_NAME:/tmp/tls-certificate.pem deployments/tyk/volumes/tyk-gateway/certs >>logs/bootstrap.log +docker exec $OPENSSL_CONTAINER_NAME chmod -R a+r /tyk-dashboard-certs >/dev/null 2>>logs/bootstrap.log if [ "$?" != "0" ]; then - echo "ERROR: Could not copy tls-certificate.pem to gateway volume mount" - exit 1 -fi -log_ok -bootstrap_progress - -log_message "Copying tls-private-key.pem to gateway volume mount" -docker cp $OPENSSL_CONTAINER_NAME:/tmp/tls-private-key.pem deployments/tyk/volumes/tyk-gateway/certs >>logs/bootstrap.log -if [ "$?" != "0" ]; then - echo "ERROR: Could not copy tls-private-key.pem to gateway volume mount" + echo "ERROR: Could not set read permissions on /tyk-dashboard-certs volume" exit 1 fi log_ok bootstrap_progress log_message "Removing temporary OpenSSL container $OPENSSL_CONTAINER_NAME" -docker rm -f $OPENSSL_CONTAINER_NAME +docker rm -f $OPENSSL_CONTAINER_NAME >/dev/null 2>>logs/bootstrap.log if [ "$?" != "0" ]; then echo "ERROR: Could not remove temporary OpenSSL container $OPENSSL_CONTAINER_NAME" exit 1 @@ -195,17 +141,10 @@ fi log_ok bootstrap_progress -log_message "Recreating containers to ensure new certificates are loaded (tyk-gateway, tyk-gateway-2, tyk-dashboard)" -eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-gateway tyk-gateway-2 tyk-dashboard -# if there are gateways from other deployments connecting to this deployment -# (such as MDCB), then they must be recreated to. The MDCB deployment already -# handles recreation. -if [ "$?" != "0" ]; then - echo "ERROR: Could not recreate containers" - exit 1 -fi +log_message "Recreating containers to load new certificates" +eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-dashboard +eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-gateway tyk-gateway-2 log_ok -bootstrap_progress log_message "Wait for services to be available after restart" wait_for_liveness diff --git a/deployments/tyk/docker-compose.yml b/deployments/tyk/docker-compose.yml index 5cfbe75b..12fe5b8c 100755 --- a/deployments/tyk/docker-compose.yml +++ b/deployments/tyk/docker-compose.yml @@ -1,14 +1,14 @@ --- services: tyk-dashboard: - image: tykio/tyk-dashboard:${DASHBOARD_VERSION:-v5.4.0} + image: tykio/tyk-dashboard:${DASHBOARD_VERSION:-v5.5.0} ports: - 3000:3000 networks: - tyk volumes: - ./deployments/tyk/volumes/tyk-dashboard/tyk_analytics.conf:/opt/tyk-dashboard/tyk_analytics.conf - - ./deployments/tyk/volumes/tyk-dashboard/certs:/opt/tyk-dashboard/certs + - tyk-dashboard-certs:/opt/tyk-dashboard/certs - ./deployments/tyk/volumes/tyk-dashboard/catalogue.html:/opt/tyk-dashboard/portal/templates/catalogue.html - ./deployments/tyk/volumes/tyk-dashboard/navigation.html:/opt/tyk-dashboard/portal/templates/navigation.html - ./deployments/tyk/volumes/tyk-dashboard/audit:/opt/tyk-dashboard/audit @@ -22,7 +22,7 @@ services: - tyk-redis - tyk-mongo tyk-gateway: - image: tykio/tyk-gateway:${GATEWAY_VERSION:-v5.4.0} + image: tykio/tyk-gateway:${GATEWAY_VERSION:-v5.5.0} ports: - 8080:8080 - 8086:8086 @@ -49,7 +49,7 @@ services: - .env volumes: - ./deployments/tyk/volumes/tyk-gateway/tyk.conf:/opt/tyk-gateway/tyk.conf - - ./deployments/tyk/volumes/tyk-gateway/certs:/opt/tyk-gateway/certs + - tyk-gateway-certs:/opt/tyk-gateway/certs - ./deployments/tyk/volumes/tyk-gateway/middleware:/opt/tyk-gateway/middleware - ./deployments/tyk/volumes/tyk-gateway/plugins:/opt/tyk-gateway/plugins - ./deployments/tyk/volumes/tyk-gateway/templates/error_401.json:/opt/tyk-gateway/templates/error_401.json @@ -58,7 +58,7 @@ services: - tyk-redis - tyk-dashboard tyk-gateway-2: - image: tykio/tyk-gateway:${GATEWAY2_VERSION:-v5.4.0} + image: tykio/tyk-gateway:${GATEWAY2_VERSION:-v5.5.0} ports: - 8081:8080 networks: @@ -71,14 +71,14 @@ services: - .env volumes: - ./deployments/tyk/volumes/tyk-gateway/tyk-2.conf:/opt/tyk-gateway/tyk.conf - - ./deployments/tyk/volumes/tyk-gateway/certs:/opt/tyk-gateway/certs + - tyk-gateway-certs:/opt/tyk-gateway/certs - ./deployments/tyk/volumes/tyk-gateway/middleware:/opt/tyk-gateway/middleware depends_on: - tyk-redis - tyk-gateway - tyk-dashboard tyk-pump: - image: tykio/tyk-pump-docker-pub:${PUMP_VERSION:-v1.10.0} + image: tykio/tyk-pump-docker-pub:${PUMP_VERSION:-v1.11.0} ports: - 8083:8083 networks: @@ -156,6 +156,9 @@ services: volumes: tyk-redis-data: tyk-mongo-data: + tyk-kafka-data: + tyk-gateway-certs: + tyk-dashboard-certs: networks: tyk: diff --git a/deployments/tyk/volumes/tyk-dashboard/certs/certificates-go-here b/deployments/tyk/volumes/tyk-dashboard/certs/certificates-go-here deleted file mode 100644 index 0adc19d6..00000000 --- a/deployments/tyk/volumes/tyk-dashboard/certs/certificates-go-here +++ /dev/null @@ -1,2 +0,0 @@ -this directory is mapped to the container -certs generated by the bootstrap script are stored here \ No newline at end of file diff --git a/deployments/tyk/volumes/tyk-gateway/certs/certificates-go-here b/deployments/tyk/volumes/tyk-gateway/certs/certificates-go-here deleted file mode 100644 index 0adc19d6..00000000 --- a/deployments/tyk/volumes/tyk-gateway/certs/certificates-go-here +++ /dev/null @@ -1,2 +0,0 @@ -this directory is mapped to the container -certs generated by the bootstrap script are stored here \ No newline at end of file diff --git a/deployments/tyk/volumes/tyk-gateway/tyk.conf b/deployments/tyk/volumes/tyk-gateway/tyk.conf index fffae459..c631c95d 100755 --- a/deployments/tyk/volumes/tyk-gateway/tyk.conf +++ b/deployments/tyk/volumes/tyk-gateway/tyk.conf @@ -1,195 +1,200 @@ { - "listen_address": "", - "listen_port": 8080, - "secret": "28d220fd77974a4facfb07dc1e49c2aa", - "node_secret": "352d20ee67be67f6340b4c0605b044b7", - "template_path": "./templates", - "tyk_js_path": "./js/tyk.js", - "middleware_path": "./middleware", - "policies": { - "policy_source": "service", - "policy_connection_string": "http://tyk-dashboard:3000", - "policy_record_name": "tyk_policies", - "allow_explicit_policy_id": true - }, - "use_db_app_configs": true, - "db_app_conf_options": { - "connection_string": "http://tyk-dashboard:3000", - "node_is_segmented": false, - "tags": [] - }, - "disable_ports_whitelist": true, - "disable_dashboard_zeroconf": false, - "app_path": "./test_apps/", - "storage": { - "type": "redis", - "host": "tyk-redis", - "port": 6379, - "hosts": null, - "username": "", - "password": "", - "database": 0, - "optimisation_max_idle": 3000, - "optimisation_max_active": 5000, - "enable_cluster": false - }, - "enable_separate_cache_store": false, - "enable_analytics": true, - "analytics_config": { - "type": "mongo", - "ignored_ips": [], - "enable_detailed_recording": false, - "enable_geo_ip": false, - "geo_ip_db_path": "./GeoLite2-City.mmdb", - "normalise_urls": { - "enabled": true, - "normalise_uuids": true, - "normalise_numbers": true, - "custom_patterns": [] - } - }, - "health_check": { - "enable_health_checks": false, - "health_check_value_timeouts": 0 - }, - "optimisations_use_async_session_write": true, - "allow_master_keys": false, - "hash_keys": true, - "hash_key_function": "murmur64", - "suppress_redis_signal_reload": false, - "suppress_default_org_store": false, - "use_redis_log": true, - "sentry_code": "", - "use_sentry": false, - "use_syslog": false, - "use_graylog": false, - "use_logstash": false, - "graylog_network_addr": "", - "logstash_network_addr": "", - "syslog_transport": "", - "logstash_transport": "", - "syslog_network_addr": "", - "enforce_org_data_age": true, - "enforce_org_data_detail_logging": false, - "enforce_org_quotas": true, - "experimental_process_org_off_thread": false, - "enable_redis_rolling_limiter": false, - "enable_non_transactional_rate_limiter": true, - "enable_sentinel_rate_limiter": false, - "drl_threshold": 5, - "Monitor": { - "enable_trigger_monitors": false, - "configuration": { - "method": "", - "target_path": "", - "template_path": "", - "header_map": null, - "event_timeout": 0 - }, - "global_trigger_limit": 0, - "monitor_user_keys": false, - "monitor_org_keys": false - }, - "oauth_refresh_token_expire": 0, - "oauth_token_expire": 0, - "oauth_redirect_uri_separator": ";", - "slave_options": { - "use_rpc": false, - "connection_string": "", - "rpc_key": "", - "api_key": "", - "enable_rpc_cache": false, - "bind_to_slugs": false, - "disable_keyspace_sync": false, - "group_id": "" - }, - "disable_virtual_path_blobs": false, - "local_session_cache": { - "disable_cached_session_state": true, - "cached_session_timeout": 0, - "cached_session_eviction": 0 - }, - "http_server_options": { - "override_defaults": false, - "read_timeout": 0, - "write_timeout": 0, - "use_ssl": false, - "use_ssl_le": false, - "enable_websockets": true, - "certificates": [], - "server_name": "", - "min_version": 0, - "flush_interval": 0, - "enable_http2": true - }, - "proxy_enable_http2": true, - "service_discovery": { - "default_cache_timeout": 0 - }, - "close_connections": false, - "auth_override": { - "force_auth_provider": false, - "auth_provider": { - "name": "", - "storage_engine": "", - "meta": null - }, - "force_session_provider": false, - "session_provider": { - "name": "", - "storage_engine": "", - "meta": null - } - }, - "uptime_tests": { - "disable": false, - "config": { - "failure_trigger_sample_size": 1, - "time_wait": 30, - "checker_pool_size": 50, - "enable_uptime_analytics": true - } - }, - "hostname": "", - "enable_api_segregation": false, - "control_api_hostname": "", - "enable_custom_domains": true, - "enable_jsvm": true, - "coprocess_options": { - "enable_coprocess": true - }, - "hide_generator_header": false, - "event_handlers": { - "events": {} - }, - "event_trigers_defunct": {}, - "pid_file_location": "./tyk-gateway.pid", - "allow_insecure_configs": false, - "public_key_path": "certs/public-key.pem", - "close_idle_connections": false, - "allow_remote_config": true, - "enable_bundle_downloader": true, - "bundle_base_url": "", - "max_idle_connections_per_host": 500, - "tracing": { - "enabled": false, - "name": "zipkin", - "options": { - "reporter": { - "url": "http://zipkin:9411/api/v2/spans" - } + "listen_address": "", + "listen_port": 8080, + "secret": "28d220fd77974a4facfb07dc1e49c2aa", + "node_secret": "352d20ee67be67f6340b4c0605b044b7", + "template_path": "./templates", + "tyk_js_path": "./js/tyk.js", + "middleware_path": "./middleware", + "policies": { + "policy_source": "service", + "policy_connection_string": "http://tyk-dashboard:3000", + "policy_record_name": "tyk_policies", + "allow_explicit_policy_id": true + }, + "use_db_app_configs": true, + "db_app_conf_options": { + "connection_string": "http://tyk-dashboard:3000", + "node_is_segmented": false, + "tags": [] + }, + "disable_ports_whitelist": true, + "disable_dashboard_zeroconf": false, + "app_path": "./test_apps/", + "storage": { + "type": "redis", + "host": "tyk-redis", + "port": 6379, + "hosts": null, + "username": "", + "password": "", + "database": 0, + "optimisation_max_idle": 3000, + "optimisation_max_active": 5000, + "enable_cluster": false + }, + "enable_separate_cache_store": false, + "enable_analytics": true, + "analytics_config": { + "type": "mongo", + "ignored_ips": [], + "enable_detailed_recording": false, + "enable_geo_ip": false, + "geo_ip_db_path": "./GeoLite2-City.mmdb", + "normalise_urls": { + "enabled": true, + "normalise_uuids": true, + "normalise_numbers": true, + "custom_patterns": [] + } + }, + "health_check": { + "enable_health_checks": false, + "health_check_value_timeouts": 0 + }, + "optimisations_use_async_session_write": true, + "allow_master_keys": false, + "hash_keys": true, + "hash_key_function": "murmur64", + "suppress_redis_signal_reload": false, + "suppress_default_org_store": false, + "use_redis_log": true, + "sentry_code": "", + "use_sentry": false, + "use_syslog": false, + "use_graylog": false, + "use_logstash": false, + "graylog_network_addr": "", + "logstash_network_addr": "", + "syslog_transport": "", + "logstash_transport": "", + "syslog_network_addr": "", + "enforce_org_data_age": true, + "enforce_org_data_detail_logging": false, + "enforce_org_quotas": true, + "experimental_process_org_off_thread": false, + "enable_redis_rolling_limiter": false, + "enable_non_transactional_rate_limiter": true, + "enable_sentinel_rate_limiter": false, + "drl_threshold": 5, + "Monitor": { + "enable_trigger_monitors": false, + "configuration": { + "method": "", + "target_path": "", + "template_path": "", + "header_map": null, + "event_timeout": 0 + }, + "global_trigger_limit": 0, + "monitor_user_keys": false, + "monitor_org_keys": false + }, + "oauth_refresh_token_expire": 0, + "oauth_token_expire": 0, + "oauth_redirect_uri_separator": ";", + "slave_options": { + "use_rpc": false, + "connection_string": "", + "rpc_key": "", + "api_key": "", + "enable_rpc_cache": false, + "bind_to_slugs": false, + "disable_keyspace_sync": false, + "group_id": "" + }, + "disable_virtual_path_blobs": false, + "local_session_cache": { + "disable_cached_session_state": true, + "cached_session_timeout": 0, + "cached_session_eviction": 0 + }, + "http_server_options": { + "override_defaults": false, + "read_timeout": 0, + "write_timeout": 0, + "use_ssl": false, + "use_ssl_le": false, + "enable_websockets": true, + "certificates": [], + "server_name": "", + "min_version": 0, + "flush_interval": 0, + "enable_http2": true + }, + "proxy_enable_http2": true, + "service_discovery": { + "default_cache_timeout": 0 + }, + "close_connections": false, + "auth_override": { + "force_auth_provider": false, + "auth_provider": { + "name": "", + "storage_engine": "", + "meta": null + }, + "force_session_provider": false, + "session_provider": { + "name": "", + "storage_engine": "", + "meta": null + } + }, + "uptime_tests": { + "disable": false, + "config": { + "failure_trigger_sample_size": 1, + "time_wait": 30, + "checker_pool_size": 50, + "enable_uptime_analytics": true + } + }, + "hostname": "", + "enable_api_segregation": false, + "control_api_hostname": "", + "enable_custom_domains": true, + "enable_jsvm": true, + "coprocess_options": { + "enable_coprocess": true + }, + "hide_generator_header": false, + "event_handlers": { + "events": {} + }, + "event_trigers_defunct": {}, + "pid_file_location": "./tyk-gateway.pid", + "allow_insecure_configs": false, + "public_key_path": "certs/public-key.pem", + "close_idle_connections": false, + "allow_remote_config": true, + "enable_bundle_downloader": true, + "bundle_base_url": "", + "max_idle_connections_per_host": 500, + "tracing": { + "enabled": false, + "name": "zipkin", + "options": { + "reporter": { + "url": "http://zipkin:9411/api/v2/spans" + } + } + }, + "enable_hashed_keys_listing": true, + "statsd_connection_string": "graphite:8125", + "secrets": { + "target_url": "http://httpbin/", + "listen_path": "/secret-path/", + "header": "secret-header-value" + }, + "opentelemetry": { + "enabled": false, + "exporter": "grpc", + "endpoint": "collector-gateway:4317" + }, + "labs": { + "streaming": { + "enabled": true + } } - }, - "enable_hashed_keys_listing": true, - "statsd_connection_string": "graphite:8125", - "secrets": { - "target_url": "http://httpbin/", - "listen_path": "/secret-path/", - "header": "secret-header-value" - }, - "opentelemetry": { - "enabled": false, - "exporter": "grpc", - "endpoint": "collector-gateway:4317" - } -} +} \ No newline at end of file diff --git a/deployments/tyk2/bootstrap.sh b/deployments/tyk2/bootstrap.sh index 1dda6ee8..a819bb5c 100755 --- a/deployments/tyk2/bootstrap.sh +++ b/deployments/tyk2/bootstrap.sh @@ -8,9 +8,13 @@ bootstrap_progress dashboard_base_url="http://localhost:3002" gateway2_base_url="http://localhost:8085" -log_message "Waiting for Tyk 2 Dashboard to respond ok" +log_message "Restarting Tyk 2 services to use newly created certificates" +eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk2-dashboard tyk2-gateway +log_ok + +wait_for_liveness "http://localhost:8085/hello" + dashboard_admin_api_credentials=$(cat deployments/tyk/volumes/tyk-dashboard/tyk_analytics.conf | jq -r .admin_secret) -wait_for_response "$dashboard_base_url/admin/organisations" "200" "admin-auth: $dashboard_admin_api_credentials" log_message "Importing organisation" log_json_result "$(curl $dashboard_base_url/admin/organisations/import -s \ diff --git a/deployments/tyk2/docker-compose.yml b/deployments/tyk2/docker-compose.yml index dbe7974b..e75fb5e1 100644 --- a/deployments/tyk2/docker-compose.yml +++ b/deployments/tyk2/docker-compose.yml @@ -1,14 +1,14 @@ --- services: tyk2-dashboard: - image: tykio/tyk-dashboard:${DASHBOARD_VERSION:-v5.4.0} + image: tykio/tyk-dashboard:${DASHBOARD_VERSION:-v5.5.0} ports: - 3002:3000 networks: - tyk volumes: - ./deployments/tyk/volumes/tyk-dashboard/tyk_analytics.conf:/opt/tyk-dashboard/tyk_analytics.conf - - ./deployments/tyk/volumes/tyk-dashboard/certs:/opt/tyk-dashboard/certs + - tyk-dashboard-certs:/opt/tyk-dashboard/certs environment: - TYK_DB_LICENSEKEY=${DASHBOARD_LICENCE:?Please set DASHBOARD_LICENCE in .env} - TYK_DB_MONGOURL=mongodb://tyk2-mongo:27017/tyk_analytics @@ -20,7 +20,7 @@ services: - tyk2-redis - tyk2-mongo tyk2-gateway: - image: tykio/tyk-gateway:${TYK2_GATEWAY_VERSION:-v5.4.0} + image: tykio/tyk-gateway:${TYK2_GATEWAY_VERSION:-v5.5.0} ports: - 8085:8080 networks: @@ -28,7 +28,7 @@ services: volumes: - ./deployments/tyk/volumes/tyk-gateway/tyk.conf:/opt/tyk-gateway/tyk.conf - ./deployments/tyk/volumes/tyk-gateway/middleware:/opt/tyk-gateway/middleware - - ./deployments/tyk/volumes/tyk-gateway/certs:/opt/tyk-gateway/certs + - tyk-gateway-certs:/opt/tyk-gateway/certs environment: - TYK_GW_POLICIES_POLICYCONNECTIONSTRING=http://tyk2-dashboard:3000 - TYK_GW_DBAPPCONFOPTIONS_CONNECTIONSTRING=http://tyk2-dashboard:3000 @@ -37,7 +37,7 @@ services: depends_on: - tyk2-redis tyk2-pump: - image: tykio/tyk-pump-docker-pub:${TYK2_PUMP_VERSION:-v1.10.0} + image: tykio/tyk-pump-docker-pub:${TYK2_PUMP_VERSION:-v1.11.0} networks: - tyk volumes: diff --git a/scripts/add-gateway.sh b/scripts/add-gateway.sh index f75e6452..2250aebb 100755 --- a/scripts/add-gateway.sh +++ b/scripts/add-gateway.sh @@ -6,26 +6,28 @@ if [ "$1" == "" ]; then docker run \ -d \ + --expose 8080 \ -P \ -v $(pwd)/deployments/tyk/volumes/tyk-gateway/tyk.conf:/opt/tyk-gateway/tyk.conf \ - -v $(pwd)/deployments/tyk/volumes/tyk-gateway/certs:/opt/tyk-gateway/certs \ + -v tyk-demo_tyk-gateway-certs:/opt/tyk-gateway/certs \ -v $(pwd)/deployments/tyk/volumes/tyk-gateway/middleware:/opt/tyk-gateway/middleware \ -v $(pwd)/deployments/tyk/volumes/tyk-gateway/plugins:/opt/tyk-gateway/plugins \ -v $(pwd)/deployments/tyk/volumes/tyk-gateway/templates/error_401.json:/opt/tyk-gateway/templates/error_401.json \ -v $(pwd)/deployments/tyk/volumes/databases/GeoLite2-Country.mmdb:/opt/tyk-gateway/databases/GeoLite2-Country.mmdb \ --network tyk-demo_tyk \ - tykio/tyk-gateway:v5.4.0 + tykio/tyk-gateway:v5.5.0 else docker run \ --name $1 \ -d \ + --expose 8080 \ -P \ -v $(pwd)/deployments/tyk/volumes/tyk-gateway/tyk.conf:/opt/tyk-gateway/tyk.conf \ - -v $(pwd)/deployments/tyk/volumes/tyk-gateway/certs:/opt/tyk-gateway/certs \ + -v tyk-demo_tyk-gateway-certs:/opt/tyk-gateway/certs \ -v $(pwd)/deployments/tyk/volumes/tyk-gateway/middleware:/opt/tyk-gateway/middleware \ -v $(pwd)/deployments/tyk/volumes/tyk-gateway/plugins:/opt/tyk-gateway/plugins \ -v $(pwd)/deployments/tyk/volumes/tyk-gateway/templates/error_401.json:/opt/tyk-gateway/templates/error_401.json \ -v $(pwd)/deployments/tyk/volumes/databases/GeoLite2-Country.mmdb:/opt/tyk-gateway/databases/GeoLite2-Country.mmdb \ --network tyk-demo_tyk \ - tykio/tyk-gateway:v5.4.0 + tykio/tyk-gateway:v5.5.0 fi diff --git a/scripts/common.sh b/scripts/common.sh index b821a2cf..feb1636c 100755 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -159,6 +159,57 @@ wait_for_response () { done } +# TODO: make function here for this, and then check all certs etc exist in bootstrap +wait_for_file () { + local file_path="$1" + local container_name="$2" + local try_max=10 + local try_count=0 + log_message "Waiting for $file_path to be present in $container_name" + while true; do + ((try_count++)) + if [ "$try_count" -gt "$try_max" ]; then + echo "ERROR: Maximum retry count reached for file $file_path in container $container_name" + exit 1 + fi + + docker exec $container_name sh -c "[ -s $file_path ]" + if [ $? -eq 0 ]; then + log_ok + bootstrap_progress + return 0 + else + log_message " File not present, waiting... $try_count/$try_max" + bootstrap_progress + sleep 2 + fi + done +} + +wait_for_file_local() { + local file_path="$1" + local try_max=10 + local try_count=0 + log_message "Waiting for $file_path to be present" + while true; do + ((try_count++)) + if [ "$try_count" -gt "$try_max" ]; then + echo "ERROR: Maximum retry count reached for file $file_path" + exit 1 + fi + + if [ -s $file_path ]; then + log_ok + bootstrap_progress + return 0 + else + log_message " File not present, waiting... $try_count/$try_max" + bootstrap_progress + sleep 2 + fi + done +} + hot_reload () { local gateway_host="$1" local gateway_secret="$2" @@ -315,8 +366,10 @@ build_go_plugin () { cp $go_plugin_directory/*.so $go_plugin_cache_version_directory # limit the number of plugin caches to prevent uncontrolled growth - local PLUGIN_CACHE_MAX_SIZE=3 - local plugin_cache_count=$(find "$go_plugin_cache_directory" -maxdepth 1 -type d -not -path "$go_plugin_cache_directory" | wc -l) + local PLUGIN_CACHE_MAX_SIZE=$(grep -E '^PLUGIN_CACHE_MAX_SIZE=[0-9]+' .env | cut -d '=' -f2) + PLUGIN_CACHE_MAX_SIZE=${PLUGIN_CACHE_MAX_SIZE:-3} + local plugin_cache_count=$(find "$go_plugin_cache_directory" -maxdepth 1 -type d -not -path "$go_plugin_cache_directory" | wc -l | xargs) + log_message " Plugin cache used/max: $plugin_cache_count/$PLUGIN_CACHE_MAX_SIZE" if [ "$plugin_cache_count" -gt "$PLUGIN_CACHE_MAX_SIZE" ]; then oldest_plugin_cache_path=$(find "$go_plugin_cache_directory" -type d -not -path "$go_plugin_cache_directory" -exec ls -ld -ltr {} + | head -n 1 | awk '{print $9}') if [ -n "$oldest_plugin_cache_path" ]; then @@ -631,12 +684,12 @@ create_api () { api_id=$(jq -r '.["x-tyk-api-gateway"].info.id' $api_data_path) # import endpoint differs between classic and OAS APIs api_endpoint="$api_endpoint/oas" - log_message " Importing OAS API: $api_name" + log_message " Creating OAS API: $api_name" else # Tyk API api_name=$(jq -r '.api_definition.name' $api_data_path) api_id=$(jq -r '.api_definition.api_id' $api_data_path) - log_message " Importing Classic API: $api_name" + log_message " Creating Classic API: $api_name" fi log_message " Id: $api_id" @@ -680,7 +733,7 @@ create_policy () { check_variables - log_message " Importing Policy: $policy_name" + log_message " Creating Policy: $policy_name" api_response="$(curl $dashboard_base_url/api/portal/policies -s \ -H "authorization: $dashboard_api_key" \ @@ -879,20 +932,19 @@ wait_for_api_loaded () { } wait_for_liveness () { + local status_endpoint="${1:-http://tyk-gateway.localhost:8080/hello}" local attempt_count=0 - local pass="pass" log_message "Waiting for Gateway, Dashboard and Redis to be up and running" - while true - do + while true; do attempt_count=$((attempt_count+1)) #Check Gateway, Redis and Dashboard status - local hello=$(curl http://tyk-gateway.localhost:8080/hello -s) - local gw_status=$(echo "$hello" | jq -r '.status') - local dash_status=$(echo "$hello" | jq -r '.details.dashboard.status') - local redis_status=$(echo "$hello" | jq -r '.details.redis.status') + local status_response=$(curl $status_endpoint -s) + local gw_status=$(echo "$status_response" | jq -r '.status') + local dash_status=$(echo "$status_response" | jq -r '.details.dashboard.status') + local redis_status=$(echo "$status_response" | jq -r '.details.redis.status') if [[ "$gw_status" = "pass" ]] && [[ "$dash_status" = "pass" ]] && [[ "$redis_status" = "pass" ]] then @@ -903,6 +955,5 @@ wait_for_liveness () { fi sleep 2 - done } diff --git a/scripts/test-all.sh b/scripts/test-all.sh index ef22aab9..ce9d04bc 100755 --- a/scripts/test-all.sh +++ b/scripts/test-all.sh @@ -111,7 +111,7 @@ do --network tyk-demo_tyk \ -v $(pwd)/$postman_collection_path:/etc/postman/tyk_demo.postman_collection.json \ -v $(pwd)/test.postman_environment.json:/etc/postman/test.postman_environment.json \ - postman/newman:alpine \ + postman/newman:6.1.3-alpine \ run "/etc/postman/tyk_demo.postman_collection.json" \ --environment /etc/postman/test.postman_environment.json \ --insecure \ diff --git a/scripts/test.sh b/scripts/test.sh index bb68200c..e41324ef 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -32,7 +32,7 @@ while IFS= read -r deployment; do --network tyk-demo_tyk \ -v $collection_path:/etc/postman/tyk_demo.postman_collection.json \ -v $(pwd)/test.postman_environment.json:/etc/postman/test.postman_environment.json \ - postman/newman:alpine \ + postman/newman:6.1.3-alpine \ run "/etc/postman/tyk_demo.postman_collection.json" \ --environment /etc/postman/test.postman_environment.json \ --insecure diff --git a/up.sh b/up.sh index f19a5c6e..0c234ea2 100755 --- a/up.sh +++ b/up.sh @@ -2,6 +2,8 @@ source scripts/common.sh +up_start_time=$(date +%s) + # persistence of log files is disabled by default, meaning the files are recreated between each bootstrap to prevent them from growing too large # to enable persistence, use argument "persist-log" when running this script persist_log=false @@ -165,6 +167,16 @@ for deployment in "${deployments_to_create[@]}"; do fi done +up_end_time=$(date +%s) +up_elapsed_time=$((up_end_time - up_start_time)) +up_minutes=$((up_elapsed_time / 60)) +up_seconds=$((up_elapsed_time % 60)) +if [ $up_minutes -gt 0 ]; then + log_message "Elapsed time: $up_minutes minutes $up_seconds seconds" +else + log_message "Elapsed time: $up_seconds seconds" +fi + # Confirm initialisation process is complete printf "\nTyk Demo initialisation process completed" printf "\n-----------------------------------------\n\n"