diff --git a/attributes/default.rb b/attributes/default.rb index 6ed580ef..966a9af7 100644 --- a/attributes/default.rb +++ b/attributes/default.rb @@ -43,12 +43,12 @@ default['cloud']['cloudwatch']['download_url']['centos'] = "https://s3.amazonaws.com/amazoncloudwatch-agent/centos/amd64/#{node['cloud']['cloudwatch']['agent_version']}/amazon-cloudwatch-agent.rpm" default['cloud']['init']['expat_dir'] = "#{node['cloud']['init']['install_dir']}/expat" -default['cloud']['ndb-agent']['version'] = "0.11.0-SNAPSHOT" +default['cloud']['ndb-agent']['version'] = "1.0" default['cloud']['ndb-agent']['base_dir'] = "#{node['cloud']['install_dir']}/ndb-agent/ndb-agent-#{node['cloud']['ndb-agent']['version']}" default['cloud']['ndb-agent']['home'] = "#{node['cloud']['install_dir']}/ndb-agent/ndb-agent" default['cloud']['ndb-agent']['bin'] = "#{node['cloud']['ndb-agent']['home']}/bin" default['cloud']['ndb-agent']['config'] = "#{node['cloud']['ndb-agent']['home']}/config" -default['cloud']['ndb-agent']['log-level'] = "debug" +default['cloud']['ndb-agent']['log-level'] = "info" default['cloud']['ndb-agent']['templates-dir'] = "#{node['cloud']['ndb-agent']['home']}/templates" default['cloud']['metrics']['dir'] = "#{node["cloud"]["install_dir"]}/cloud-metrics-collector" diff --git a/files/default/ndb_templates/autobench_dbt2.conf b/files/default/ndb_templates/autobench_dbt2.conf deleted file mode 100644 index cd93e3b5..00000000 --- a/files/default/ndb_templates/autobench_dbt2.conf +++ /dev/null @@ -1,22 +0,0 @@ -# -# Software definition -# -MYSQL_BIN_INSTALL_DIR="{{.InstallDir}}" -BENCHMARK_TO_RUN="dbt2" -# -# Storage definition (empty here) -# -# -# MySQL Server definition -# -SERVER_HOST="{{.MysqlHosts}}" -MYSQL_PASSWORD='{{.MysqlPassword}}' -# -# NDB node definitions (empty here) -# -# -# Benchmark definition -# -DBT2_TIME="30" -DBT2_WAREHOUSES="64" -DBT2_DATA_DIR="{{.Dbt2DataDir}}" diff --git a/files/default/ndb_templates/autobench_sysbench.conf b/files/default/ndb_templates/autobench_sysbench.conf deleted file mode 100644 index 2347002c..00000000 --- a/files/default/ndb_templates/autobench_sysbench.conf +++ /dev/null @@ -1,57 +0,0 @@ -# -# Software definition -# -MYSQL_BIN_INSTALL_DIR="{{.InstallDir}}" -BENCHMARK_TO_RUN="sysbench" -# -# Storage definition (empty here) -# -# -# MySQL Server definition -# -SERVER_HOST="{{.MysqlHosts}}" -MYSQL_PASSWORD='{{.MysqlPassword}}' -# -# NDB node definitions (empty here) -# -# -# Benchmark definition -# -SYSBENCH_TEST="oltp_rw" -SYSBENCH_INSTANCES="{{.SysbenchInstances}}" -THREAD_COUNTS_TO_RUN="1;2;4;8;12;16;24;32;48;64;96;112;128" -MAX_TIME="30" -# Sysbench OLTP RW does 10 primary key lookups per transaction, -# 2 Updates, 1 Insert and 1 Delete and 4 range scan queries that -# each fetch 100 rows. This makes OLTP RW using a lot of network -# bandwidth. To test with less networking and also test RonDB's -# handling of filtering of rows one can set SB_USE_FILTER to -# yes. -#SB_USE_FILTER="yes" -# -# To run a simple key-value lookup benchmark using Sysbench one -# wants to remove the writes, remove the scan operations and -# use the IN statement to fetch multiple keyed rows per SQL -# statement. The below configuration will fetch 100 * 10 rows -# per transaction. -#SYSBENCH_TEST="oltp_ro" -#SB_POINT_SELECTS="10" -#SB_USE_IN_STATEMENT="100" -#SB_SUM_RANGES="0" -#SB_ORDER_RANGES="0" -#SB_DISTINCT_RANGES="0" -#SB_SIMPLE_RANGES="0" -# -# Other parameters that could be useful to vary are -# SB_RANGE_SIZE to define number of rows per scan -# SYSBENCH_ROWS number of rows per table -# SB_USE_SECONDARY_INDEX to use one secondary index -# per table -# SB_NUM_TABLES number of tables per sysbench run -# SYSBENCH_INSTANCES sets the number of parallel -# sysbench instances running -#SB_RANGE_SIZE="10" -#SYSBENCH_ROWS="10000000" -#SB_USE_SECONDARY_INDEX="yes" -#SB_NUM_TABLES="2" -#SYSBENCH_INSTANCES="2" diff --git a/files/default/ndb_templates/ndb_config.ini b/files/default/ndb_templates/ndb_config.ini deleted file mode 100644 index 539f8f25..00000000 --- a/files/default/ndb_templates/ndb_config.ini +++ /dev/null @@ -1,109 +0,0 @@ -# For documentation on config params, see: -# https://dev.mysql.com/doc/refman/8.0/en/mysql-cluster-ndbd-definition.html - -[NDBD DEFAULT] -AutomaticMemoryConfig=true -AutomaticThreadConfig=true - -NoOfReplicas={{.Configuration.NdbdDefault.MaxReplicationFactor}} -{{ if .Configuration.NdbdDefault.TotalMemoryConfig }} -TotalMemoryConfig={{ .Configuration.NdbdDefault.TotalMemoryConfig }} -{{ end }} -# Setting this parameter to TRUE or 1 binds IP_ADDR_ANY so that connections can be made from anywhere (for autogenerated connections). The default is FALSE (0). -TcpBind_INADDR_ANY=FALSE - -MaxNoOfConcurrentOperations=200000 -MaxDMLOperationsPerTransaction=32K - -MaxNoOfConcurrentScans=500 - -#The maximum time in ms that is permitted to lapse between operations in the same transaction before the transaction is aborted. -TransactionInactiveTimeout=1500 - -TransactionDeadlockDetectionTimeout=1500 - -#0: Disables locking. This is the default value. -#1: Performs the lock after allocating memory for the process. -#2: Performs the lock before memory for the process is allocated. -# Needs root privileges -LockPagesInMainMemory=1 - -# Savings of up to 50% over noncompressed LCPs and backups -CompressedLCP=0 -CompressedBackup=1 - -BackupLogBufferSize= 16M - -#The maximum size of the memory unit to use when allocating memory for tables -MaxAllocate=32M - -# Needs root privileges -ODirect=0 - -Numa=1 - -# Move this to another drive if you have a high number of ops/sec -InitialLogFileGroup=name=lg_1;undo_buffer_size=128M;undo_log_0.log:500M - -InitialTablespace=name=ts_1;extent_size=16M;ts_1_data_file_0.dat:1500M - -[MYSQLD DEFAULT] - -[NDB_MGMD DEFAULT] - -[TCP DEFAULT] -OverloadLimit=0 - -# -# Supports up to Max 3 ndb_mgmds. First one gets ArbitrationRank=1, others gets lower priority ArbitrationRank=2 -# -{{range .Mgmd}} -[NDB_MGMD] -NodeId={{.NodeId}} -LocationDomainId=0 -HostName={{.Ip}} -PortNumber={{.Port}} -NodeActive={{.NodeActive}} -DataDir=/srv/hops/mysql-cluster/log -LogDestination=FILE:filename=/srv/hops/mysql-cluster/log/cluster.log,maxsize=10000000,maxfiles=6 -ArbitrationRank={{.ArbitrationRank}} -{{end}} - -#Inactive Data nodes, to enable increase of replication level -#Hostname doesn't matter since it isn't allowed to connect to -#the cluster and before this can be done the node must be -#activated and an inactive node can change its hostname. -#Inactive nodes have NodeActive set to 0. -{{range .Ndbd}} -[NDBD] -NodeId={{.NodeId}} -NodeGroup={{.NodeGroupId}} -NodeActive={{.NodeActive}} -LocationDomainId=0 -HostName={{.Ip}} -ServerPort={{.Port}} -DataDir=/srv/hops/mysql-cluster/log -FileSystemPath=/srv/hops/mysql-cluster/ndb_data/{{.NodeId}} -FileSystemPathDD=/srv/hops/mysql-cluster/ndb_disk_columns -BackupDataDir=/srv/hops/mysql-cluster/ndb/backups -{{end}} - -#MySQL Servers, Memcached servers, and Clusterj clients. -{{range .Mysqld}} -[MYSQLD] -NodeId={{.NodeId}} -LocationDomainId=0 -NodeActive={{.NodeActive}} -ArbitrationRank={{.ArbitrationRank}} -{{if .Ip}}HostName={{.Ip}}{{end}} -{{end}} - -# List of all the services which are allowed to talk to NDB -{{range .Api}} -[API] -NodeId={{.NodeId}} -LocationDomainId=0 -NodeActive={{.NodeActive}} -ArbitrationRank={{.ArbitrationRank}} -{{if .Ip}}HostName={{.Ip}}{{end}} -{{end}} \ No newline at end of file diff --git a/recipes/ndb.rb b/recipes/ndb.rb index c24d44a6..91946447 100644 --- a/recipes/ndb.rb +++ b/recipes/ndb.rb @@ -46,9 +46,6 @@ tar xzf #{Chef::Config['file_cache_path']}/#{filename} -C #{Chef::Config['file_cache_path']} cp #{Chef::Config['file_cache_path']}/#{ndb_agent_folder}/ndb-agent #{node['cloud']['ndb-agent']['bin']} chmod 750 #{node['cloud']['ndb-agent']['bin']}/ndb-agent - - cp -r #{Chef::Config['file_cache_path']}/#{ndb_agent_folder}/contrib #{node['cloud']['ndb-agent']['bin']} - chmod -R 750 #{node['cloud']['ndb-agent']['bin']}/contrib EOH not_if { File.exist? "#{node['cloud']['ndb-agent']['bin']}/ndb-agent" } end @@ -67,14 +64,10 @@ }) end -remote_directory "#{node['cloud']['ndb-agent']['templates-dir']}" do - source 'ndb_templates' +directory "#{node['cloud']['ndb-agent']['templates-dir']}" do owner 'root' group 'root' mode 0750 - files_owner 'root' - files_group 'root' - files_mode 0550 end template "#{node['cloud']['ndb-agent']['templates-dir']}/join_hopsworks_cluster.sh" do diff --git a/templates/default/ndb-agent/config.yml.erb b/templates/default/ndb-agent/config.yml.erb index 37f90f05..80f37e8c 100644 --- a/templates/default/ndb-agent/config.yml.erb +++ b/templates/default/ndb-agent/config.yml.erb @@ -1,17 +1,14 @@ general: log-level: <%= node['cloud']['ndb-agent']['log-level'] %> - handler-channel-size: 20 - cluster-topology-location: <%= node['cloud']['ndb-agent']['config'] %>/cluster-topology.json backup-staging-directory: <%= node['ndb']['BackupDataDir'] %> - extra-mysqld-vms: 1 + public-state-filepath: <%= node['cloud']['ndb-agent']['config'] %>/public-state.json + fallback-service-discovery: true + agent-running-since-filepath: <%= node['cloud']['ndb-agent']['config'] %>/agent-running-since.txt http: - listen-port: 8181 api-key: API_KEY -heartbeat: - enabled: HEARTBEAT_ENABLED - endpoint: HEARTBEAT_ENDPOINT +leader: is-leader: HEARTBEAT_ENABLED cloud-base-endpoint: HEARTBEAT_ENDPOINT @@ -20,18 +17,12 @@ templates: ndb-config-target: <%= node['ndb']['root_dir'] %>/config.ini benchmarks-target-home: <%= node['ndb']['user-home']%>/benchmarks -rondb-internal: - base: - mysqld-start-timeout: "2h" +timeouts: + start-ndbds: "2h" + start-mysqlds: "2h" <% if @template_db_section -%> database: hostname: 127.0.0.1 username: root <% end -%> - -plugins: - unmarshal: - Clusterpoststartunmarshal: <%= node['cloud']['ndb-agent']['bin'] %>/contrib/plugins/hopsworks_plugin.so - handlers: - Clusterpoststarthandler: <%= node['cloud']['ndb-agent']['bin'] %>/contrib/plugins/hopsworks_plugin.so