modules: tt.yaml modules/directory support list

The logic of working with configuration file tt.yaml has been updated,
now  modules/directory can be specified both as a single line and
as a list.

Closes #1012

CHANGELOG.md

@@ -17,6 +17,7 @@ operations ending with the given timestamp. This value can be specified
 as a number or using [RFC3339/RFC3339Nano](https://go.dev/src/time/format.go) time format.
 - `tt connect`: add new `--evaler` option to support for customizing
 the way user input is processed.
+- `tt.yaml` allow specify a list of modules directories.
 
 ### Changed
 

cli/cfg/dump_test.go

@@ -93,7 +93,8 @@ env:
   restart_on_failure: false
   tarantoolctl_layout: false
 modules:
-  directory: /root/modules
+  directory:
+  - /root/modules
 app:
   run_dir: var/run
   log_dir: var/log
@@ -129,7 +130,8 @@ env:
   restart_on_failure: false
   tarantoolctl_layout: false
 modules:
-  directory: %[1]s/my_modules
+  directory:
+  - %[1]s/my_modules
 app:
   run_dir: var/run
   log_dir: var/log
@@ -166,7 +168,8 @@ env:
   restart_on_failure: false
   tarantoolctl_layout: false
 modules:
-  directory: /root/modules
+  directory:
+  - /root/modules
 app:
   run_dir: var/run
   log_dir: var/log

cli/config/config.go

@@ -21,11 +21,15 @@ package config
 //  ee:
 //    credential_path: path
 
+// FieldStringArrayType a custom type used with mapstructure's hook to accept values
+// as a single string as well as a list of strings.
+type FieldStringArrayType []string
+
 // ModuleOpts is used to store all module options.
 type ModulesOpts struct {
-	// Directory is a path to directory where the external modules
+	// Directories is a list of paths to directories where the external modules
 	// are stored.
-	Directory string
+	Directories FieldStringArrayType `mapstructure:"directory" yaml:"directory"`
 }
 
 // EEOpts is used to store tarantool-ee options.

cli/configure/configure.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"reflect"
 	"strings"
 	"syscall"
 
@@ -103,7 +104,7 @@ func getSystemAppOpts() *config.AppOpts {
 // GetDefaultCliOpts returns `CliOpts` filled with default values.
 func GetSystemCliOpts() *config.CliOpts {
 	modules := config.ModulesOpts{
-		Directory: ModulesPath,
+		Directories: []string{ModulesPath},
 	}
 	ee := config.EEOpts{
 		CredPath: "",
@@ -128,7 +129,7 @@ func GetSystemCliOpts() *config.CliOpts {
 // GetDefaultCliOpts returns `CliOpts` filled with default values.
 func GetDefaultCliOpts() *config.CliOpts {
 	modules := config.ModulesOpts{
-		Directory: ModulesPath,
+		Directories: []string{ModulesPath},
 	}
 	ee := config.EEOpts{
 		CredPath: "",
@@ -179,6 +180,24 @@ func adjustPathWithConfigLocation(filePath string, configDir string,
 	return filepath.Abs(filepath.Join(configDir, filePath))
 }
 
+func adjustListPathWithConfigLocation(listPaths []string, configDir string,
+	defaultDirName string) ([]string, error) {
+	if len(listPaths) == 0 {
+		listPaths = append(listPaths, defaultDirName)
+	}
+
+	result := make([]string, 0, len(listPaths))
+	for _, path := range listPaths {
+		path, err := adjustPathWithConfigLocation(path, configDir, defaultDirName)
+		if err != nil {
+			return result, err
+		}
+		result = append(result, path)
+
+	}
+	return result, nil
+}
+
 // resolveConfigPaths resolves all paths in config relative to specified location, and
 // sets uninitialized values to defaults.
 func updateCliOpts(cliOpts *config.CliOpts, configDir string) error {
@@ -211,8 +230,8 @@ func updateCliOpts(cliOpts *config.CliOpts, configDir string) error {
 	}
 
 	if cliOpts.Modules != nil {
-		if cliOpts.Modules.Directory, err = adjustPathWithConfigLocation(cliOpts.Modules.Directory,
-			configDir, ModulesPath); err != nil {
+		if cliOpts.Modules.Directories, err = adjustListPathWithConfigLocation(
+			cliOpts.Modules.Directories, configDir, ModulesPath); err != nil {
 			return err
 		}
 	}
@@ -227,6 +246,26 @@ func updateCliOpts(cliOpts *config.CliOpts, configDir string) error {
 	return nil
 }
 
+func decodeStringAsArrayField(from reflect.Type, to reflect.Type, value interface{}) (
+	interface{}, error) {
+	if to != reflect.TypeOf(config.FieldStringArrayType{}) || from.Kind() != reflect.String {
+		return value, nil
+	}
+	return []string{value.(string)}, nil
+}
+
+func decodeConfig(input map[string]any, cfg *config.CliOpts) error {
+	decoder_config := mapstructure.DecoderConfig{
+		Result:     cfg,
+		DecodeHook: mapstructure.ComposeDecodeHookFunc(decodeStringAsArrayField),
+	}
+	decoder, err := mapstructure.NewDecoder(&decoder_config)
+	if err != nil {
+		return err
+	}
+	return decoder.Decode(input)
+}
+
 // GetCliOpts returns Tarantool CLI options from the config file
 // located at path configurePath.
 func GetCliOpts(configurePath string, repository integrity.Repository) (
@@ -250,7 +289,7 @@ func GetCliOpts(configurePath string, repository integrity.Repository) (
 			return nil, "", fmt.Errorf("failed to parse Tarantool CLI configuration: %s", err)
 		}
 
-		if err := mapstructure.Decode(rawConfigOpts, &cfg); err != nil {
+		if err := decodeConfig(rawConfigOpts, cfg); err != nil {
 			return nil, "", fmt.Errorf("failed to parse Tarantool CLI configuration: %s", err)
 		}
 

cli/configure/configure_test.go

@@ -334,6 +334,67 @@ func TestUpdateCliOpts(t *testing.T) {
 	assert.Equal(t, "./var/lib/vinyl", cliOpts.App.VinylDir)
 	assert.Equal(t, "./var/lib/snap", cliOpts.App.MemtxDir)
 	assert.Equal(t, filepath.Join(configDir, "..", "include_dir"), cliOpts.Env.IncludeDir)
-	assert.Equal(t, filepath.Join(configDir, ModulesPath), cliOpts.Modules.Directory)
+	assert.Equal(t, 1, len(cliOpts.Modules.Directories))
+	assert.Equal(t, filepath.Join(configDir, ModulesPath), cliOpts.Modules.Directories[0])
 	assert.Equal(t, configDir, cliOpts.Env.InstancesEnabled)
 }
+
+func TestGetCliOpts_modules_directory(t *testing.T) {
+	work_dir, err := os.Getwd()
+	require.NoError(t, err)
+	work_dir = filepath.Join(work_dir, "testdata/modules_cfg")
+
+	tests := []struct {
+		name        string
+		config      string
+		modules_dir config.FieldStringArrayType
+		cfg_path    string
+	}{
+		{
+			name:        "Single string relative path",
+			config:      "tt-modules1",
+			modules_dir: []string{filepath.Join(work_dir, "modules-dir")},
+			cfg_path:    "tt-modules1.yaml",
+		},
+		{
+			name:        "Single entry list",
+			config:      "tt-modules2",
+			modules_dir: []string{filepath.Join(work_dir, "modules-dir")},
+			cfg_path:    "tt-modules2.yml",
+		},
+		{
+			name:   "Multiple entries list",
+			config: "tt-modules3.",
+			modules_dir: []string{
+				filepath.Join(work_dir, "modules-dir"),
+				"/ext/path/modules",
+				filepath.Join(work_dir, "local_modules"),
+			},
+			cfg_path: "tt-modules3.yaml",
+		},
+		{
+			name:        "Empty list = default value",
+			config:      "tt-modules4.yaml",
+			modules_dir: []string{filepath.Join(work_dir, "modules")},
+			cfg_path:    "tt-modules4.yml",
+		},
+		{
+			name:        "Single string absolute path",
+			config:      "tt-modules5.yml",
+			modules_dir: []string{"/ext/path/modules"},
+			cfg_path:    "tt-modules5.yaml",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			mockRepo := newMockRepository()
+			config := filepath.Join(work_dir, tt.config)
+			opts, cfg, err := GetCliOpts(config, &mockRepo)
+			require.NoError(t, err)
+			require.NotNil(t, opts.Modules)
+			require.Equal(t, tt.modules_dir, opts.Modules.Directories)
+			require.Equal(t, filepath.Join(work_dir, tt.cfg_path), cfg)
+		})
+	}
+}

cli/configure/testdata/modules_cfg/tt-modules1.yaml

@@ -0,0 +1,3 @@
+# Config with single string with relative path to modules directory.
+modules:
+  directory: modules-dir

cli/configure/testdata/modules_cfg/tt-modules2.yml

@@ -0,0 +1,4 @@
+# Config with single list entry of relative path to modules directory.
+modules:
+  directory:
+  - modules-dir

cli/configure/testdata/modules_cfg/tt-modules3.yaml

@@ -0,0 +1,6 @@
+# Multiple directory items.
+modules:
+  directory:
+    - modules-dir
+    - /ext/path/modules
+    - local_modules

cli/configure/testdata/modules_cfg/tt-modules4.yml

@@ -0,0 +1,3 @@
+# Empty config value, to use default.
+modules:
+  directory:

cli/configure/testdata/modules_cfg/tt-modules5.yaml

@@ -0,0 +1,3 @@
+# Config with single string with absolute path to modules directory.
+modules:
+  directory: /ext/path/modules

cli/init/init.go

@@ -168,11 +168,12 @@ func generateTtEnv(configPath string, sourceCfg configData) error {
 
 	directoriesToCreate := []string{
 		cfg.Env.InstancesEnabled,
-		cfg.Modules.Directory,
 		cfg.Env.IncludeDir,
 		cfg.Env.BinDir,
 		cfg.Repo.Install,
 	}
+	// FIXME: Need select only internal directories https://github.com/tarantool/tt/issues/1014
+	directoriesToCreate = append(directoriesToCreate, cfg.Modules.Directories...)
 	for _, templatesPathOpts := range cfg.Templates {
 		directoriesToCreate = append(directoriesToCreate, templatesPathOpts.Path)
 	}

cli/init/init_test.go

@@ -63,7 +63,7 @@ func checkDefaultEnv(t *testing.T, configName string, instancesEnabled string) {
 	assert.Equal(t, "var/run", cfg.App.RunDir)
 	assert.Equal(t, "var/log", cfg.App.LogDir)
 	assert.Equal(t, "bin", cfg.Env.BinDir)
-	assert.Equal(t, "modules", cfg.Modules.Directory)
+	assert.Equal(t, config.FieldStringArrayType{"modules"}, cfg.Modules.Directories)
 	assert.Equal(t, "distfiles", cfg.Repo.Install)
 	assert.Equal(t, "include", cfg.Env.IncludeDir)
 	assert.Equal(t, "templates", cfg.Templates[0].Path)

cli/init/templates/tt.yaml.default

@@ -1,6 +1,6 @@
 modules:
   # Directory where the external modules are stored.
-  directory: {{ .Modules.Directory }}
+  directory: {{ .Modules.Directories }}
 
 env:
   # Restart instance on failure.

cli/modules/modules.go

@@ -94,7 +94,8 @@ func getExternalModulesDir(cmdCtx *cmdcontext.CmdCtx, cliOpts *config.CliOpts) (
 	// 1. If a directory field is specified;
 	// 2. Specified path exists;
 	// 3. Path points to not a directory.
-	modulesDir := cliOpts.Modules.Directory
+	// FIXME: Add working with a list https://github.com/tarantool/tt/issues/1014
+	modulesDir := cliOpts.Modules.Directories[0]
 	if info, err := os.Stat(modulesDir); err == nil {
 		// TODO: Add warning in next patches, discussion
 		// what if the file exists, but access is denied, etc.

cli/pack/common.go

@@ -115,7 +115,7 @@ func ttEnvironmentFilters(packCtx *PackCtx, cliOpts *config.CliOpts) []func(
 			cliOpts.Env.InstancesEnabled, cliOpts.Env.BinDir)
 	}
 	if cliOpts.Modules != nil {
-		envPaths = append(envPaths, cliOpts.Modules.Directory)
+		envPaths = append(envPaths, cliOpts.Modules.Directories...)
 	}
 	if cliOpts.Repo != nil {
 		envPaths = append(envPaths, cliOpts.Repo.Install)
@@ -204,24 +204,27 @@ func updateEnvPath(basePath string, packCtx *PackCtx, cliOpts *config.CliOpts) (
 // copyEnvModules copies tt modules.
 func copyEnvModules(bundleEnvPath string, packCtx *PackCtx, cliOpts, newOpts *config.CliOpts) {
 	if packCtx.WithoutModules || packCtx.CartridgeCompat || cliOpts.Modules == nil ||
-		cliOpts.Modules.Directory == "" {
+		len(cliOpts.Modules.Directories) == 0 {
 		return
 	}
 
-	if !util.IsDir(cliOpts.Modules.Directory) {
-		log.Debugf("Skip copying modules from %q: does not exist or not a directory",
-			cliOpts.Modules.Directory)
-	} else {
-		dir, err := os.Open(cliOpts.Modules.Directory)
-		if err != nil {
-			log.Warnf("cannot open %q for reading: %s", cliOpts.Modules.Directory, err)
-		}
-		if files, _ := dir.Readdir(1); len(files) == 0 {
-			return // No modules.
-		}
-		if err := copy.Copy(cliOpts.Modules.Directory,
-			util.JoinPaths(bundleEnvPath, newOpts.Modules.Directory)); err != nil {
-			log.Warnf("Failed to copy modules from %q: %s", cliOpts.Modules.Directory, err)
+	for _, directory := range cliOpts.Modules.Directories {
+		if !util.IsDir(directory) {
+			log.Debugf("Skip copying modules from %q: does not exist or not a directory",
+				directory)
+		} else {
+			dir, err := os.Open(directory)
+			if err != nil {
+				log.Warnf("cannot open %q for reading: %s", directory, err)
+			}
+			if files, _ := dir.Readdir(1); len(files) == 0 {
+				return // No modules.
+			}
+			// FIXME: Add working with a list https://github.com/tarantool/tt/issues/1014
+			if err := copy.Copy(directory,
+				util.JoinPaths(bundleEnvPath, newOpts.Modules.Directories[0])); err != nil {
+				log.Warnf("Failed to copy modules from %q: %s", directory, err)
+			}
 		}
 	}
 }

cli/pack/common_test.go

@@ -243,7 +243,7 @@ func Test_createNewOpts(t *testing.T) {
 					RunDir:   "var/run",
 				},
 				Modules: &config.ModulesOpts{
-					Directory: "modules",
+					Directories: []string{"modules"},
 				},
 				Repo: &config.RepoOpts{
 					Rocks:   "",
@@ -280,7 +280,7 @@ func Test_createNewOpts(t *testing.T) {
 					RunDir:   "var/run",
 				},
 				Modules: &config.ModulesOpts{
-					Directory: "modules",
+					Directories: []string{"modules"},
 				},
 				Repo: &config.RepoOpts{
 					Rocks:   "",
@@ -317,7 +317,7 @@ func Test_createNewOpts(t *testing.T) {
 					RunDir:   "/var/run/tarantool/bundle",
 				},
 				Modules: &config.ModulesOpts{
-					Directory: "modules",
+					Directories: []string{"modules"},
 				},
 				Repo: &config.RepoOpts{
 					Rocks:   "",
@@ -360,7 +360,7 @@ func Test_createNewOpts(t *testing.T) {
 					RunDir:   "var/run",
 				},
 				Modules: &config.ModulesOpts{
-					Directory: "modules",
+					Directories: []string{"modules"},
 				},
 				Repo: &config.RepoOpts{
 					Rocks:   "",