From 17ecdbeb5e898fb1a5f046f52cfc645d0ea3d8f5 Mon Sep 17 00:00:00 2001 From: Wei Shen Date: Fri, 5 May 2017 21:55:28 +0800 Subject: [PATCH] v0.2.0 --- README.md | 100 +++++++++++++++++++++++++++----- brename.go | 164 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 244 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index e226713..35b8b2d 100755 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# brename -- a cross-platform command-line tool for safely batch renaming files/directories via regular expression +# brename -- a practical cross-platform command-line tool for safely batch renaming files/directories via regular expression [![Built with GoLang](https://img.shields.io/badge/powered_by-go-6362c2.svg?style=flat)](https://golang.org) [![Go Report Card](https://goreportcard.com/badge/github.com/shenwei356/brename)](https://goreportcard.com/report/github.com/shenwei356/brename) @@ -28,6 +28,8 @@ - **Safe**. By ***checking potential conflicts and errors***. - **File filtering**. Supporting including and excluding files via regular expression. No need to run commands like `find ./ -name "*.html" -exec CMD`. +- **Renaming submatch with corresponding value via key-value file** +- **Renaming via ascending integer** - **Recursively renaming both files and directories**. - **Supporting dry run**. - **Colorful output**. Screenshots: @@ -46,18 +48,18 @@ #### Method 1: Download binaries -[brename v2.1.3](https://github.com/shenwei356/brename/releases/tag/v2.1.3) -[![Github Releases (by Release)](https://img.shields.io/github/downloads/shenwei356/brename/v2.1.3/total.svg)](https://github.com/shenwei356/brename/releases/tag/v2.1.3) +[brename v2.2.0](https://github.com/shenwei356/brename/releases/tag/v2.2.0) +[![Github Releases (by Release)](https://img.shields.io/github/downloads/shenwei356/brename/v2.2.0/total.svg)](https://github.com/shenwei356/brename/releases/tag/v2.2.0) OS |Arch |File, (mirror为中国用户下载镜像链接) |Download Count :------|:---------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -Linux |32-bit |[brename_linux_386.tar.gz](https://github.com/shenwei356/brename/releases/download/v2.1.3/brename_linux_386.tar.gz), ([mirror](http://app.shenwei.me/data/brename/brename_linux_386.tar.gz)) |[![Github Releases (by Asset)](https://img.shields.io/github/downloads/shenwei356/brename/latest/brename_linux_386.tar.gz.svg?maxAge=3600)](https://github.com/shenwei356/brename/releases/download/v2.1.3/brename_linux_386.tar.gz) -Linux |**64-bit**|[**brename_linux_amd64.tar.gz**](https://github.com/shenwei356/brename/releases/download/v2.1.3/brename_linux_amd64.tar.gz), ([mirror](http://app.shenwei.me/data/brename/brename_linux_amd64.tar.gz)) |[![Github Releases (by Asset)](https://img.shields.io/github/downloads/shenwei356/brename/latest/brename_linux_amd64.tar.gz.svg?maxAge=3600)](https://github.com/shenwei356/brename/releases/download/v2.1.3/brename_linux_amd64.tar.gz) -OS X |32-bit |[brename_darwin_386.tar.gz](https://github.com/shenwei356/brename/releases/download/v2.1.3/brename_darwin_386.tar.gz), ([mirror](http://app.shenwei.me/data/brename/brename_darwin_386.tar.gz)) |[![Github Releases (by Asset)](https://img.shields.io/github/downloads/shenwei356/brename/latest/brename_darwin_386.tar.gz.svg?maxAge=3600)](https://github.com/shenwei356/brename/releases/download/v2.1.3/brename_darwin_386.tar.gz) -OS X |**64-bit**|[**brename_darwin_amd64.tar.gz**](https://github.com/shenwei356/brename/releases/download/v2.1.3/brename_darwin_amd64.tar.gz), ([mirror](http://app.shenwei.me/data/brename/brename_darwin_amd64.tar.gz)) |[![Github Releases (by Asset)](https://img.shields.io/github/downloads/shenwei356/brename/latest/brename_darwin_amd64.tar.gz.svg?maxAge=3600)](https://github.com/shenwei356/brename/releases/download/v2.1.3/brename_darwin_amd64.tar.gz) -Windows|32-bit |[brename_windows_386.exe.tar.gz](https://github.com/shenwei356/brename/releases/download/v2.1.3/brename_windows_386.exe.tar.gz), ([mirror](http://app.shenwei.me/data/brename/brename_windows_386.exe.tar.gz)) |[![Github Releases (by Asset)](https://img.shields.io/github/downloads/shenwei356/brename/latest/brename_windows_386.exe.tar.gz.svg?maxAge=3600)](https://github.com/shenwei356/brename/releases/download/v2.1.3/brename_windows_386.exe.tar.gz) -Windows|**64-bit**|[**brename_windows_amd64.exe.tar.gz**](https://github.com/shenwei356/brename/releases/download/v2.1.3/brename_windows_amd64.exe.tar.gz), ([mirror](http://app.shenwei.me/data/brename/brename_windows_amd64.exe.tar.gz))|[![Github Releases (by Asset)](https://img.shields.io/github/downloads/shenwei356/brename/latest/brename_windows_amd64.exe.tar.gz.svg?maxAge=3600)](https://github.com/shenwei356/brename/releases/download/v2.1.3/brename_windows_amd64.exe.tar.gz) +Linux |32-bit |[brename_linux_386.tar.gz](https://github.com/shenwei356/brename/releases/download/v2.2.0/brename_linux_386.tar.gz), ([mirror](http://app.shenwei.me/data/brename/brename_linux_386.tar.gz)) |[![Github Releases (by Asset)](https://img.shields.io/github/downloads/shenwei356/brename/latest/brename_linux_386.tar.gz.svg?maxAge=3600)](https://github.com/shenwei356/brename/releases/download/v2.2.0/brename_linux_386.tar.gz) +Linux |**64-bit**|[**brename_linux_amd64.tar.gz**](https://github.com/shenwei356/brename/releases/download/v2.2.0/brename_linux_amd64.tar.gz), ([mirror](http://app.shenwei.me/data/brename/brename_linux_amd64.tar.gz)) |[![Github Releases (by Asset)](https://img.shields.io/github/downloads/shenwei356/brename/latest/brename_linux_amd64.tar.gz.svg?maxAge=3600)](https://github.com/shenwei356/brename/releases/download/v2.2.0/brename_linux_amd64.tar.gz) +OS X |32-bit |[brename_darwin_386.tar.gz](https://github.com/shenwei356/brename/releases/download/v2.2.0/brename_darwin_386.tar.gz), ([mirror](http://app.shenwei.me/data/brename/brename_darwin_386.tar.gz)) |[![Github Releases (by Asset)](https://img.shields.io/github/downloads/shenwei356/brename/latest/brename_darwin_386.tar.gz.svg?maxAge=3600)](https://github.com/shenwei356/brename/releases/download/v2.2.0/brename_darwin_386.tar.gz) +OS X |**64-bit**|[**brename_darwin_amd64.tar.gz**](https://github.com/shenwei356/brename/releases/download/v2.2.0/brename_darwin_amd64.tar.gz), ([mirror](http://app.shenwei.me/data/brename/brename_darwin_amd64.tar.gz)) |[![Github Releases (by Asset)](https://img.shields.io/github/downloads/shenwei356/brename/latest/brename_darwin_amd64.tar.gz.svg?maxAge=3600)](https://github.com/shenwei356/brename/releases/download/v2.2.0/brename_darwin_amd64.tar.gz) +Windows|32-bit |[brename_windows_386.exe.tar.gz](https://github.com/shenwei356/brename/releases/download/v2.2.0/brename_windows_386.exe.tar.gz), ([mirror](http://app.shenwei.me/data/brename/brename_windows_386.exe.tar.gz)) |[![Github Releases (by Asset)](https://img.shields.io/github/downloads/shenwei356/brename/latest/brename_windows_386.exe.tar.gz.svg?maxAge=3600)](https://github.com/shenwei356/brename/releases/download/v2.2.0/brename_windows_386.exe.tar.gz) +Windows|**64-bit**|[**brename_windows_amd64.exe.tar.gz**](https://github.com/shenwei356/brename/releases/download/v2.2.0/brename_windows_amd64.exe.tar.gz), ([mirror](http://app.shenwei.me/data/brename/brename_windows_amd64.exe.tar.gz))|[![Github Releases (by Asset)](https://img.shields.io/github/downloads/shenwei356/brename/latest/brename_windows_amd64.exe.tar.gz.svg?maxAge=3600)](https://github.com/shenwei356/brename/releases/download/v2.2.0/brename_windows_amd64.exe.tar.gz) Just [download](https://github.com/shenwei356/brename/releases) compressed @@ -87,9 +89,9 @@ And then: ## Usage ``` -brename -- a cross-platform command-line tool for safely batch renaming files/directories via regular expression +brename -- a practical cross-platform command-line tool for safely batch renaming files/directories via regular expression -Version: 2.1.3 +Version: 2.2.0 Author: Wei Shen @@ -102,6 +104,12 @@ Attention: e.g., -f ".html" -f ".htm". But ATTENTION: comma in filter is treated as separater of multiple filters. +Special replacement symbols: + + {nr} Ascending integer + {kv} Corresponding value of the key (captured variable $n) by key-value file, + n can be specified by flag -I/--key-capt-idx (default: 1) + Usage: brename [flags] @@ -119,6 +127,8 @@ Examples: or brename -p "(a)" -r '$1$1' in Linux/Mac OS X 6. renaming directory too brename -p ":" -r "-" -R -D pdf-dirs + 7. using key-value file + brename -p "(.+)" -r "{kv}" -k kv.tsv More examples: https://github.com/shenwei356/brename @@ -128,9 +138,14 @@ Flags: -i, --ignore-case ignore case -f, --include-filters stringSlice include file filter(s) (regular expression, case ignored). multiple values supported, e.g., -f ".html" -f ".htm", but ATTENTION: comma in filter is treated as separater of multiple filters (default [.]) -D, --including-dir rename directories + -K, --keep-key keep the key as value when no value found for the key + -I, --key-capt-idx int capture variable index of key (1-based) (default 1) + -m, --key-miss-repl string replacement for key with no corresponding value + -k, --kv-file string tab-delimited key-value file for replacing key with value when using "{kv}" in -r (--replacement) -p, --pattern string search pattern (regular expression) -R, --recursive rename recursively - -r, --replacement string replacement. capture variables supported. e.g. $1 represents the first submatch. ATTENTION: for *nix OS, use SINGLE quote NOT double quotes or use the \ escape character. + -r, --replacement string replacement. capture variables supported. e.g. $1 represents the first submatch. ATTENTION: for *nix OS, use SINGLE quote NOT double quotes or use the \ escape character. Ascending integer is also supported by "{nr}" + -n, --start-num int starting number when using {nr} in replacement (default 1) -v, --verbose int verbose level (0 for all, 1 for warning and error, 2 for only error) -V, --version print version information and check for update @@ -282,11 +297,68 @@ Take a directory for example: [INFO] checking: [ ok ] 'b.jpg' -> 'c.jpg' [INFO] 2 path(s) to be renamed +1. Rename with number (-r `{nr}`) + + $ brename -p '(.+)\.' -r 'pic-{nr}.' -f .jpg + [INFO] checking: [ ok ] 'AA.jpg' -> 'pic-1.jpg' + [INFO] checking: [ ok ] 'b.jpg' -> 'pic-2.jpg' + [INFO] 2 path(s) to be renamed + +1. Replace submatch with corresponding value via key-value file (`-k/--kv-file`) + + $ more kv.tsv + a 一 + b 二 + c 三 + + $ brename -p '^(\w)' -r '{kv}' -k kv.tsv -K -i -d + [INFO] read key-value file: kv.tsv + [INFO] 3 pairs of key-value loaded + [INFO] checking: [ ok ] 'AA.jpg' -> '一A.jpg' + [INFO] checking: [ ok ] 'b.jpg' -> '二.jpg' + [WARN] checking: [ unchanged ] 'hello b.html' -> 'hello b.html' + [WARN] checking: [ unchanged ] 'kv.tsv' -> 'kv.tsv' + + ## Real-world examples +1. Replace matches with corresponding pairing values + + 1. Original files + + $ tree + . + ├── barcodes.tsv + ├── tag_ATGCGTA.fasta + ├── tag_CCCCCCC.fasta + ├── tag_CGACGTC.fasta + ├── tag_TCATAGC.fasta + └── tag_TCTATAG.fasta + + 1. Tab-delimited key-value file. Notice that `CCCCCCC` is not in it. + + $ cat barcodes.tsv + CGACGTC S1 + ATGCGTA S2 + TCTATAG S4 + TCATAGC S3 + + 1. Renaming tag as sample names, marking `unknown` for non-existing tag. + + $ brename -p 'tag_(\w+)' -r '{kv}' -k barcodes.tsv -m unknown -d + [INFO] read key-value file: barcodes.tsv + [INFO] 4 pairs of key-value loaded + [INFO] checking: [ ok ] 'tag_ATGCGTA.fasta' -> 'S2.fasta' + [INFO] checking: [ ok ] 'tag_CCCCCCC.fasta' -> 'unknown.fasta' + [INFO] checking: [ ok ] 'tag_CGACGTC.fasta' -> 'S1.fasta' + [INFO] checking: [ ok ] 'tag_TCATAGC.fasta' -> 'S3.fasta' + [INFO] checking: [ ok ] 'tag_TCTATAG.fasta' -> 'S4.fasta' + [INFO] 5 path(s) to be renamed + + 1. Renaming PDF files for compatibility (moving from `EXT4` to `NTFS` file system): - 1. Original files: + 1. Original files $ tree -Q . @@ -312,7 +384,7 @@ Take a directory for example: $ brename -R -f .pdf -i -p "^(.{30,50})[ \.].*.pdf" -r "\$1.pdf" -d - 1. Result: + 1. Result $ tree -Q . diff --git a/brename.go b/brename.go index 68c33f4..3c8e385 100644 --- a/brename.go +++ b/brename.go @@ -29,17 +29,19 @@ import ( "path/filepath" "regexp" "runtime" + "strconv" "strings" "github.com/fatih/color" "github.com/mattn/go-colorable" + "github.com/shenwei356/breader" "github.com/shenwei356/go-logging" "github.com/spf13/cobra" ) var log *logging.Logger -var version = "2.1.3" +var version = "2.2.0" var app = "brename" // Options is the struct containing all global options @@ -59,8 +61,21 @@ type Options struct { ExcludeFilters []string IncludeFilterRes []*regexp.Regexp ExcludeFilterRes []*regexp.Regexp + + ReplaceWithNR bool + StartNum int + + ReplaceWithKV bool + KVs map[string]string + KVFile string + KeepKey bool + KeyCaptIdx int + KeyMissRepl string } +var reNR = regexp.MustCompile(`\{(NR|nr)\}`) +var reKV = regexp.MustCompile(`\{(KV|kv)\}`) + func getOptions(cmd *cobra.Command) *Options { version := getFlagBool(cmd, "version") if version { @@ -74,7 +89,8 @@ func getOptions(cmd *cobra.Command) *Options { os.Exit(1) } p := pattern - if getFlagBool(cmd, "ignore-case") { + ignoreCase := getFlagBool(cmd, "ignore-case") + if ignoreCase { p = "(?i)" + p } re, err := regexp.Compile(p) @@ -113,6 +129,45 @@ func getOptions(cmd *cobra.Command) *Options { exfilterRes = append(exfilterRes, exfilterRe) } + replacement := getFlagString(cmd, "replacement") + kvFile := getFlagString(cmd, "kv-file") + + if kvFile != "" { + if len(replacement) == 0 { + checkError(fmt.Errorf("flag -r/--replacement needed when given flag -k/--kv-file")) + } + if !reKV.MatchString(replacement) { + checkError(fmt.Errorf(`replacement symbol "{kv}"/"{KV}" not found in value of flag -r/--replacement when flag -k/--kv-file given`)) + } + } + + var replaceWithNR bool + if reNR.MatchString(replacement) { + replaceWithNR = true + } + + var replaceWithKV bool + var kvs map[string]string + if reKV.MatchString(replacement) { + replaceWithKV = true + if !regexp.MustCompile(`\(.+\)`).MatchString(pattern) { + checkError(fmt.Errorf(`value of -p/--pattern must contains "(" and ")" to capture data which is used specify the KEY`)) + } + if kvFile == "" { + checkError(fmt.Errorf(`since replacement symbol "{kv}"/"{KV}" found in value of flag -r/--replacement, tab-delimited key-value file should be given by flag -k/--kv-file`)) + } + log.Infof("read key-value file: %s", kvFile) + kvs, err = readKVs(kvFile, ignoreCase) + if err != nil { + checkError(fmt.Errorf("read key-value file: %s", err)) + } + if len(kvs) == 0 { + checkError(fmt.Errorf("no valid data in key-value file: %s", kvFile)) + } + + log.Infof("%d pairs of key-value loaded", len(kvs)) + } + return &Options{ Verbose: getFlagNonNegativeInt(cmd, "verbose"), Version: version, @@ -120,14 +175,25 @@ func getOptions(cmd *cobra.Command) *Options { Pattern: pattern, PatternRe: re, - Replacement: getFlagString(cmd, "replacement"), + Replacement: replacement, Recursive: getFlagBool(cmd, "recursive"), IncludingDir: getFlagBool(cmd, "including-dir"), + IgnoreCase: ignoreCase, IncludeFilters: infilters, IncludeFilterRes: infilterRes, ExcludeFilters: infilters, ExcludeFilterRes: exfilterRes, + + ReplaceWithNR: replaceWithNR, + StartNum: getFlagNonNegativeInt(cmd, "start-num"), + ReplaceWithKV: replaceWithKV, + + KVs: kvs, + KVFile: kvFile, + KeepKey: getFlagBool(cmd, "keep-key"), + KeyCaptIdx: getFlagPositiveInt(cmd, "key-capt-idx"), + KeyMissRepl: getFlagString(cmd, "key-miss-repl"), } } @@ -147,7 +213,7 @@ func init() { RootCmd.Flags().BoolP("dry-run", "d", false, "print rename operations but do not run") RootCmd.Flags().StringP("pattern", "p", "", "search pattern (regular expression)") - RootCmd.Flags().StringP("replacement", "r", "", `replacement. capture variables supported. e.g. $1 represents the first submatch. ATTENTION: for *nix OS, use SINGLE quote NOT double quotes or use the \ escape character.`) + RootCmd.Flags().StringP("replacement", "r", "", `replacement. capture variables supported. e.g. $1 represents the first submatch. ATTENTION: for *nix OS, use SINGLE quote NOT double quotes or use the \ escape character. Ascending integer is also supported by "{nr}"`) RootCmd.Flags().BoolP("recursive", "R", false, "rename recursively") RootCmd.Flags().BoolP("including-dir", "D", false, "rename directories") RootCmd.Flags().BoolP("ignore-case", "i", false, "ignore case") @@ -155,6 +221,13 @@ func init() { RootCmd.Flags().StringSliceP("include-filters", "f", []string{"."}, `include file filter(s) (regular expression, case ignored). multiple values supported, e.g., -f ".html" -f ".htm", but ATTENTION: comma in filter is treated as separater of multiple filters`) RootCmd.Flags().StringSliceP("exclude-filters", "F", []string{}, `exclude file filter(s) (regular expression, case ignored). multiple values supported, e.g., -F ".html" -F ".htm", but ATTENTION: comma in filter is treated as separater of multiple filters`) + RootCmd.Flags().StringP("kv-file", "k", "", + `tab-delimited key-value file for replacing key with value when using "{kv}" in -r (--replacement)`) + RootCmd.Flags().BoolP("keep-key", "K", false, "keep the key as value when no value found for the key") + RootCmd.Flags().IntP("key-capt-idx", "I", 1, "capture variable index of key (1-based)") + RootCmd.Flags().StringP("key-miss-repl", "m", "", "replacement for key with no corresponding value") + RootCmd.Flags().IntP("start-num", "n", 1, `starting number when using {nr} in replacement`) + RootCmd.Example = ` 1. dry run and showing potential dangerous operations brename -p "abc" -d 2. dry run and only show operations that will cause error @@ -168,6 +241,8 @@ func init() { or brename -p "(a)" -r '$1$1' in Linux/Mac OS X 6. renaming directory too brename -p ":" -r "-" -R -D pdf-dirs + 7. using key-value file + brename -p "(.+)" -r "{kv}" -k kv.tsv More examples: https://github.com/shenwei356/brename` @@ -248,6 +323,15 @@ func getFlagStringSlice(cmd *cobra.Command, flag string) []string { return value } +func getFlagPositiveInt(cmd *cobra.Command, flag string) int { + value, err := cmd.Flags().GetInt(flag) + checkError(err) + if value <= 0 { + checkError(fmt.Errorf("value of flag --%s should be greater than 0", flag)) + } + return value +} + func getFlagNonNegativeInt(cmd *cobra.Command, flag string) int { value, err := cmd.Flags().GetInt(flag) checkError(err) @@ -284,7 +368,7 @@ var RootCmd = &cobra.Command{ Use: app, Short: "a cross-platform command-line tool for safely batch renaming files/directories via regular expression", Long: fmt.Sprintf(` -brename -- a cross-platform command-line tool for safely batch renaming files/directories via regular expression +brename -- a practical cross-platform command-line tool for safely batch renaming files/directories via regular expression Version: %s @@ -299,6 +383,13 @@ Attention: e.g., -f ".html" -f ".htm". But ATTENTION: comma in filter is treated as separater of multiple filters. +Special replacement symbols: + + {nr} Ascending integer + {kv} Corresponding value of the key (captured variable $n) by key-value file, + n can be specified by flag -I/--key-capt-idx (default: 1) + + `, version), Run: func(cmd *cobra.Command, args []string) { // var err error @@ -429,7 +520,35 @@ func checkOperation(opt *Options, path string) (bool, operation) { return false, operation{} } - filename2 := opt.PatternRe.ReplaceAllString(filename, opt.Replacement) + r := opt.Replacement + + if opt.ReplaceWithNR { + r = reNR.ReplaceAllString(r, strconv.Itoa(opt.StartNum)) + opt.StartNum++ + } + + if opt.ReplaceWithKV { + founds := opt.PatternRe.FindAllStringSubmatch(filename, -1) + if len(founds) > 0 { + found := founds[0] + if opt.KeyCaptIdx > len(found)-1 { + checkError(fmt.Errorf("value of flag -I/--key-capt-idx overflows")) + } + k := found[opt.KeyCaptIdx] + if opt.IgnoreCase { + k = strings.ToLower(k) + } + if _, ok := opt.KVs[k]; ok { + r = reKV.ReplaceAllString(r, opt.KVs[k]) + } else if opt.KeepKey { + r = reKV.ReplaceAllString(r, found[opt.KeyCaptIdx]) + } else { + r = reKV.ReplaceAllString(r, opt.KeyMissRepl) + } + } + } + + filename2 := opt.PatternRe.ReplaceAllString(filename, r) if filename2 == "" { return true, operation{path, filepath.Join(dir, filename2), codeMissingTarget} } @@ -512,3 +631,36 @@ func walk(opt *Options, opCh chan<- operation, path string) error { return nil } + +func readKVs(file string, ignoreCase bool) (map[string]string, error) { + type KV [2]string + fn := func(line string) (interface{}, bool, error) { + if len(line) == 0 { + return nil, false, nil + } + items := strings.Split(strings.TrimRight(line, "\r\n"), "\t") + if len(items) < 2 { + return nil, false, nil + } + if ignoreCase { + return KV([2]string{strings.ToLower(items[0]), items[1]}), true, nil + } + return KV([2]string{items[0], items[1]}), true, nil + } + kvs := make(map[string]string) + reader, err := breader.NewBufferedReader(file, 2, 10, fn) + if err != nil { + return kvs, err + } + var items KV + for chunk := range reader.Ch { + if chunk.Err != nil { + return kvs, err + } + for _, data := range chunk.Data { + items = data.(KV) + kvs[items[0]] = items[1] + } + } + return kvs, nil +}