From 1897aa3153350c7a3b9f5e13558c8d5cf242d145 Mon Sep 17 00:00:00 2001 From: Tordarus Date: Wed, 15 Jan 2025 12:48:20 +0100 Subject: [PATCH] initial commit --- bool_func_utils.go | 68 ++++++++++++++++++++++++++ ffmpeg.go | 41 ++++++++++++++++ go.mod | 14 ++++++ go.sum | 10 ++++ hash_cache.go | 64 ++++++++++++++++++++++++ main.go | 118 +++++++++++++++++++++++++++++++++++++++++++++ utils.go | 25 ++++++++++ watch_dir.go | 100 ++++++++++++++++++++++++++++++++++++++ 8 files changed, 440 insertions(+) create mode 100644 bool_func_utils.go create mode 100644 ffmpeg.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 hash_cache.go create mode 100644 main.go create mode 100644 utils.go create mode 100644 watch_dir.go diff --git a/bool_func_utils.go b/bool_func_utils.go new file mode 100644 index 0000000..f0b8059 --- /dev/null +++ b/bool_func_utils.go @@ -0,0 +1,68 @@ +package main + +import ( + "errors" + "io/fs" + "os" + "path/filepath" + "strings" +) + +func AbsPath(path string) string { + if filepath.IsAbs(path) { + return path + } + + absSource, err := filepath.Abs(path) + if err != nil { + panic(err) + } + + return absSource +} + +func Exists(path string) bool { + _, err := os.Lstat(path) + return !errors.Is(err, fs.ErrNotExist) +} + +func SymlinkFilesOnly(path string) bool { + stat, err := os.Lstat(path) + if err != nil { + return false + } + + isSymlink := stat.Mode().Type()&os.ModeSymlink == os.ModeSymlink + return isSymlink && !stat.IsDir() +} + +func VideosOnly(path string) bool { + streams, err := GetStreamsFromFile(path) + return err == nil && strings.Contains(streams, "video") +} + +func Not[T any](f func(T) bool) func(T) bool { + return func(t T) bool { return !f(t) } +} + +func And[T any](functions ...func(T) bool) func(T) bool { + return func(t T) bool { + for _, f := range functions { + if !f(t) { + return false + } + } + return true + } +} + +func Or[T any](functions ...func(T) bool) func(T) bool { + return func(t T) bool { + for _, f := range functions { + if f(t) { + return true + } + } + return false + } +} diff --git a/ffmpeg.go b/ffmpeg.go new file mode 100644 index 0000000..2cc66d1 --- /dev/null +++ b/ffmpeg.go @@ -0,0 +1,41 @@ +package main + +import ( + "context" + "fmt" + "os/exec" +) + +func GetStreamsFromFile(path string) (string, error) { + cmd := exec.Command( + "ffprobe", + "-loglevel", "error", + "-show_entries", "stream=codec_type", + "-of", "csv=p=0", + path, + ) + + data, err := cmd.Output() + return string(data), err +} + +func ConvertToAudio(ctx context.Context, videoFile, targetPath string) error { + cmd := exec.CommandContext( + ctx, "ffmpeg", + "-i", videoFile, + "-y", // overwrite existing file + "-vn", // no video + "-ar", "44100", // sample rate + "-ac", "2", // audio channels + "-ab", "192k", // dont know dont care + "-f", "mp3", // output format + targetPath, + ) + + data, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("error during process execution: %w\nOutput:\n%s", err, string(data)) + } + + return nil +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..9ea6f91 --- /dev/null +++ b/go.mod @@ -0,0 +1,14 @@ +module git.tordarus.net/tordarus/music-library + +go 1.23 + +toolchain go1.23.4 + +require ( + git.tordarus.net/tordarus/channel v0.1.18 + git.tordarus.net/tordarus/cmap v0.0.4 + git.tordarus.net/tordarus/envvars v0.0.0-20250114175450-d73e12b838a5 + github.com/fsnotify/fsnotify v1.7.0 +) + +require golang.org/x/sys v0.20.0 // indirect diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..218a2db --- /dev/null +++ b/go.sum @@ -0,0 +1,10 @@ +git.tordarus.net/tordarus/channel v0.1.18 h1:/9BDbkyXbVpFB+dQbToniX6g/ApBnzjslYt4NiycMQo= +git.tordarus.net/tordarus/channel v0.1.18/go.mod h1:8/dWFTdGO7g4AeSZ7cF6GerkGbe9c4dBVMVDBxOd9m4= +git.tordarus.net/tordarus/cmap v0.0.4 h1:w4J6wDrfM9xhmbMdLcLh9Ilsl6VjbDfMtn43Vux9u7Y= +git.tordarus.net/tordarus/cmap v0.0.4/go.mod h1:1durIqqr4b2/J0EcM1k2VpzAQHAqJzAWfRm7qU0fcPw= +git.tordarus.net/tordarus/envvars v0.0.0-20250114175450-d73e12b838a5 h1:rKNDX/YGunqg8TEU6q1rgS2BcDKVmUW2cg61JOE/wws= +git.tordarus.net/tordarus/envvars v0.0.0-20250114175450-d73e12b838a5/go.mod h1:/qVGwrEmqtIrZyuuoIQl4vquSkPWUNJmlGNedDrdYfg= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= diff --git a/hash_cache.go b/hash_cache.go new file mode 100644 index 0000000..55c8212 --- /dev/null +++ b/hash_cache.go @@ -0,0 +1,64 @@ +package main + +import ( + "crypto/sha512" + "encoding/gob" + "fmt" + "os" + "path/filepath" + "time" + + "git.tordarus.net/tordarus/cmap" +) + +var HashCache = cmap.New[[sha512.Size]byte, [sha512.Size]byte]() + +func InitCache() { + path := os.ExpandEnv(FlagCacheFile) + + file, err := os.Open(path) + if err == nil { + defer file.Close() + data := map[[sha512.Size]byte][sha512.Size]byte{} + gob.NewDecoder(file).Decode(&data) + HashCache.Do(func(m map[[64]byte][64]byte) { + for key, value := range data { + m[key] = value + } + }) + } + + go func() { + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + for range ticker.C { + if err := WriteCache(); err != nil { + fmt.Fprintln(os.Stderr, fmt.Errorf("could not write cache file: %w", err)) + } + } + }() +} + +func WriteCache() error { + path := os.ExpandEnv(FlagCacheFile) + + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return err + } + + file, err := os.Create(path) + if err != nil { + return err + } + defer file.Close() + + data := HashCache.Clone() + + if err := gob.NewEncoder(file).Encode(data); err != nil { + return err + } + + fmt.Println("hashsum cache written") + + return nil +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..70e0711 --- /dev/null +++ b/main.go @@ -0,0 +1,118 @@ +package main + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "git.tordarus.net/tordarus/channel" + "git.tordarus.net/tordarus/envvars" + "github.com/fsnotify/fsnotify" +) + +var ( // flags + FlagSourceDir = envvars.String("SOURCE_PATH", "/video") + FlagTargetDir = envvars.String("TARGET_PATH", "/audio") + FlagCacheFile = envvars.String("CACHE_FILE", "/hashes.bin") +) + +func main() { + if _, err := exec.LookPath("ffmpeg"); err != nil { + fmt.Println("ffmpeg not found in PATH") + os.Exit(1) + } + + FlagSourceDir = AbsPath(FlagSourceDir) + FlagTargetDir = AbsPath(FlagTargetDir) + FlagCacheFile = AbsPath(FlagCacheFile) + + InitCache() + + ops := []fsnotify.Op{ + fsnotify.Create, + fsnotify.Remove, + fsnotify.Write, + fsnotify.Rename, + } + + files, err := WatchDirectory(context.Background(), FlagSourceDir, ops...) + if err != nil { + panic(err) + } + + groupedFiles := channel.GroupByTime(files, time.Second, func(events map[string]struct{}, event string) map[string]struct{} { + if events == nil { + events = map[string]struct{}{} + } + events[event] = struct{}{} + return events + }) + + filesByGroup := channel.FlatMap(groupedFiles, func(key string, _ struct{}) string { return key }) + filteredFiles := channel.Filter(filesByGroup, Or(Not(Exists), And(SymlinkFilesOnly, VideosOnly))) + + for file := range filteredFiles { + if err := TranscodeFile(file); err != nil { + fmt.Fprintln(os.Stderr, fmt.Errorf("could not transcode file '%s': %w", file, err)) + } + } +} + +func TranscodeFile(file string) error { + relPath := strings.TrimPrefix(file, FlagSourceDir) + newPath := filepath.Join(FlagTargetDir, relPath) + newDir := filepath.Dir(newPath) + + fmt.Printf("check '%s'\n", relPath) + + // file was deleted in source directory. + // delete corresponding file in target directory as well + if _, err := os.Stat(file); errors.Is(err, fs.ErrNotExist) { + fmt.Printf("remove '%s'\n", relPath) + return os.Remove(newPath) + } + + oldFileHash, err := HashFile(file) + if err != nil { + return err + } + + if err := os.MkdirAll(newDir, 0755); err != nil { + return fmt.Errorf("creation of playlist failed: %w", err) + } + + // file does exist in target directory. + // compare hashsums to determine if file should be re-transcoded + if _, err := os.Stat(newPath); !errors.Is(err, fs.ErrNotExist) { + if expectedFileHash, exists := HashCache.GetHas(oldFileHash); exists { + currentFileHash, err := HashFile(newPath) + if err != nil { + return err + } + + if currentFileHash == expectedFileHash { + fmt.Printf("skip '%s'. hashes match\n", relPath) + return nil + } + } + } + + fmt.Printf("convert '%s'\n", relPath) + if err := ConvertToAudio(context.Background(), file, newPath); err != nil { + fmt.Fprintln(os.Stderr, fmt.Errorf("encoding of video '%s' failed: %w", file, err)) + } + + newFileHash, err := HashFile(newPath) + if err != nil { + return err + } + + HashCache.Put(oldFileHash, newFileHash) + return nil +} diff --git a/utils.go b/utils.go new file mode 100644 index 0000000..9507949 --- /dev/null +++ b/utils.go @@ -0,0 +1,25 @@ +package main + +import ( + "crypto/sha512" + "io" + "os" +) + +func HashFile(path string) ([sha512.Size]byte, error) { + file, err := os.Open(path) + if err != nil { + return [sha512.Size]byte{}, err + } + defer file.Close() + + hasher := sha512.New() + if _, err := io.Copy(hasher, file); err != nil { + return [sha512.Size]byte{}, err + } + + res := [sha512.Size]byte{} + hash := hasher.Sum(nil) + copy(res[:], hash) + return res, nil +} diff --git a/watch_dir.go b/watch_dir.go new file mode 100644 index 0000000..833a8c2 --- /dev/null +++ b/watch_dir.go @@ -0,0 +1,100 @@ +package main + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "github.com/fsnotify/fsnotify" +) + +func WatchDirectory(ctx context.Context, path string, op ...fsnotify.Op) (<-chan string, error) { + watcher, err := fsnotify.NewWatcher() + if err != nil { + return nil, err + } + + ops := fsnotify.Op(0) + for _, op := range op { + ops |= op + } + + err = watcher.Add(path) + if err != nil { + return nil, err + } + + files, err := os.ReadDir(path) + if err != nil { + return nil, err + } + + ch := make(chan string, 1000) + + // close channel on app shutdown + go func() { + <-ctx.Done() + watcher.Close() + }() + + go func(watcher *fsnotify.Watcher, ch chan<- string) { + defer watcher.Close() + defer close(ch) + + for _, file := range files { + path := filepath.Join(path, file.Name()) + if file.IsDir() { + //fmt.Println("watching directory", path) + watcher.Add(path) + + if dirFiles, err := os.ReadDir(path); err == nil { + for _, dirFile := range dirFiles { + ch <- filepath.Join(path, dirFile.Name()) + } + } + } else { + ch <- path + } + } + + for { + select { + case event, ok := <-watcher.Events: + if !ok { + close(ch) + return + } + + if fi, err := os.Stat(event.Name); err == nil && fi.IsDir() { + if event.Op&fsnotify.Create == fsnotify.Create { + //fmt.Println("watching directory", event.Name) + watcher.Add(event.Name) + } + + // read dir immediately because directory files could change simultanously with its parent directory + if dirFiles, err := os.ReadDir(event.Name); err == nil { + for _, dirFile := range dirFiles { + ch <- filepath.Join(event.Name, dirFile.Name()) + } + } + } else if err != nil && event.Op&fsnotify.Remove == fsnotify.Remove { + //fmt.Println("stopped watching directory", event.Name) + watcher.Remove(event.Name) + } + + if event.Op&ops > 0 { + ch <- event.Name + } + case err, ok := <-watcher.Errors: + if ok { + fmt.Println(err, ok) + } + close(ch) + return + } + } + }(watcher, ch) + + return ch, nil +}