Skip to content

Commit

Permalink
convert to mp3: better concurrency allow to convert multiple tracks a…
Browse files Browse the repository at this point in the history 
…t the same time, show confirm dialog on app quit if there are some pending tasks
  • Loading branch information
@marcio199226
marcio199226 committed Dec 16, 2021
1 parent abc0d64 commit 8f02abe
Show file tree
Hide file tree
Showing 17 changed files with 241 additions and 95 deletions.
179 changes: 101 additions & 78 deletions app.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ type AppState struct {
updater *Updater
offlinePlaylistService *OfflinePlaylistService
ngrok *NgrokService
convertQueue chan GenericEntry
}

func (state *AppState) PreWailsInit(ctx context.Context) {
Expand All @@ -67,6 +68,7 @@ func (state *AppState) PreWailsInit(ctx context.Context) {
state.ngrok.Context = ctx
state.Config = state.Config.Init()
state.AppVersion = version
state.convertQueue = make(chan GenericEntry)

// configure i18n paths
gotext.Configure("/Users/oskarmarciniak/projects/golang/ytd/i18n", state.Config.Language, "default")
Expand Down Expand Up @@ -109,6 +111,7 @@ func (state *AppState) WailsInit(runtime *wails.Runtime) {
// initialize plugins
for _, plugin := range plugins {
plugin.SetWailsRuntime(runtime)
plugin.SetContext(state.Context)
plugin.SetAppConfig(state.Config)
plugin.SetAppStats(state.Stats)
}
Expand All @@ -134,24 +137,45 @@ func (state *AppState) WailsInit(runtime *wails.Runtime) {
}() */

go func() {
restart := make(chan int)
time.Sleep(3 * time.Second)
go state.convertToMp3(restart)
ticker := time.NewTicker(3 * time.Second)
pending := make(chan int, maxConvertJobs)

for {
select {
// @TODO
// qui leggiamo da newEntries dove andiamo a scrivere o dentro yt.Fetch alla fine del metodo
// oppure da dentro AddToDownload e facciamo tornare a plugin.fetch l'entry creata
// cosi se legge prima da newEntries converte per prima quella altrimenti convertira le tracce che sono in attesa da prima
/* case <-newEntries:
fmt.Println("Converto new track...")
go state.convertToMp3(restart, newEntries) */
case <-restart:
fmt.Println("Restart converting...")
go state.convertToMp3(restart)
case entry := <-state.convertQueue:
go func() {
if len(pending) == cap(pending) {
state.runtime.Events.Emit("ytd:track:convert:queued", entry)
}
pending <- 1
state.convertToMp3(ticker, &entry, true)
<-pending
}()
case <-ticker.C:
// get entries that could be converted
for _, t := range DbGetAllEntries() {
entry := t
plugin := getPluginFor(entry.Source)

if entry.Type == "track" && entry.Track.Status == TrackStatusDownladed && !entry.Track.IsConvertedToMp3 && plugin.IsTrackFileExists(entry.Track, "webm") {
// skip tracks which has failed at least 3 times in a row
if entry.Track.ConvertingStatus.Attempts >= 3 {
fmt.Printf("Skipping audio extraction for %s(%s)...due to too many attempts\n", entry.Track.Name, entry.Track.ID)
continue
}

go func() {
// this will block until pending channel is full
pending <- 1
state.convertToMp3(ticker, &entry, false)
<-pending
}()
}
}
default:
fmt.Printf("Convert to mp3 nothing to do....max parralel %d | converting %d | pending %d entries\n\n", maxConvertJobs, state.Stats.ConvertingCount, len(pending))
time.Sleep(1 * time.Second)
}

}
}()

Expand Down Expand Up @@ -312,85 +336,79 @@ func (state *AppState) checkForTracksToDownload() error {
return nil
}

func (state *AppState) convertToMp3(restart chan<- int) error {
func (state *AppState) convertToMp3(restartTicker *time.Ticker, entry *GenericEntry, force bool) error {
if !state.Config.ConvertToMp3 {
// if option is not enabled restart check after 30s
time.Sleep(60 * time.Second)
restart <- 1
// if option is not enabled restart check after 3s
restartTicker.Stop()
restartTicker.Reset(3 * time.Second)
return nil
}

fmt.Println("Converting....")
ffmpeg, _ := state.IsFFmpegInstalled()
plugin := getPluginFor(entry.Source)

for _, t := range DbGetAllEntries() {
entry := t
plugin := getPluginFor(entry.Source)
if entry.Type == "track" && entry.Track.Status == TrackStatusDownladed && !entry.Track.IsConvertedToMp3 && plugin.IsTrackFileExists(entry.Track, "webm") {
// skip tracks which has failed at least 3 times in a row
if entry.Track.ConvertingStatus.Attempts >= 3 && !force {
fmt.Printf("Skipping audio extraction for %s - (%s)...due to too many attempts\n", entry.Track.Name, entry.Track.ID)
return nil
}

if entry.Type == "track" && entry.Track.Status == TrackStatusDownladed && !entry.Track.IsConvertedToMp3 && plugin.IsTrackFileExists(entry.Track, "webm") {
// skip tracks which has failed at least 3 times in a row
if entry.Track.ConvertingStatus.Attempts >= 3 {
fmt.Printf("Skipping audio extraction for %s(%s)...due to too many attempts\n", entry.Track.Name, entry.Track.ID)
return nil
fmt.Printf("Extracting audio for %s - (%s)...\n", entry.Track.Name, entry.Track.ID)
entry.Track.ConvertingStatus.Status = TrakcConverting
DbWriteEntry(entry.Track.ID, entry)
state.Stats.IncConvertCount()
state.runtime.Events.Emit("ytd:track", entry)

// ffmpeg -i "41qC3w3UUkU.webm" -vn -ab 128k -ar 44100 -y "41qC3w3UUkU.mp3"
outputPath := fmt.Sprintf("%s/%s.mp3", plugin.GetDir(), entry.Track.ID)
cmd := exec.CommandContext(
state.Context,
ffmpeg,
"-loglevel", "quiet",
"-i", fmt.Sprintf("%s/%s.webm", plugin.GetDir(), entry.Track.ID),
"-vn",
"-ab", "128k",
"-ar", "44100",
"-y", outputPath,
)
cmd.Stdin = os.Stdin
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr

if err := cmd.Run(); err != nil {
fmt.Println("Failed to extract audio:", err)
entry.Track.ConvertingStatus.Status = TrakcConvertFailed
entry.Track.ConvertingStatus.Err = err.Error()
entry.Track.ConvertingStatus.Attempts += 1
DbWriteEntry(entry.Track.ID, entry)
state.Stats.DecConvertCount()
state.runtime.Events.Emit("ytd:track", entry)
} else {
entry.Track.ConvertingStatus.Status = TrakcConverted
entry.Track.IsConvertedToMp3 = true

// check new filesize and save it
fileInfo, err := os.Stat(outputPath)
if err == nil {
entry.Track.ConvertingStatus.Filesize = int(fileInfo.Size())
}

fmt.Printf("Extracting audio for %s...\n", entry.Track.Name)
entry.Track.ConvertingStatus.Status = TrakcConverting
DbWriteEntry(entry.Track.ID, entry)
state.Stats.DecConvertCount()
state.runtime.Events.Emit("ytd:track", entry)

// ffmpeg -i "41qC3w3UUkU.webm" -vn -ab 128k -ar 44100 -y "41qC3w3UUkU.mp3"
outputPath := fmt.Sprintf("%s/%s.mp3", plugin.GetDir(), entry.Track.ID)
cmd := exec.Command(
ffmpeg,
"-loglevel", "quiet",
"-i", fmt.Sprintf("%s/%s.webm", plugin.GetDir(), entry.Track.ID),
"-vn",
"-ab", "128k",
"-ar", "44100",
"-y", outputPath,
)
cmd.Stdin = os.Stdin
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr

if err := cmd.Run(); err != nil {
fmt.Println("Failed to extract audio:", err)
entry.Track.ConvertingStatus.Status = TrakcConvertFailed
entry.Track.ConvertingStatus.Err = err.Error()
entry.Track.ConvertingStatus.Attempts += 1
DbWriteEntry(entry.Track.ID, entry)
state.runtime.Events.Emit("ytd:track", entry)

} else {
entry.Track.ConvertingStatus.Status = TrakcConverted
entry.Track.IsConvertedToMp3 = true

// check new filesize and save it
fileInfo, err := os.Stat(outputPath)
if err == nil {
entry.Track.ConvertingStatus.Filesize = int(fileInfo.Size())
}

DbWriteEntry(entry.Track.ID, entry)
state.runtime.Events.Emit("ytd:track", entry)

// remove webm if needed
if state.Config.CleanWebmFiles && plugin.IsTrackFileExists(entry.Track, "webm") {
err = os.Remove(fmt.Sprintf("%s/%s.webm", plugin.GetDir(), entry.Track.ID))
if err != nil && !os.IsNotExist(err) {
fmt.Printf("Cannot remove %s.webm file after successfull converting to mp3\n", entry.Track.ID)
}
// remove webm if needed
if state.Config.CleanWebmFiles && plugin.IsTrackFileExists(entry.Track, "webm") {
err = os.Remove(fmt.Sprintf("%s/%s.webm", plugin.GetDir(), entry.Track.ID))
if err != nil && !os.IsNotExist(err) {
fmt.Printf("Cannot remove %s.webm file after successfull converting to mp3\n", entry.Track.ID)
}
}
restart <- 1
return nil
}
return nil
}

// if there are no tracks to convert delay between restart
time.Sleep(15 * time.Second)
restart <- 1
return nil
}

Expand Down Expand Up @@ -616,10 +634,10 @@ func (state *AppState) AddToDownload(url string, isFromClipboard bool) error {
if support := plugin.Supports(url); support {
if appState.GetAppConfig().ConcurrentDownloads {
go func() {
plugin.Fetch(url, isFromClipboard)
newEntries <- plugin.Fetch(url, isFromClipboard)
}()
} else {
plugin.Fetch(url, isFromClipboard)
newEntries <- plugin.Fetch(url, isFromClipboard)
}
continue
}
Expand Down Expand Up @@ -654,6 +672,11 @@ func (state *AppState) StartDownload(record map[string]interface{}) error {
return nil
}

func (state *AppState) AddToConvertQueue(entry GenericEntry) error {
state.convertQueue <- entry
return nil
}

func (state *AppState) IsSupportedUrl(url string) bool {
for _, plugin := range plugins {
if support := plugin.Supports(url); support {
Expand Down
5 changes: 5 additions & 0 deletions constants/constants.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,14 @@ const (
TrakcConverting = "converting"
TrakcConverted = "converted"
TrakcConvertFailed = "failed"
TrakcConvertQueued = "queued"

NgrokStatusRunning = "running"
NgrokStatusError = "error"
NgrokStatusTimeout = "timeout"
NgrokStatusStopped = "killed"

NotificationTypeSuccess = "success"
NotificationTypeWarning = "warning"
NotificationTypeError = "error"
)
5 changes: 5 additions & 0 deletions frontend/src/app/common/custom-icons.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,9 @@ export function RegisterCustomIcons(matIconRegistry: MatIconRegistry, domSanitiz
'qrcode_scan',
domSanitizer.bypassSecurityTrustResourceUrl('http://localhost:8080/static/frontend/dist/assets/icons/qrcode-scan.svg')
)

matIconRegistry.addSvgIcon(
'download_failed',
domSanitizer.bypassSecurityTrustResourceUrl('http://localhost:8080/static/frontend/dist/assets/icons/download-failed.svg')
)
}
1 change: 1 addition & 0 deletions frontend/src/app/models/app-state.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ export interface BackendCallbacks {
SaveSettingBoolValue: (name: string, val: boolean) => Promise<any>;
SaveSettingValue: (name: string, val: string) => Promise<any>;
RemoveEntry: (entry: Entry) => Promise<any>;
AddToConvertQueue: (entry: Entry) => Promise<any>;
IsFFmpegInstalled: () => Promise<boolean>;
OpenUrl: (url: string) => Promise<any>;
ReloadNewLanguage: () => Promise<void>;
Expand Down
2 changes: 1 addition & 1 deletion frontend/src/app/models/track.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ export interface Track {
thumbnails: string[];
isConvertedToMp3: boolean;
converting: {
status: "converting" | "converted" | "failed";
status: "converting" | "converted" | "queued" | "failed";
error: string;
attempts: number;
}
Expand Down
29 changes: 26 additions & 3 deletions frontend/src/app/pages/home/home.component.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,14 @@
<mat-icon>delete</mat-icon>
{{ ('HOME.REMOVE.' + entry.type.toUpperCase()) | translate }}
</button>
<button mat-menu-item *ngIf="entry.track.status === 'failed'">Download</button>
<button mat-menu-item (click)="startDownload(entry)" *ngIf="entry.track.status === 'failed'">
<mat-icon>file_download</mat-icon>
Download
</button>
<button mat-menu-item (click)="convertTrack(entry)" *ngIf="entry.track.converting.status === 'failed'">
<mat-icon>file_download</mat-icon>
Convert to mp3
</button>
<button mat-menu-item (click)="openInYt(entry.track.url)">
<mat-icon>open_in_browser</mat-icon>
{{ 'HOME.OPEN_ON_YT' | translate }}
Expand All @@ -97,16 +104,26 @@
[class.lastOfRow]="(idx + 1) % 4 === 0"
[class.playing]="inPlaybackTrackId === entry.track?.id"
[class.downloading]="entry.track.status === 'downloading'"
[class.converting]="entry.track.converting.status === 'converting'"
[class.converting]="['converting', 'queued'].indexOf(entry.track.converting.status) > -1"
[ngClass]="entry.type"
*ngFor="let entry of filteredEntries; trackBy: trackById; let idx = index;"
(mouseenter.silent)="onMouseEnter($event, entry)"
(mouseleave.silent)="onMouseLeave($event, entry)">
<div class="bg" [ngStyle]="{'background': getBgUrl(entry)}"></div>
<div class="title">{{ entry.type === 'track' ? entry.track.name : entry.playlist.name }}</div>
<div class="errors" *ngIf="entry.track.status === 'failed' || entry.track.converting.status === 'failed'">
<mat-icon
svgIcon="download_failed"
[matTooltip]="entry.track.statusError">
error
</mat-icon>
<mat-icon [matTooltip]="entry.track.converting.error">
error
</mat-icon>
</div>
<div class="fg">
<div class="wrapper">
<ng-container *ngIf="entry.track.status === 'downloaded' && entry.track.converting.status !== 'converting'">
<ng-container *ngIf="entry.track.status === 'downloaded' && ['converting', 'queued'].indexOf(entry.track.converting.status) === -1">
<mat-icon class="clickable" title="{{ 'HOME.PLAYLISTS.ADD_TRACK' | translate }}" (click)="addToPlaylist(entry)">playlist_add</mat-icon>
<mat-icon class="clickable play" (click)="playback(entry)" title="{{ 'PLAYER.PLAY' | translate }}">play_circle_filled</mat-icon>
<mat-icon class="clickable stop" (click)="stop(entry)" title="{{ 'PLAYER.STOP' | translate }}">stop</mat-icon>
Expand Down Expand Up @@ -147,6 +164,12 @@
<mat-progress-bar mode="indeterminate"></mat-progress-bar>
</div>
</ng-container>
<ng-container *ngIf="entry.track.converting.status === 'queued'">
<div fxLayout="column" fxLayoutAlign="center center" fxLayoutGap="4px" fxFlex>
<span>{{ 'HOME.WAITING_TO_CONVERT_MP3' | translate }}</span>
<mat-progress-bar mode="indeterminate"></mat-progress-bar>
</div>
</ng-container>
</div>
</div>
</div>
Expand Down
17 changes: 17 additions & 0 deletions frontend/src/app/pages/home/home.component.scss
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,23 @@ app-home {
min-height: 52px;
}

.errors {
position: absolute;
top: 5px;
left: 0px;
z-index: 1000;
display: flex;
justify-content: space-between;
width: calc(100% - 16px);
padding: 0px 8px;


.mat-icon {
cursor: pointer;
color: #fff;
}
}

.fg {
display: none;

Expand Down
Loading

0 comments on commit 8f02abe

Please sign in to comment.