SoundCloud Export & Lookup (#147)

* added soundcloud lookup

* fix create http session

* api fix

* api fix

* soudncloud track id fix

* http session fix

* http session fix

* http session fix

* added fixup

* commented fixup

* fixed func

* fixed func

* fixed func

* fixed list

* fixed list

* fixed lookup soundcloud

* fixed lookup soundcloud

* fix soundcloud export cli and unplayable check

* added pagination

* added hook

* fixed fixup playlist

* remove duplicate function

---------

Co-authored-by: Rimma Kubanova <[email protected]>
Co-authored-by: Kartik Ohri <[email protected]>

troi/__init__.py

@@ -311,6 +311,7 @@ def __init__(self,
                  ranking=None,
                  year=None,
                  spotify_id=None,
+                 soundcloud_id=None,
                  apple_music_id=None,
                  musicbrainz=None,
                  listenbrainz=None,
@@ -325,6 +326,7 @@ def __init__(self,
         self.year = year
         self.spotify_id = spotify_id
         self.apple_music_id = apple_music_id
+        self.soundcloud_id = soundcloud_id
 
     def __str__(self):
         return "<Recording('%s', %s, %s)>" % (self.name, self.mbid, self.msid)

troi/cli.py

@@ -74,9 +74,20 @@ def cli():
               type=str,
               required=False,
               multiple=True)
+@click.option('--soundcloud-user-id', help="The soundcloud user name to upload the playlist to", type=str, required=False)
+@click.option('--soundcloud-token',
+              help="The soundcloud token with the correct permissions required to upload playlists",
+              type=str,
+              required=False)
+@click.option('--soundcloud-url',
+              help="instead of creating a new soundcloud playlist, update the existing playlist at this url",
+              type=str,
+              required=False,
+              multiple=True)
 @click.argument('args', nargs=-1, type=click.UNPROCESSED)
 def playlist(patch, quiet, save, token, upload, args, created_for, name, desc, min_recordings, spotify_user_id, spotify_token,
-             spotify_url, apple_music_developer_token, apple_music_user_token, apple_music_url):
+             spotify_url, soundcloud_user_id, soundcloud_token, soundcloud_url,
+             apple_music_developer_token, apple_music_user_token, apple_music_url):
     """
     Generate a global MBID based playlist using a patch
     """
@@ -106,7 +117,6 @@ def playlist(patch, quiet, save, token, upload, args, created_for, name, desc, m
             "is_collaborative": False,
             "existing_urls": spotify_url
         }
-
     if apple_music_developer_token:
         patch_args["apple_music"] = {
             "developer_token": apple_music_developer_token,
@@ -115,6 +125,14 @@ def playlist(patch, quiet, save, token, upload, args, created_for, name, desc, m
             "is_collaborative": False,
             "existing_urls": apple_music_url
         }
+    if soundcloud_token:
+        patch_args["soundcloud"] = {
+            "user_id": soundcloud_user_id,
+            "token": soundcloud_token,
+            "is_public": True,
+            "is_collaborative": False,
+            "existing_urls": soundcloud_url
+        }
 
     if args is None:
         args = []

troi/patch.py

@@ -19,6 +19,7 @@
                           min_recordings=10,
                           spotify=None,
                           apple_music=None,
+                          soundcloud=None,
                           quiet=False)
 
 
@@ -147,6 +148,7 @@ def generate_playlist(self):
         * min-recordings: The minimum number of recordings that must be present in a playlist to consider it complete. If it doesn't have sufficient numbers of tracks, ignore the playlist and don't submit it. Default: Off, a playlist with at least one track will be considere complete.
         * spotify: if present, attempt to submit the playlist to spotify as well. should be a dict and contain the spotify user id, spotify auth token with appropriate permissions, whether the playlist should be public, private or collaborative. it can also optionally have the existing urls to update playlists instead of creating new ones.
         * apple_music: if present, attempt to submit the playlist to Apple Music as well. should be a dict and contain the apple developer token, user music token, whether the playlist should be public, private. it can also optionally have the existing urls to update playlists instead of creating new ones.
+        * soundcloud: if present, attempt to submit the palylist to soundcloud. should contain soundcloud auth token, whether the playlist should be public or private
         """
 
         try:
@@ -172,7 +174,8 @@ def generate_playlist(self):
         token = self.patch_args["token"]
         spotify = self.patch_args["spotify"]
         apple_music = self.patch_args["apple_music"]
-        if upload and not token and not spotify and not apple_music:
+        soundcloud = self.patch_args["soundcloud"]
+        if upload and not token and not spotify and not apple_music and not soundcloud:
             raise RuntimeError("In order to upload a playlist, you must provide an auth token. Use option --token.")
 
         min_recordings = self.patch_args["min_recordings"]
@@ -186,6 +189,11 @@ def generate_playlist(self):
                                                      spotify["is_collaborative"], spotify.get("existing_urls", [])):
                 logger.info("Submitted playlist to spotify: %s" % url)
 
+        if result is not None and soundcloud and upload:
+            for url, _ in playlist.submit_to_soundcloud(soundcloud["user_id"], soundcloud["token"], soundcloud["is_public"],
+                                                soundcloud.get("existing_urls", [])):
+                logger.info("Submitted playlist to soundcloud: %s" % url)
+
         if result is not None and apple_music and upload:
             for url, _ in playlist.submit_to_apple_music(apple_music["music_user_token"], apple_music["developer_token"], apple_music["is_public"], apple_music.get("existing_urls", [])):
                 logger.info("Submitted playlist to apple music: %s" % url)

troi/patches/playlist_from_ms.py

@@ -20,7 +20,7 @@ def inputs():
         MS_TOKEN is the music service token from which the playlist is retrieved. For now, only Spotify tokens are accepted. 
         PLAYLIST_ID is the playlist id to retrieve the tracks from it.
         MUSIC_SERVICE is the music service from which the playlist is retrieved
-        APPLE_USER_TOKEN is the apple user token. Optional, if music services is not Apple Music
+        APPLE_USER_TOKEN is the apple user token. Optional, if music service is not Apple Music
         """
         return [
             {"type": "argument", "args": ["ms_token"], "kwargs": {"required": False}},

troi/playlist.py

@@ -13,6 +13,8 @@
 from troi.tools.common_lookup import music_service_tracks_to_mbid
 from troi.tools.spotify_lookup import submit_to_spotify
 from troi.tools.apple_lookup import submit_to_apple_music
+from troi.tools.soundcloud_lookup import submit_to_soundcloud
+from troi.tools.utils import SoundcloudAPI
 
 logger = logging.getLogger(__name__)
 
@@ -333,7 +335,7 @@ def submit_to_apple_music(self,
         for idx, playlist in enumerate(self.playlists):
             if len(playlist.recordings) == 0:
                 continue
-            
+
             existing_url = None
             if existing_urls and idx < len(existing_urls) and existing_urls[idx]:
                 existing_url = existing_urls[idx]
@@ -343,9 +345,34 @@ def submit_to_apple_music(self,
 
         return submitted
 
+    def submit_to_soundcloud(self,
+                             user_id: str,
+                             access_token: str,
+                             is_public: bool = True,
+                             existing_urls: str = None):
+        """ Given soundcloud user id, soundcloud auth token, upload the playlists generated in the current element to Soundcloud and return the
+        urls of submitted playlists.
+
+        """
+        sd = SoundcloudAPI(access_token=access_token)
+        submitted = []
+
+        for idx, playlist in enumerate(self.playlists):
+            if len(playlist.recordings) == 0:
+                continue
+
+            existing_url = None
+            if existing_urls and idx < len(existing_urls) and existing_urls[idx]:
+                existing_url = existing_urls[idx]
+
+            playlist_url, playlist_id = submit_to_soundcloud(sd, playlist, is_public, existing_url)
+            submitted.append((playlist_url, playlist_id))
+
+        return submitted
+
 
 class PlaylistRedundancyReducerElement(Element):
-    
+
     '''
         This element takes a larger playlist and whittles it down to a smaller playlist by
         removing some tracks in order to reduce the number of times a single artist appears

troi/tools/soundcloud_lookup.py

@@ -1,22 +1,141 @@
-from .utils import create_http_session
+import requests
+import logging
+
+from collections import defaultdict
+from more_itertools import chunked
+from .utils import SoundcloudAPI, SoundCloudException
+
+logger = logging.getLogger(__name__)
+
+SOUNDCLOUD_IDS_LOOKUP_URL = "https://labs.api.listenbrainz.org/soundcloud-id-from-mbid/json"
+
+def lookup_soundcloud_ids(recordings):
+    """ Given a list of Recording elements, try to find soundcloud track ids from labs api soundcloud lookup using mbids
+    and add those to the recordings. """
+    response = requests.post(
+        SOUNDCLOUD_IDS_LOOKUP_URL,
+        json=[{"recording_mbid": recording.mbid} for recording in recordings]
+    )
+    response.raise_for_status()
+
+    soundcloud_data = response.json()
+    mbid_soundcloud_ids_index = {}
+    soundcloud_id_mbid_index = {}
+    for recording, lookup in zip(recordings, soundcloud_data):
+        if len(lookup["soundcloud_track_ids"]) > 0:
+            recording.soundcloud_id = lookup["soundcloud_track_ids"][0]
+            mbid_soundcloud_ids_index[recording.mbid] = lookup["soundcloud_track_ids"]
+            for soundcloud_id in lookup["soundcloud_track_ids"]:
+                soundcloud_id_mbid_index[soundcloud_id] = recording.mbid
+    return recordings, mbid_soundcloud_ids_index, soundcloud_id_mbid_index
+
+
+def _check_unplayable_tracks(soundcloud: SoundcloudAPI, playlist_id: str):
+    """ Retrieve tracks for given soundcloud playlist and split into lists of playable and unplayable tracks """
+    tracks = soundcloud.get_playlist_tracks(playlist_id, linked_partitioning=True, limit=100, access=['playable, preview ,blocked'])
+    track_details = [
+        {
+            "id": track["id"],
+            "title": track["title"],
+            "access": track["access"]
+        }
+        for track in tracks
+    ]
+
+    playable = []
+    unplayable = []
+    for idx, item in enumerate(track_details):
+        if item["access"] == "playable":
+            playable.append((idx, item["id"]))
+        else:
+            unplayable.append((idx, item["id"]))
+    return playable, unplayable
+
+
+def _get_alternative_track_ids(unplayable, mbid_soundcloud_id_idx, soundcloud_id_mbid_idx):
+    """ For the list of unplayable track ids, find alternative tracks ids.
+
+    mbid_soundcloud_id_idx is an index with mbid as key and list of equivalent soundcloud ids as value.
+    soundcloud_id_mbid_idx is an index with soundcloud_id as key and the corresponding mbid as value.
+    """
+    index = defaultdict(list)
+    soundcloud_ids = []
+    for idx, soundcloud_id in unplayable:
+        mbid = soundcloud_id_mbid_idx[str(soundcloud_id)]
+        other_soundcloud_ids = mbid_soundcloud_id_idx[mbid]
+
+        for new_idx, new_soundcloud_id in enumerate(other_soundcloud_ids):
+            if new_soundcloud_id == soundcloud_id:
+                continue
+            soundcloud_ids.append(new_soundcloud_id)
+            index[idx].append(new_soundcloud_id)
+
+    return soundcloud_ids, index
+
+
+def _get_fixed_up_tracks(soundcloud: SoundcloudAPI, soundcloud_ids, index):
+    """ Lookup the all alternative soundcloud track ids, filter playable ones and if multiple track ids
+        for same item match, prefer the one occurring earlier. If no alternative is playable, ignore the
+        item altogether.
+    """
+    new_tracks = soundcloud.get_track_details(soundcloud_ids)
+
+    new_tracks_ids = set()
+    for item in new_tracks:
+        if item["access"] == "playable":
+            new_tracks_ids.add(item["id"])
+
+    fixed_up_items = []
+    for idx, soundcloud_ids in index.items():
+        for soundcloud_id in soundcloud_ids:
+            if soundcloud_id in new_tracks_ids:
+                fixed_up_items.append((idx, soundcloud_id))
+                break
+    return fixed_up_items
+
+
+def fixup_soundcloud_playlist(soundcloud: SoundcloudAPI, playlist_id: str, mbid_soundcloud_id_idx, soundcloud_id_mbid_idx):
+    """ Fix unplayable tracks in the given soundcloud playlist.
+
+    Given a soundcloud playlist id, look it up and find unstreamable tracks. If there are any unstreamable tracks, try
+    alternative soundcloud track ids from index/reverse_index if available. If alternative is not available or alternatives
+    also are unplayable, remove the track entirely from the playlist. Finally, update the playlist if needed.
+    """
+    playable, unplayable = _check_unplayable_tracks(soundcloud, playlist_id)
+    if not unplayable:
+        return
+
+    alternative_ids, index = _get_alternative_track_ids(unplayable, mbid_soundcloud_id_idx, soundcloud_id_mbid_idx)
+    if not alternative_ids:
+        return
+
+    fixed_up = _get_fixed_up_tracks(soundcloud, alternative_ids, index)
+    all_items = []
+    all_items.extend(playable)
+    all_items.extend(fixed_up)
+
+    # sort all items by index value to ensure the order of tracks of original playlist is preserved
+    all_items.sort(key=lambda x: x[0])
+    # update all track ids the soundcloud playlist
+    finalized_ids = [x[1] for x in all_items]
+
+    # clear existing playlist
+    soundcloud.update_playlist_details(playlist_id, track_ids=[])
+
+    # chunking requests to avoid hitting rate limits
+    for chunk in chunked(finalized_ids, 100):
+        soundcloud.add_playlist_tracks(playlist_id, chunk)
 
-SOUNDCLOUD_URL = f"https://api.soundcloud.com/"
 
 def get_tracks_from_soundcloud_playlist(developer_token, playlist_id):
     """ Get tracks from the Soundcloud playlist.
     """
-    http = create_http_session()
+    soundcloud = SoundcloudAPI(developer_token)
+    data = soundcloud.get_playlist_tracks(playlist_id, linked_partitioning=True, limit=100, access=['playable, preview ,blocked'])
 
-    headers = {
-        "Authorization": f"OAuth {developer_token}",
-    }
-    response = http.get(f"{SOUNDCLOUD_URL}/playlists/{playlist_id}", headers=headers)
-    response.raise_for_status()
-
-    response = response.json()
-    tracks = response["tracks"]
-    name = response["title"]
-    description = response["description"]
+    tracks = data["tracks"]
+    name = data["title"]
+    description = data["description"]
 
     mapped_tracks = [
         {
@@ -27,3 +146,54 @@ def get_tracks_from_soundcloud_playlist(developer_token, playlist_id):
     ]
 
     return mapped_tracks, name, description
+
+
+def submit_to_soundcloud(soundcloud: SoundcloudAPI, playlist, is_public: bool = True,
+                    existing_url: str = None):
+    """ Submit or update an existing soundcloud playlist.
+
+    If existing urls are specified then is_public and is_collaborative arguments are ignored.
+    """
+    filtered_recordings = [r for r in playlist.recordings if r.mbid]
+
+    _, mbid_soundcloud_index, soundcloud_mbid_index = lookup_soundcloud_ids(filtered_recordings)
+    soundcloud_track_ids = [r.soundcloud_id for r in filtered_recordings if r.soundcloud_id]
+    if len(soundcloud_track_ids) == 0:
+        return None, None
+
+    logger.info("submit %d tracks" % len(soundcloud_track_ids))
+
+    playlist_id, playlist_url = None, None
+    if existing_url:
+        # update existing playlist
+        playlist_url = existing_url
+        playlist_id = existing_url.split("/")[-1]
+        try:
+            soundcloud.update_playlist_details(playlist_id=playlist_id, title=playlist.name, description=playlist.description)
+        except SoundCloudException as err:
+            # one possibility is that the user has deleted the soundcloud from playlist, so try creating a new one
+            logger.info("provided playlist url has been unfollowed/deleted by the user, creating a new one")
+            playlist_id, playlist_url = None, None
+
+    if not playlist_id:
+        # create new playlist
+        soundcloud_playlist = soundcloud.create_playlist(
+            title=playlist.name,
+            sharing=is_public,
+            description=playlist.description
+        )
+        playlist_id = soundcloud_playlist["id"]
+        playlist_url = soundcloud_playlist["permalink"]
+    else:
+        # existing playlist, clear it
+        tracks = map(lambda id: dict(id=id), [0])
+        soundcloud.update_playlist(playlist_id, tracks)
+
+    for chunk in chunked(soundcloud_track_ids, 100):
+        soundcloud.add_playlist_tracks(playlist_id, chunk)
+
+    fixup_soundcloud_playlist(soundcloud, playlist_id, mbid_soundcloud_index, soundcloud_mbid_index)
+
+    playlist.add_metadata({"external_urls": {"soundcloud": playlist_url}})
+
+    return playlist_url, playlist_id

troi/tools/spotify_lookup.py

@@ -181,11 +181,11 @@ def get_tracks_from_spotify_playlist(spotify_token, playlist_id):
     name = playlist_info["name"]
     description = playlist_info["description"]
 
-    tracks = convert_spotify_tracks_to_json(tracks)
+    tracks = _convert_spotify_tracks_to_json(tracks)
     return tracks, name, description
 
 
-def convert_spotify_tracks_to_json(spotify_tracks):
+def _convert_spotify_tracks_to_json(spotify_tracks):
     tracks = []
     for track in spotify_tracks["items"]:
         artists = track["track"].get("artists", [])